* Bobo Browse Engine - High performance faceted/parametric search implementation

* that handles various types of semi-structured data.  Written in Java.

*

* Copyright (C) 2005-2006  John Wang

*

* This library is free software; you can redistribute it and/or

* modify it under the terms of the GNU Lesser General Public

* License as published by the Free Software Foundation; either

* version 2.1 of the License, or (at your option) any later version.

*

* This library is distributed in the hope that it will be useful,

* but WITHOUT ANY WARRANTY; without even the implied warranty of

* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

* Lesser General Public License for more details.

*

* You should have received a copy of the GNU Lesser General Public

* License along with this library; if not, write to the Free Software

* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

*

* To contact the project administrators for the bobo-browse project,

* please go to https://sourceforge.net/projects/bobo-browse/, or

* send mail to owner@browseengine.com.

*/

  static Logger log = Logger.getLogger(BoboTestCase.class);

  private Directory _indexDir;

  private List<FacetHandler<?>> _fconf;

  static final private Term tagSizePayloadTerm = new Term("tagSizePayload", "size");

 

  private static class TestDataDigester extends DataDigester {

    private List<FacetHandler<?>> _fconf;

    private Document[] _data;

    TestDataDigester(List<FacetHandler<?>> fConf,Document[] data){

      super();

      _fconf=fConf;

      _data=data;

    }

    @Override

    public void digest(DataHandler handler) throws IOException {

      for (int i=0;i<_data.length;++i){

        handler.handleDocument(_data[i]);

      }

    }

  }

 

  public BoboTestCase(String testName){

    super(testName);

    String confdir = System.getProperty("conf.dir");

    if (confdir == null) confdir ="./resource";

    org.apache.log4j.PropertyConfigurator.configure(confdir+"/log4j.properties");

    _fconf=buildFieldConf();

    _indexDir=createIndex();

  }

 

  private BoboIndexReader newIndexReader() throws IOException{

    return newIndexReader(true);

  }

 

  private BoboIndexReader newIndexReader(boolean readonly) throws IOException{

    IndexReader srcReader=IndexReader.open(_indexDir,readonly);

      try{

        BoboIndexReader reader= BoboIndexReader.getInstance(srcReader,_fconf, null);

        return reader;

      }

      catch(IOException ioe){

        if (srcReader!=null){

          srcReader.close();

        }

        throw ioe;

      }

  }

  private BoboBrowser newBrowser() throws IOException{

    return new BoboBrowser(newIndexReader());

  }

   

  public static Field buildMetaField(String name,String val)

  {

    Field f = new Field(name,val,Field.Store.NO,Index.NOT_ANALYZED_NO_NORMS);

    f.setOmitTermFreqAndPositions(true);

    return f;

  }

 

  static class MetaTokenStream extends TokenStream {

        private boolean returnToken = false;

        private PayloadAttribute payloadAttr;

        private TermAttribute termAttr;

        MetaTokenStream(Term term,int size) {

          byte[] buffer = new byte[4];

          buffer[0] = (byte) (size);

          buffer[1] = (byte) (size >> 8);

          buffer[2] = (byte) (size >> 16);

          buffer[3] = (byte) (size >> 24);

          payloadAttr = (PayloadAttribute)addAttribute(PayloadAttribute.class);

          payloadAttr.setPayload(new Payload(buffer));

          termAttr = (TermAttribute)addAttribute(TermAttribute.class);

          termAttr.setTermBuffer(term.text());

          returnToken = true;

        }

        @Override

    public boolean incrementToken() throws IOException {

          if (returnToken) {

                returnToken = false;

                return true;

              } else {

                return false;

              }

    } 

    }

 

    public static Field buildMetaSizePayloadField(final Term term, final int size)

    {

      Field f = new Field(term.field(), new MetaTokenStream(term,size));

      return f;

    }

 

 

  public static Document[] buildData(){

    ArrayList<Document> dataList=new ArrayList<Document>();

   

    Document d1=new Document();

    d1.add(buildMetaField("id","1"));

    d1.add(buildMetaField("shape","square"));

    d1.add(buildMetaField("color","red"));

    d1.add(buildMetaField("size","4"));

    d1.add(buildMetaField("location","toy/lego/block/"));

    d1.add(buildMetaField("tag","rabbit"));

    d1.add(buildMetaField("tag","pet"));

        d1.add(buildMetaField("tag","animal"));

        d1.add(buildMetaSizePayloadField(tagSizePayloadTerm,3));

    d1.add(buildMetaField("number","0010"));

    d1.add(buildMetaField("date","2000/01/01"));

    d1.add(buildMetaField("name","ken"));

    d1.add(buildMetaField("char","k"));

    d1.add(buildMetaField("multinum","001"));

    d1.add(buildMetaField("multinum","003"));

    d1.add(buildMetaField("compactnum","001"));

    d1.add(buildMetaField("compactnum","003"));

    d1.add(buildMetaField("numendorsers","000003"));

    d1.add(buildMetaField("path","a-b"));

    d1.add(buildMetaField("multipath","a-b"));

    d1.add(buildMetaField("custom","000003"));

    d1.add(buildMetaField("latitude", "60"));

    d1.add(buildMetaField("longitude", "120"));

    d1.add(buildMetaField("salary", "04500"));

   

    Field sf = new Field("testStored","stored",Store.YES,Index.NOT_ANALYZED_NO_NORMS);

    sf.setOmitTermFreqAndPositions(true);

    d1.add(sf);

   

    Document d2=new Document();

    d2.add(buildMetaField("id","2"));

    d2.add(buildMetaField("shape","rectangle"));

    d2.add(buildMetaField("color","red"));

    d2.add(buildMetaField("size","2"));

    d2.add(buildMetaField("location","toy/lego/block/"));

    d2.add(buildMetaField("tag","dog"));

    d2.add(buildMetaField("tag","pet"));

    d2.add(buildMetaField("tag","poodle"));

        d2.add(buildMetaSizePayloadField(tagSizePayloadTerm,3));

    d2.add(buildMetaField("number","0011"));

    d2.add(buildMetaField("date","2003/02/14"));

    d2.add(buildMetaField("name","igor"));

    d2.add(buildMetaField("char","i"));

    d2.add(buildMetaField("multinum","002"));

    d2.add(buildMetaField("multinum","004"));

    d2.add(buildMetaField("compactnum","002"));

    d2.add(buildMetaField("compactnum","004"));

    d2.add(buildMetaField("numendorsers","000010"));

    d2.add(buildMetaField("path","a-c-d"));

    d2.add(buildMetaField("multipath","a-c-d"));

    d2.add(buildMetaField("multipath","a-b"));

    d2.add(buildMetaField("custom","000010"));

    d2.add(buildMetaField("latitude", "50"));

    d2.add(buildMetaField("longitude", "110"));

    d2.add(buildMetaField("salary", "08500"));

   

    Document d3=new Document();

    d3.add(buildMetaField("id","3"));

    d3.add(buildMetaField("shape","circle"));

    d3.add(buildMetaField("color","green"));

    d3.add(buildMetaField("size","3"));

    d3.add(buildMetaField("location","toy/lego/"));

    d3.add(buildMetaField("tag","rabbit"));

    d3.add(buildMetaField("tag","cartoon"));

    d3.add(buildMetaField("tag","funny")); 

        d3.add(buildMetaSizePayloadField(tagSizePayloadTerm,3));

    d3.add(buildMetaField("number","0230"));

    d3.add(buildMetaField("date","2001/12/25"));

    d3.add(buildMetaField("name","john"));

    d3.add(buildMetaField("char","j"));

    d3.add(buildMetaField("multinum","007"));

    d3.add(buildMetaField("multinum","012"));

    d3.add(buildMetaField("compactnum","007"));

    d3.add(buildMetaField("compactnum","012"));

    d3.add(buildMetaField("numendorsers","000015"));

    d3.add(buildMetaField("path","a-e"));

    d3.add(buildMetaField("multipath","a-e"));

    d3.add(buildMetaField("multipath","a-b"));

    d3.add(buildMetaField("custom","000015"));

    d3.add(buildMetaField("latitude", "35"));

    d3.add(buildMetaField("longitude", "70"));

    d3.add(buildMetaField("salary", "06500"));

   

    Document d4=new Document();

    d4.add(buildMetaField("id","4"));

    d4.add(buildMetaField("shape","circle"));

    d4.add(buildMetaField("color","blue"));

    d4.add(buildMetaField("size","1"));

    d4.add(buildMetaField("location","toy/"));

    d4.add(buildMetaField("tag","store"));

    d4.add(buildMetaField("tag","pet"));

    d4.add(buildMetaField("tag","animal"));   

        d4.add(buildMetaSizePayloadField(tagSizePayloadTerm,3));

    d4.add(buildMetaField("number","0913"));

    d4.add(buildMetaField("date","2004/11/24"));

    d4.add(buildMetaField("name","cathy"));

    d4.add(buildMetaField("char","c"));

    d4.add(buildMetaField("multinum","007"));

    d4.add(buildMetaField("multinum","007"));

    d4.add(buildMetaField("compactnum","007"));

    d4.add(buildMetaField("numendorsers","000019"));

    d4.add(buildMetaField("path","a-c"));

    d4.add(buildMetaField("multipath","a-c"));

    d4.add(buildMetaField("multipath","a-b"));

    d4.add(buildMetaField("custom","000019"));

    d4.add(buildMetaField("latitude", "30"));

    d4.add(buildMetaField("longitude", "75"));

    d4.add(buildMetaField("salary", "11200"));

   

    Document d5=new Document();

    d5.add(buildMetaField("id","5"));

    d5.add(buildMetaField("shape","square"));

    d5.add(buildMetaField("color","blue"));

    d5.add(buildMetaField("size","5"));

    d5.add(buildMetaField("location","toy/lego/"));

    d5.add(buildMetaField("tag","cartoon"));

    d5.add(buildMetaField("tag","funny"));

    d5.add(buildMetaField("tag","disney")); 

        d5.add(buildMetaSizePayloadField(tagSizePayloadTerm,3));

    d5.add(buildMetaField("number","1013"));

    d5.add(buildMetaField("date","2002/03/08"));

    d5.add(buildMetaField("name","mike"));

    d5.add(buildMetaField("char","m"));

    d5.add(buildMetaField("multinum","001"));

    d5.add(buildMetaField("multinum","001"));

    d5.add(buildMetaField("compactnum","001"));

    d5.add(buildMetaField("compactnum","001"));

    d5.add(buildMetaField("numendorsers","000002"));

    d5.add(buildMetaField("path","a-e-f"));

    d5.add(buildMetaField("multipath","a-e-f"));

    d5.add(buildMetaField("multipath","a-b"));

    d5.add(buildMetaField("custom","000002"));

    d5.add(buildMetaField("latitude", "60"));

    d5.add(buildMetaField("longitude", "120"));

    d5.add(buildMetaField("salary", "10500"));

   

    Document d6=new Document();

    d6.add(buildMetaField("id","6"));

    d6.add(buildMetaField("shape","rectangle"));

    d6.add(buildMetaField("color","green"));

    d6.add(buildMetaField("size","6"));

    d6.add(buildMetaField("location","toy/lego/block/"));

    d6.add(buildMetaField("tag","funny"));

    d6.add(buildMetaField("tag","humor"));

    d6.add(buildMetaField("tag","joke"));   

        d6.add(buildMetaSizePayloadField(tagSizePayloadTerm,3));

    d6.add(buildMetaField("number","2130"));

    d6.add(buildMetaField("date","2007/08/01"));

    d6.add(buildMetaField("name","doug"));

    d6.add(buildMetaField("char","d"));

    d6.add(buildMetaField("multinum","001"));

    d6.add(buildMetaField("multinum","002"));

    d6.add(buildMetaField("multinum","003"));

    d6.add(buildMetaField("compactnum","001"));

    d6.add(buildMetaField("compactnum","002"));

    d6.add(buildMetaField("compactnum","003"));

    d6.add(buildMetaField("numendorsers","000009"));

    d6.add(buildMetaField("path","a-c-d"));

    d6.add(buildMetaField("multipath","a-c-d"));

    d6.add(buildMetaField("multipath","a-b"));

    d6.add(buildMetaField("custom","000009"));

    d6.add(buildMetaField("latitude", "80"));

    d6.add(buildMetaField("longitude", "-90"));

    d6.add(buildMetaField("salary", "08900"));

   

    Document d7=new Document();

    d7.add(buildMetaField("id","7"));

    d7.add(buildMetaField("shape","square"));

    d7.add(buildMetaField("color","red"));

    d7.add(buildMetaField("size","7"));

    d7.add(buildMetaField("location","toy/lego/"));

    d7.add(buildMetaField("tag","humane"));

    d7.add(buildMetaField("tag","dog"));

    d7.add(buildMetaField("tag","rabbit")); 

        d7.add(buildMetaSizePayloadField(tagSizePayloadTerm,3));

    d7.add(buildMetaField("number","0005"));

    d7.add(buildMetaField("date","2006/06/01"));

    d7.add(buildMetaField("name","abe"));

    d7.add(buildMetaField("char","a"));

    d7.add(buildMetaField("multinum","008"));

    d7.add(buildMetaField("multinum","003"));

    d7.add(buildMetaField("compactnum","008"));

    d7.add(buildMetaField("compactnum","003"));

    d7.add(buildMetaField("numendorsers","000013"));

    d7.add(buildMetaField("path","a-c"));

    d7.add(buildMetaField("multipath","a-c"));

    d7.add(buildMetaField("multipath","a-b"));

    d7.add(buildMetaField("custom","000013"));

    d7.add(buildMetaField("latitude", "70"));

    d7.add(buildMetaField("longitude", "-60"));

    d7.add(buildMetaField("salary", "28500"));

   

    Document d8 = new Document();

    d8.add(buildMetaField("latitude", "35"));

    d8.add(buildMetaField("longitude", "120"));

    d8.add(buildMetaField("salary", "00120"));

   

    dataList.add(d1);

    dataList.add(d2);

    dataList.add(d3);

    dataList.add(d4);

    dataList.add(d5);

    dataList.add(d6);

    dataList.add(d7);

    dataList.add(d8);

   

    return dataList.toArray(new Document[dataList.size()]);

  }

 

  private Directory createIndex(){

    RAMDirectory idxDir=new RAMDirectory();

   

    try {

      Document[] data=buildData();

     

      TestDataDigester testDigester=new TestDataDigester(_fconf,data);

      BoboIndexer indexer=new BoboIndexer(testDigester,idxDir);

      indexer.index();

      IndexReader r = IndexReader.open(idxDir,false);

      r.deleteDocument(r.maxDoc() - 1);

      //r.flush();

      r.close();

    } catch (UnsupportedEncodingException e) {

      e.printStackTrace();

    } catch (IOException e) {

      e.printStackTrace();

    }