Examples of org.apache.lucene.index.MultiReader

org.apache.lucene.index.MultiReader
An IndexReader which reads multiple indexes, appending their content.

          }
        }
      }
      if (indexes.size() == 0) throw new Exception("No input indexes.");
      IndexReader[] readers = (IndexReader[])indexes.toArray(new IndexReader[0]);
      reader = new MultiReader(readers);
    }
    if (LOG.isInfoEnabled()) {
      LOG.info(dr + "Opened " + numIdx + " index(es) with total " +
               reader.numDocs() + " documents.");
    }

View Full Code Here

    this.conf = conf;
    this.fs = FileSystem.get(conf);
    for (int i = 0; i < indexDirs.length; i++) {
      readers[i] = IndexReader.open(getDirectory(indexDirs[i]));
    }
    init(new MultiReader(readers), conf);
  }

View Full Code Here

        //try {
            IndexReader[] readers = new IndexReader[children.size()];
            for (int i = 0; i < readers.length; i++) {
                readers[i] = children.get(i).getReader();
            }
            reader = new MultiReader(readers);
            return reader;
        //} catch (IOException e) {
        //    logger.error(e,e);
        //    throw new RuntimeException("Error while getting reader: " + e.getMessage(),e);
        //}

View Full Code Here

  }


  private static int countHits(ZoieSystem<IndexReader,String, DefaultZoieVersion> idxSystem, Query q) throws IOException
  {
    Searcher searcher = null;
    MultiReader reader= null;
    List<ZoieIndexReader<IndexReader>> readers = null;
    try
    {
      readers=idxSystem.getIndexReaders();
      reader = new MultiReader(readers.toArray(new IndexReader[readers.size()]),false);


      searcher=new IndexSearcher(reader);


      TopDocs hits = searcher.search(q,10);


      return hits.totalHits;
    }
    finally
    {
      try{
        if (searcher != null){
          searcher.close();
          searcher = null;
          reader.close();
          reader = null;
        }
      }
      finally{
        idxSystem.returnIndexReaders(readers);

View Full Code Here


    memoryProvider.flush();
    idxSystem.syncWthVersion(10000, zvt2);
    List<ZoieIndexReader<IndexReader>> readers = null;
    Searcher searcher =null;
    MultiReader reader = null;
    try
    {     
      readers=idxSystem.getIndexReaders();
      reader = new MultiReader(readers.toArray(new IndexReader[readers.size()]),false);
      for(int i=0; i<readers.size(); i++)
      {
        IndexReader ir = readers.get(i);
        //Map<String, String> commitData = reader.getCommitUserData(ir.directory()); // = new HashMap<String,String>();
        //System.out.println("ZoieTest: directory: " + ir.directory());
        //System.out.println("ZoieTest: commitData(ZoieVersion): " + commitData);  
      }
      //Map<String, String> commitData = reader.getCommitUserData(reader.directory());// = new HashMap<String,String>();
//      for(int i=0; i<readers.size(); i++)
//      {
//        IndexReader ir = readers.get(i);
//        Map<String, String> commitData = IndexReader.getCommitUserData(ir.directory()); // = new HashMap<String,String>();
//        System.out.println("i:" + i + "ZoieTest: directory: " + ir.directory());
//        System.out.println("i:" + i + "ZoieTest: commitData(ZoieVersion): " + commitData);  
//      }
     
      //Map<String, String> commitData = IndexReader.getCommitUserData(reader.directory());// = new HashMap<String,String>();
      //System.out.println("ZoieTest:commitData" + commitData);  
     
      //commitData = reader.getCommitUserData();      
      
//int x = reader.maxDoc();
//for(int y = 0; y<x; y++)
//{
//  Document d = reader.document(y);
//  System.out.println(d.toString());  
//}


//TermEnum terms = reader.terms();
//while(terms.next())
//{
//  System.out.println(terms.term().text());
//  TermDocs td = reader.termDocs(terms.term());
//  while(td.next())
//  {
//    System.out.println(td.doc());
//  }
//}


//terms = reader.terms();
//while(terms.next())
//{
//  System.out.println("term:" + terms.term().text());
//  TermPositions tp = reader.termPositions(terms.term());
//  
//  while(tp.next())
//  {
//    System.out.println("docID: " + tp.doc() + "freq: " + tp.freq());
//    System.out.println("positions");
//    for(int i=0; i<tp.freq();i++)
//    {
//      System.out.println(tp.nextPosition());
//    }
//  }
//}
      searcher=new IndexSearcher(reader);


      TopDocs hits=searcher.search(new TermQuery(new Term("contents","hao,yan")),10);


      assertEquals(1,hits.totalHits);
      //assertEquals(String.valueOf((long)((long)Integer.MAX_VALUE*2L)),searcher.doc(hits.scoreDocs[0].doc).get("id"));
     assertEquals(String.valueOf((long)((long)Integer.MAX_VALUE*2L+1L)),searcher.doc(hits.scoreDocs[0].doc).get("id"));


      hits=searcher.search(new TermQuery(new Term("contents","hao")),10);
      assertEquals(1,hits.totalHits);
      //assertEquals(String.valueOf((long)((long)Integer.MAX_VALUE*2L)),searcher.doc(hits.scoreDocs[0].doc).get("id"));
      System.out.println("hao: searching is successful");
    }
    finally
    {
      try{
        if (searcher != null){
          searcher.close();
          searcher = null;
          reader.close();
          reader = null;
        }
      }
      finally{
        idxSystem.returnIndexReaders(readers);

View Full Code Here

      int[] expected = new int[repeat];
      Arrays.fill(expected, count);


      // should be consumed by the idxing system
      Searcher searcher=null;
      MultiReader reader= null;
      List<ZoieIndexReader<IndexReader>> readers = null;
      for (int i=0;i<repeat;++i)
      {
        try
        {
          readers=idxSystem.getIndexReaders();
          reader=new MultiReader(readers.toArray(new IndexReader[readers.size()]),false);


//          for(int j=0; j<readers.size(); j++)
//          {
//            IndexReader ir = readers.get(j);
//            Map<String, String> commitData = IndexReader.getCommitUserData(ir.directory()); // = new HashMap<String,String>();
//            System.out.println("j:" + j + "ZoieTest: directory: " + ir.directory());
//            System.out.println("j:" + j + "ZoieTest: commitData(ZoieVersion): " + commitData);  
//          }
          
          searcher=new IndexSearcher(reader);


          TopDocs hits=searcher.search(q,10);
          results[idx++] = hits.totalHits;


        }
        finally
        {
          try{
            if (searcher != null){
              searcher.close();
              searcher = null;
              reader.close();
              reader = null;
            }
          }
          finally{
            idxSystem.returnIndexReaders(readers);

View Full Code Here

          int expected = TestData.testdata.length;
          while(!stop)
          {
            Searcher searcher = null;
            List<ZoieIndexReader<IndexReader>> readers = null;
            MultiReader reader=null;
            try
            {
              readers=idxSystem.getIndexReaders();
              reader=new MultiReader(readers.toArray(new IndexReader[readers.size()]),false);


              searcher=new IndexSearcher(reader);


              TopDocs hits = searcher.search(q,10);
              int count = hits.totalHits;


              if (count != expected)
              {
                mismatch = true;
                message = "hit count: " + count +" / expected: "+expected;
                stop = true;
                StringBuffer sb = new StringBuffer();
                sb.append(message + "\n");
                sb.append("each\n");
                sb.append(groupDump(readers, q));
                sb.append("main\n");
                sb.append(dump(reader, hits));
                System.out.println(sb.toString());
                log.info(sb.toString());
              }
              Thread.sleep(20);
            }
            catch(Exception ex)
            {
              ex.printStackTrace();
              exception = ex;
              stop = true;
            }
            finally
            {
              try{
                if (searcher != null){
                  searcher.close();
                  reader.close();
                  reader = null;
                  searcher = null;
                }
              }
              catch(IOException ioe){
                log.error(ioe.getMessage(),ioe);
              }
              finally{
                idxSystem.returnIndexReaders(readers);
              }
            }
          }
        }
        private String groupDump(List<ZoieIndexReader<IndexReader>> readers, Query q) throws IOException
        {
          StringBuffer sb = new StringBuffer();
          for(ZoieIndexReader<IndexReader> reader : readers)
          {
            sb.append(reader).append("\n");
            Searcher searcher = new IndexSearcher(reader);
            TopDocs hits = searcher.search(q, 20);
            sb.append(dump(reader, hits));
            searcher.close();
            searcher = null;
          }
          return sb.toString();
        }


        private String dump(IndexReader reader, TopDocs hits)
        throws CorruptIndexException, IOException
        {
          StringBuffer sb = new StringBuffer();
          ScoreDoc[] sd = hits.scoreDocs;
          long[] uids = new long[sd.length];
          try
          {
            if (reader.hasDeletions()) sb.append(" there are deletions @ version: " + reader.getVersion());
          } catch(UnsupportedOperationException e)
          {
            if (reader.hasDeletions()) sb.append(" there are deletions @ version: N/A");
          }
            sb.append("\n");
          for(int i =0; i< sd.length; i++)
          {
            Document doc = reader.document(sd[i].doc);
            uids[i] = Long.parseLong(doc.get("id"));
            if (reader.isDeleted(sd[i].doc))
            {
              sb.append("doc: " + sd[i].doc + " with uid: " +
                  uids[i] + " has been deleted").append("\n");
            }
          }
          sb.append(Thread.currentThread() + Arrays.toString(uids)).append("\n");
          int max = reader.maxDoc();
          uids = new long[max];
          for(int i=0; i<max; i++)
          {
            Document doc = reader.document(i);
            uids[i] = Long.parseLong(doc.get("id"));
            if (reader.isDeleted(i))
            {
              sb.append("doc: " + i + " with uid: " + uids[i] + " has been deleted").append("\n");
            }
          }
          sb.append("uids: " + Arrays.toString(uids)).append("\n");

View Full Code Here

        System.out.println("i:" + i + "ZoieTest: commitData(ZoieVersion): " + commitData);  
      }
      
      if(2>1) return;
      
      MultiReader reader=new MultiReader(readers.toArray(new IndexReader[readers.size()]),false);


      searcher=new IndexSearcher(reader);
      hits = searcher.search(q,10);
      int expected = TestData.testdata.length;
      assertEquals("before update: zoie count mismatch[hit count: " + hits.totalHits +" / expected: "+TestData.testdata.length +"]", count, hits.totalHits);
      q = parser.parse("zoie2");


      searcher.close();
      reader.close();
      idxSystem.returnIndexReaders(readers);


      readers=idxSystem.getIndexReaders();
      reader=new MultiReader(readers.toArray(new IndexReader[readers.size()]),false);


      searcher=new IndexSearcher(reader);
      hits = searcher.search(q,10);
      assertEquals("before update: zoie2 count mismatch[hit count: " + hits.totalHits +" / expected: "+ 0 +"]", 0, hits.totalHits);
      searcher.close();
      reader.close();
      idxSystem.returnIndexReaders(readers);


      list = new ArrayList<DataEvent<String,DefaultZoieVersion>>(TestData.testdata2.length);
      for(int i = 0; i < TestData.testdata2.length; i++)
      {
        version = count + i;
        zvt = new DefaultZoieVersion();
        zvt.setVersionId(version);
        list.add(new DataEvent<String,DefaultZoieVersion>(TestData.testdata2[i],zvt));
      }
      memoryProvider.addEvents(list);
      idxSystem.syncWthVersion(10000, zvt);


      q = parser.parse("zoie");
      readers=idxSystem.getIndexReaders();
      reader=new MultiReader(readers.toArray(new IndexReader[readers.size()]),false);


      searcher=new IndexSearcher(reader);
      hits = searcher.search(q,10);
      expected = 0;
      assertEquals("after update: zoie count mismatch[hit count: " + hits.totalHits +" / expected: "+ 0 +"]", 0,  hits.totalHits);
      searcher.close();
      reader.close();
      idxSystem.returnIndexReaders(readers);


      q = parser.parse("zoie2");


      readers=idxSystem.getIndexReaders();
      reader=new MultiReader(readers.toArray(new IndexReader[readers.size()]),false);


      searcher=new IndexSearcher(reader);


      hits = searcher.search(q,10);
      expected = TestData.testdata2.length;
      assertEquals("after update: zoie2 count mismatch[hit count: " + hits.totalHits +" / expected: "+ expected +"]", expected, hits.totalHits);
      searcher.close();
      reader.close();
      idxSystem.returnIndexReaders(readers);
    }
    finally
    {
      memoryProvider.stop();

View Full Code Here

    
    SearchResult result=new SearchResult();
    
    List<ZoieIndexReader<R>> readers=null;


    MultiReader multiReader=null;
    Searcher searcher = null;
    try
    {
      Query q=null;
      if (queryString == null || queryString.length() ==0)
      {
        q = new MatchAllDocsQuery();
      }
      else
      {
        q = qparser.parse(queryString); 
      }
      readers=_idxReaderFactory.getIndexReaders();
      multiReader=new MultiReader(readers.toArray(new IndexReader[readers.size()]), false);
      searcher=new IndexSearcher(multiReader);
      
      long start=System.currentTimeMillis();
      TopDocs docs=searcher.search(q, null, 10);
      long end=System.currentTimeMillis();
      
      result.setTime(end-start);
      result.setTotalDocs(multiReader.numDocs());
      result.setTotalHits(docs.totalHits);
      


      ScoreDoc[] scoreDocs=docs.scoreDocs;
      ArrayList<SearchHit> hitList=new ArrayList<SearchHit>(scoreDocs.length);
      for (ScoreDoc scoreDoc : scoreDocs)
      {
        SearchHit hit=new SearchHit();
        hit.setScore(scoreDoc.score);
        int docid=scoreDoc.doc;
        
        Document doc=multiReader.document(docid);
        String content=doc.get("content");
        
        Scorer qs=new QueryScorer(q);
        
        SimpleHTMLFormatter formatter=new SimpleHTMLFormatter("<span class=\"hl\">","</span>");
        Highlighter hl=new Highlighter(formatter,qs); 
        String[] fragments=hl.getBestFragments(analyzer, "content",content, 1);
        
        Map<String,String[]> fields=convert(doc);
        fields.put("fragment",fragments);
        hit.setFields(fields);
        hitList.add(hit);
      }
      
      result.setHits(hitList.toArray(new SearchHit[hitList.size()]));
      return result;
    }
    catch(Exception e)
    {
      log.error(e.getMessage(),e);
      throw new ZoieException(e.getMessage(),e);
    }
    finally
    {
      try{
        if (searcher!=null)
        {
        try {
          searcher.close();
        } catch (IOException e) {
          log.error(e.getMessage(),e);
        }
        finally{
            if (multiReader!=null){
              try{
                multiReader.close();
              }
              catch(IOException e){
                log.error(e.getMessage(),e);
              }
            }

View Full Code Here

import proj.zoie.api.IndexReaderMerger;
import proj.zoie.api.ZoieIndexReader;


public class DefaultIndexReaderMerger<T extends IndexReader> implements IndexReaderMerger<MultiReader, T> {
  public MultiReader mergeIndexReaders(List<ZoieIndexReader<T>> readerList) {
    MultiReader r = new MultiReader(readerList.toArray(new IndexReader[readerList.size()]),false);
    return r;
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.lucene.index.MultiReader

com.browseengine.bobo.api.BoboIndexReader

com.flaptor.hounder.MultiIndex

com.gentics.cr.lucene.indexaccessor.DefaultMultiIndexAccessor

com.gnizr.core.search.SearchSuggestIndexer

com.uwyn.drone.webui.elements.pub.Search

net.nutch.searcher.IndexSearcher

net.nutch.tools.PruneIndexTool

net.sf.regain.search.results.SearchResultsImpl

org.apache.hadoop.contrib.index.mapred.TestDistributionPolicy

org.apache.hadoop.contrib.index.mapred.TestIndexUpdater

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.