Examples of org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter

Package org.apache.lucene.facet.taxonomy.directory

Examples of org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter

org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter
{@link TaxonomyWriter} which uses a {@link Directory} to store the taxonomyinformation on disk, and keeps an additional in-memory cache of some or all categories.
In addition to the permanently-stored information in the {@link Directory}, efficiency dictates that we also keep an in-memory cache of recently seen or all categories, so that we do not need to go back to disk for every category addition to see which ordinal this category already has, if any. A {@link TaxonomyWriterCache} object determines the specific cachingalgorithm used.
This class offers some hooks for extending classes to control the {@link IndexWriter} instance that is used. See {@link #openIndexWriter}. @lucene.experimental

        pair.searchDir = newDirectory();
        pair.taxoDir = newDirectory();
      }
      
      RandomIndexWriter iw = new RandomIndexWriter(random(), pair.searchDir, getIndexWriterConfig(getAnalyzer()));
      TaxonomyWriter taxo = new DirectoryTaxonomyWriter(pair.taxoDir, OpenMode.CREATE);
      
      populateIndex(iw, taxo, fip);
      
      // commit changes (taxonomy prior to search index for consistency)
      taxo.commit();
      iw.commit();
      taxo.close();
      iw.close();
      
      dirsPerPartitionSize.put(Integer.valueOf(partitionSize), pair);
    }

View Full Code Here

  }


  private void index100Docs(Directory indexDir, Directory taxoDir, FacetIndexingParams fip) throws IOException {
    IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
    IndexWriter w = new IndexWriter(indexDir, iwc);
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);
    
    FacetFields facetFields = new FacetFields(tw, fip);
    for (int i = 0; i < 100; i++) {
      Document doc = new Document();
      CategoryPath cp = new CategoryPath("root",Integer.toString(i / 10), Integer.toString(i));

View Full Code Here

   *  faceting, with DrillSideways and taxonomy. */
  public void testMixedRangeAndNonRangeTaxonomy() throws Exception {
    Directory d = newDirectory();
    RandomIndexWriter w = new RandomIndexWriter(random(), d);
    Directory td = newDirectory();
    DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(td, IndexWriterConfig.OpenMode.CREATE);
    FacetFields ff = new FacetFields(tw);


    for (long l = 0; l < 100; l++) {
      Document doc = new Document();
      // For computing range facet counts:
      doc.add(new NumericDocValuesField("field", l));
      // For drill down by numeric range:
      doc.add(new LongField("field", l, Field.Store.NO));


      CategoryPath cp;
      if ((l&3) == 0) {
        cp = new CategoryPath("dim", "a");
      } else {
        cp = new CategoryPath("dim", "b");
      }
      ff.addFields(doc, Collections.singletonList(cp));
      w.addDocument(doc);
    }


    final IndexReader r = w.getReader();
    w.close();


    final TaxonomyReader tr = new DirectoryTaxonomyReader(tw);
    tw.close();


    IndexSearcher s = newSearcher(r);


    final CountFacetRequest countRequest = new CountFacetRequest(new CategoryPath("dim"), 2);
    final RangeFacetRequest<LongRange> rangeRequest = new RangeFacetRequest<LongRange>("field",

View Full Code Here


  private static final String[] CATEGORIES = new String[] { "a/b", "c/d", "a/e", "a/d", "c/g", "c/z", "b/a", "1/2", "b/c" };


  private void index(Directory indexDir, Directory taxoDir) throws IOException {
    IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
    FacetFields facetFields = new FacetFields(taxoWriter);
    
    for (String cat : CATEGORIES) {
      Document doc = new Document();
      facetFields.addFields(doc, Collections.singletonList(new CategoryPath(cat, '/')));

View Full Code Here

    Directory indexDir = newDirectory();
    Directory taxoDir = newDirectory();


    // Create our index/taxonomy writers
    IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
    FacetIndexingParams iParams = FacetIndexingParams.DEFAULT;


    // Add a facet to the index
    addFacets(iParams, indexWriter, taxoWriter, "a", "b");


    // Commit Changes
    indexWriter.commit();
    taxoWriter.commit();


    // Open readers
    DirectoryReader indexReader = DirectoryReader.open(indexDir);
    TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);

View Full Code Here

    Directory indexDir = newDirectory();
    Directory taxoDir = newDirectory();
    
    // Create our index/taxonomy writers
    IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
    FacetIndexingParams iParams = new FacetIndexingParams() {
      @Override
      public int getPartitionSize() {
        return 2;
      }
    };
    // Add a facet to the index
    addFacets(iParams, indexWriter, taxoWriter, "a", "b");
    // Commit Changes
    indexWriter.commit();
    taxoWriter.commit();


    DirectoryReader indexReader = DirectoryReader.open(indexDir);
    TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);


    // Create TFC and write cache to disk
    File outputFile = _TestUtil.createTempFile("test", "tmp", TEMP_DIR);
    TFC.store(outputFile, indexReader, taxoReader, iParams);
    
    // Make the taxonomy grow without touching the index
    for (int i = 0; i < 10; i++) {
      taxoWriter.addCategory(new CategoryPath("foo", Integer.toString(i)));
    }
    taxoWriter.commit();
    TaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(taxoReader);
    assertNotNull(newTaxoReader);
    taxoReader.close();
    taxoReader = newTaxoReader;

View Full Code Here

    SlowRAMDirectory taxoDir = new SlowRAMDirectory(-1, null);


    // Write index using 'normal' directories
    IndexWriter w = new IndexWriter(indexDir, new IndexWriterConfig(
        TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
    DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);
    FacetIndexingParams iParams = FacetIndexingParams.DEFAULT;
    // Add documents and facets
    for (int i = 0; i < 1000; i++) {
      addFacets(iParams, w, tw, "facet", Integer.toString(i));
    }
    w.close();
    tw.close();


    indexDir.setSleepMillis(1);
    taxoDir.setSleepMillis(1);


    IndexReader r = DirectoryReader.open(indexDir);

View Full Code Here

    Directory taxoDir1 = newDirectory(), taxoDir2 = newDirectory();
    
    // Create our index/taxonomy writers
    IndexWriter indexWriter1 = new IndexWriter(indexDir1, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
    IndexWriter indexWriter2 = new IndexWriter(indexDir2, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
    TaxonomyWriter taxoWriter1 = new DirectoryTaxonomyWriter(taxoDir1);
    TaxonomyWriter taxoWriter2 = new DirectoryTaxonomyWriter(taxoDir2);
    FacetIndexingParams iParams = FacetIndexingParams.DEFAULT;


    // Add a facet to the index
    addFacets(iParams, indexWriter1, taxoWriter1, "a", "b");
    addFacets(iParams, indexWriter1, taxoWriter1, "d", "e");
    // Commit Changes
    indexWriter1.commit();
    indexWriter2.commit();
    taxoWriter1.commit();
    taxoWriter2.commit();


    // Open two readers
    DirectoryReader indexReader1 = DirectoryReader.open(indexDir1);
    DirectoryReader indexReader2 = DirectoryReader.open(indexDir2);
    TaxonomyReader taxoReader1 = new DirectoryTaxonomyReader(taxoDir1);

View Full Code Here

    taxoDir = newDirectory();
    // preparations - index, taxonomy, content
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(TEST_VERSION_CURRENT, 
        new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)));
    
    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
    
    AssociationsFacetFields assocFacetFields = new AssociationsFacetFields(taxoWriter);
    
    // index documents, 50% have only 'b' and all have 'a'
    for (int i = 0; i < 110; i++) {
      Document doc = new Document();
      CategoryAssociationsContainer associations = new CategoryAssociationsContainer();
      // every 11th document is added empty, this used to cause the association
      // aggregators to go into an infinite loop
      if (i % 11 != 0) {
        associations.setAssociation(aint, new CategoryIntAssociation(2));
        associations.setAssociation(afloat, new CategoryFloatAssociation(0.5f));
        if (i % 2 == 0) { // 50
          associations.setAssociation(bint, new CategoryIntAssociation(3));
          associations.setAssociation(bfloat, new CategoryFloatAssociation(0.2f));
        }
      }
      assocFacetFields.addFields(doc, associations);
      writer.addDocument(doc);
    }
    
    taxoWriter.close();
    reader = writer.getReader();
    writer.close();
  }

View Full Code Here

    replicator.publish(createRevision(1));
    client.updateNow();
    
    // recreate index and taxonomy
    Directory newTaxo = newDirectory();
    new DirectoryTaxonomyWriter(newTaxo).close();
    publishTaxoWriter.replaceTaxonomy(newTaxo);
    publishIndexWriter.deleteAll();
    replicator.publish(createRevision(2));
    
    client.updateNow();

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter

org.apache.lucene.analysis.KeywordAnalyzer

org.apache.lucene.benchmark.byTask.tasks.CreateTaxonomyIndexTask

org.apache.lucene.benchmark.byTask.tasks.OpenTaxonomyIndexTask

org.apache.lucene.demo.facet.AssociationsFacetsExample

org.apache.lucene.demo.facet.ExpressionAggregationFacetsExample

org.apache.lucene.demo.facet.MultiCategoryListsFacetsExample

org.apache.lucene.demo.facet.SimpleFacetsExample

org.apache.lucene.document.Document

org.apache.lucene.document.Field

org.apache.lucene.document.FieldType

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.