Examples of org.apache.lucene.facet.taxonomy.TaxonomyWriter

Package org.apache.lucene.facet.taxonomy

Examples of org.apache.lucene.facet.taxonomy.TaxonomyWriter

org.apache.lucene.facet.taxonomy.TaxonomyWriter
TaxonomyWriter is the interface which the faceted-search library uses to dynamically build the taxonomy at indexing time.
Notes about concurrent access to the taxonomy:
An implementation must allow multiple readers and a single writer to be active concurrently. Readers follow so-called "point in time" semantics, i.e., a reader object will only see taxonomy entries which were available at the time it was created. What the writer writes is only available to (new) readers after the writer's commit() is called.
Faceted search keeps two indices - namely Lucene's main index, and this taxonomy index. When one or more readers are active concurrently with the writer, care must be taken to avoid an inconsistency between the state of these two indices: When writing to the indices, the taxonomy must always be committed to disk *before* the main index, because the main index refers to categories listed in the taxonomy. Such control can best be achieved by turning off the main index's "autocommit" feature, and explicitly calling commit() for both indices (first for the taxonomy, then for the main index). In old versions of Lucene (2.2 or earlier), when autocommit could not be turned off, a more complicated solution needs to be used. E.g., use some sort of (possibly inter-process) locking to ensure that a reader is being opened only right after both indices have been flushed (and before anything else is written to them). @lucene.experimental

    dir = newDirectory();
    RandomIndexWriter writer = new RandomIndexWriter(random, dir, 
        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.KEYWORD, false)));
    
    taxoDir = newDirectory();
    TaxonomyWriter taxoWriter = new LuceneTaxonomyWriter(taxoDir);
    
    for (int i = 0; i < 100; i++) {
      ArrayList<CategoryPath> paths = new ArrayList<CategoryPath>();
      Document doc = new Document();
      if (i % 2 == 0) { // 50
        doc.add(new Field("content", "foo", Store.NO, Index.ANALYZED));
      }
      if (i % 3 == 0) { // 33
        doc.add(new Field("content", "bar", Store.NO, Index.ANALYZED));
      }
      if (i % 4 == 0) { // 25
        paths.add(new CategoryPath("a"));
      }
      if (i % 5 == 0) { // 20
        paths.add(new CategoryPath("b"));
      }
      CategoryDocumentBuilder builder = new CategoryDocumentBuilder(taxoWriter);
      builder.setCategoryPaths(paths).build(doc);
      writer.addDocument(doc);
    }
    
    taxoWriter.close();
    reader = writer.getReader();
    writer.close();
    
    taxo = new LuceneTaxonomyReader(taxoDir);
  }

View Full Code Here

   * @throws IOException
   */
  @Test
  public void testStreamDefaultParams() throws IOException {
    Directory directory = newDirectory();
    TaxonomyWriter taxonomyWriter = new LuceneTaxonomyWriter(
        directory);
    CategoryParentsStream stream = new CategoryParentsStream(
        new CategoryAttributesStream(categoryContainer),
        taxonomyWriter, new DefaultFacetIndexingParams());


    // count the number of tokens
    int nTokens;
    for (nTokens = 0; stream.incrementToken(); nTokens++) {
    }
    // should be 6 - all categories and parents
    assertEquals("Wrong number of tokens", 6, nTokens);


    taxonomyWriter.close();
    directory.close();
  }

View Full Code Here

   * @throws IOException
   */
  @Test
  public void testStreamNonTopLevelParams() throws IOException {
    Directory directory = newDirectory();
    final TaxonomyWriter taxonomyWriter = new LuceneTaxonomyWriter(
        directory);
    FacetIndexingParams indexingParams = new DefaultFacetIndexingParams() {
      @Override
      protected OrdinalPolicy fixedOrdinalPolicy() {
        return new NonTopLevelOrdinalPolicy();
      }
      @Override
      protected PathPolicy fixedPathPolicy() {
        return new NonTopLevelPathPolicy();
      }
    };
    
    CategoryParentsStream stream = new CategoryParentsStream(
        new CategoryAttributesStream(categoryContainer),
        taxonomyWriter, indexingParams);


    // count the number of tokens
    int nTokens;
    for (nTokens = 0; stream.incrementToken(); nTokens++) {
    }
    /*
     * should be 4: 3 non top level ("two", "three" and "six"), and one
     * explicit top level ("four")
     */
    assertEquals("Wrong number of tokens", 4, nTokens);


    taxonomyWriter.close();
    directory.close();
  }

View Full Code Here

   * @throws FacetException 
   */
  @Test
  public void testNoRetainableAttributes() throws IOException, FacetException {
    Directory directory = newDirectory();
    TaxonomyWriter taxonomyWriter = new LuceneTaxonomyWriter(directory);


    new CategoryParentsStream(new CategoryAttributesStream(categoryContainer),
        taxonomyWriter, new DefaultFacetIndexingParams());


    // add DummyAttribute, but do not retain, only one expected
    categoryContainer.addCategory(initialCatgeories[0], new DummyProperty());


    CategoryParentsStream stream = new CategoryParentsStream(new CategoryAttributesStream(
        categoryContainer), taxonomyWriter,
        new DefaultFacetIndexingParams());


    int nAttributes = 0;
    while (stream.incrementToken()) {
      if (stream.categoryAttribute.getProperty(DummyProperty.class) != null) {
        nAttributes++;
      }
    }
    assertEquals("Wrong number of tokens with attributes", 1, nAttributes);


    taxonomyWriter.close();
    directory.close();
  }

View Full Code Here

   * @throws FacetException 
   */
  @Test
  public void testRetainableAttributes() throws IOException, FacetException {
    Directory directory = newDirectory();
    TaxonomyWriter taxonomyWriter = new LuceneTaxonomyWriter(
        directory);


    FacetIndexingParams indexingParams = new DefaultFacetIndexingParams();
    new CategoryParentsStream(new CategoryAttributesStream(
        categoryContainer), taxonomyWriter, indexingParams);


    // add DummyAttribute and retain it, three expected
    categoryContainer.clear();
    categoryContainer
        .addCategory(initialCatgeories[0], new DummyProperty());
    CategoryParentsStream stream = new CategoryParentsStream(
        new CategoryAttributesStream(categoryContainer),
        taxonomyWriter, new DefaultFacetIndexingParams());
    stream.addRetainableProperty(DummyProperty.class);


    MyCategoryListTokenizer tokenizer = new MyCategoryListTokenizer(stream,
        indexingParams);


    int nAttributes = 0;
    try {
      while (tokenizer.incrementToken()) {
        if (stream.categoryAttribute.getProperty(DummyProperty.class) != null) {
          nAttributes++;
        }
      }
    } catch (IOException e) {
      fail("Properties retained after stream closed");
    }
    assertEquals("Wrong number of tokens with attributes", 3, nAttributes);


    taxonomyWriter.close();
    directory.close();
  }

View Full Code Here

    Directory[][] dirs = getDirs();
    // create and open an index writer
    RandomIndexWriter iw = new RandomIndexWriter(random, dirs[0][0], newIndexWriterConfig(
        TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
    // create and open a taxonomy writer
    TaxonomyWriter tw = new LuceneTaxonomyWriter(dirs[0][1], OpenMode.CREATE);


    /**
     * Configure with no custom counting lists
     */
    PerDimensionIndexingParams iParams = new PerDimensionIndexingParams();


    seedIndex(iw, tw, iParams);


    IndexReader ir = iw.getReader();
    tw.commit();


    // prepare index reader and taxonomy.
    TaxonomyReader tr = new LuceneTaxonomyReader(dirs[0][1]);


    // prepare searcher to search against
    IndexSearcher searcher = newSearcher(ir);


    FacetsCollector facetsCollector = performSearch(iParams, tr, ir,
        searcher);


    // Obtain facets results and hand-test them
    assertCorrectResults(facetsCollector);


    TermDocs td = ir.termDocs(new Term("$facets", "$fulltree$"));
    assertTrue(td.next());


    tr.close();
    ir.close();
    searcher.close();
    iw.close();
    tw.close();
    IOUtils.close(dirs[0]);
  }

View Full Code Here

    Directory[][] dirs = getDirs();
    // create and open an index writer
    RandomIndexWriter iw = new RandomIndexWriter(random, dirs[0][0], newIndexWriterConfig(
        TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
    // create and open a taxonomy writer
    TaxonomyWriter tw = new LuceneTaxonomyWriter(dirs[0][1],
        OpenMode.CREATE);


    PerDimensionIndexingParams iParams = new PerDimensionIndexingParams();
    iParams.addCategoryListParams(new CategoryPath("Author"),
        new CategoryListParams(new Term("$author", "Authors")));
    seedIndex(iw, tw, iParams);


    IndexReader ir = iw.getReader();
    tw.commit();


    // prepare index reader and taxonomy.
    TaxonomyReader tr = new LuceneTaxonomyReader(dirs[0][1]);


    // prepare searcher to search against
    IndexSearcher searcher = newSearcher(ir);


    FacetsCollector facetsCollector = performSearch(iParams, tr, ir,
        searcher);


    // Obtain facets results and hand-test them
    assertCorrectResults(facetsCollector);


    assertPostingListExists("$facets", "$fulltree$", ir);
    assertPostingListExists("$author", "Authors", ir);


    tr.close();
    ir.close();
    searcher.close();
    iw.close();
    tw.close();
    IOUtils.close(dirs[0]);
  }

View Full Code Here

    Directory[][] dirs = getDirs();
    // create and open an index writer
    RandomIndexWriter iw = new RandomIndexWriter(random, dirs[0][0], newIndexWriterConfig(
        TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
    // create and open a taxonomy writer
    TaxonomyWriter tw = new LuceneTaxonomyWriter(dirs[0][1],
        OpenMode.CREATE);


    PerDimensionIndexingParams iParams = new PerDimensionIndexingParams();
    iParams.addCategoryListParams(new CategoryPath("Band"),
        new CategoryListParams(new Term("$music", "Bands")));
    iParams.addCategoryListParams(new CategoryPath("Composer"),
        new CategoryListParams(new Term("$music", "Composers")));
    seedIndex(iw, tw, iParams);


    IndexReader ir = iw.getReader();
    tw.commit();


    // prepare index reader and taxonomy.
    TaxonomyReader tr = new LuceneTaxonomyReader(dirs[0][1]);


    // prepare searcher to search against
    IndexSearcher searcher = newSearcher(ir);


    FacetsCollector facetsCollector = performSearch(iParams, tr, ir,
        searcher);


    // Obtain facets results and hand-test them
    assertCorrectResults(facetsCollector);


    assertPostingListExists("$facets", "$fulltree$", ir);
    assertPostingListExists("$music", "Bands", ir);
    assertPostingListExists("$music", "Composers", ir);


    tr.close();
    ir.close();
    searcher.close();
    iw.close();
    tw.close();
    IOUtils.close(dirs[0]);
  }

View Full Code Here

    Directory[][] dirs = getDirs();
    // create and open an index writer
    RandomIndexWriter iw = new RandomIndexWriter(random, dirs[0][0], newIndexWriterConfig(
        TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
    // create and open a taxonomy writer
    TaxonomyWriter tw = new LuceneTaxonomyWriter(dirs[0][1], OpenMode.CREATE);


    PerDimensionIndexingParams iParams = new PerDimensionIndexingParams();
    iParams.addCategoryListParams(new CategoryPath("Band"),
        new CategoryListParams(new Term("$bands", "Bands")));
    iParams.addCategoryListParams(new CategoryPath("Composer"),
        new CategoryListParams(new Term("$composers", "Composers")));
    seedIndex(iw, tw, iParams);


    IndexReader ir = iw.getReader();
    tw.commit();


    // prepare index reader and taxonomy.
    TaxonomyReader tr = new LuceneTaxonomyReader(dirs[0][1]);


    // prepare searcher to search against
    IndexSearcher searcher = newSearcher(ir);


    FacetsCollector facetsCollector = performSearch(iParams, tr, ir,
        searcher);


    // Obtain facets results and hand-test them
    assertCorrectResults(facetsCollector);
    assertPostingListExists("$facets", "$fulltree$", ir);
    assertPostingListExists("$bands", "Bands", ir);
    assertPostingListExists("$composers", "Composers", ir);
    tr.close();
    ir.close();
    searcher.close();
    iw.close();
    tw.close();
    IOUtils.close(dirs[0]);
  }

View Full Code Here

    Directory[][] dirs = getDirs();
    // create and open an index writer
    RandomIndexWriter iw = new RandomIndexWriter(random, dirs[0][0], newIndexWriterConfig(
        TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
    // create and open a taxonomy writer
    TaxonomyWriter tw = new LuceneTaxonomyWriter(dirs[0][1],
        OpenMode.CREATE);


    PerDimensionIndexingParams iParams = new PerDimensionIndexingParams();
    iParams.addCategoryListParams(new CategoryPath("Band"),
        new CategoryListParams(new Term("$music", "music")));
    iParams.addCategoryListParams(new CategoryPath("Composer"),
        new CategoryListParams(new Term("$music", "music")));
    iParams.addCategoryListParams(new CategoryPath("Author"),
        new CategoryListParams(new Term("$literature", "Authors")));


    seedIndex(iw, tw, iParams);


    IndexReader ir = iw.getReader();
    tw.commit();


    // prepare index reader and taxonomy.
    TaxonomyReader tr = new LuceneTaxonomyReader(dirs[0][1]);


    // prepare searcher to search against
    IndexSearcher searcher = newSearcher(ir);


    FacetsCollector facetsCollector = performSearch(iParams, tr, ir,
        searcher);


    // Obtain facets results and hand-test them
    assertCorrectResults(facetsCollector);
    assertPostingListExists("$music", "music", ir);
    assertPostingListExists("$literature", "Authors", ir);


    tr.close();
    ir.close();
    searcher.close();
    iw.close();
    tw.close();
    IOUtils.close(dirs[0]);
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.lucene.facet.taxonomy.TaxonomyWriter

com.gentics.cr.lucene.indexer.index.CRLuceneIndexJob

org.apache.lucene.benchmark.byTask.tasks.CommitTaxonomyIndexTask

org.apache.lucene.facet.associations.AssociationsFacetRequestTest

org.apache.lucene.facet.complements.TestTotalFacetCounts

org.apache.lucene.facet.complements.TestTotalFacetCountsCache

org.apache.lucene.facet.enhancements.TwoEnhancementsTest

org.apache.lucene.facet.example.association.AssociationIndexer

org.apache.lucene.facet.example.simple.SimpleIndexer

org.apache.lucene.facet.FacetTestBase

org.apache.lucene.facet.index.categorypolicy.OrdinalPolicyTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.