Examples of LuceneTaxonomyWriter

org.apache.lucene.facet.taxonomy.lucene.LuceneTaxonomyWriter
{@link TaxonomyWriter} which uses a Lucene index to store the taxonomyinformation on disk, and keeps an additional in-memory cache of some or all categories.
By using a Lucene index to store the information on disk, rather than some specialized file format, we get for "free" Lucene's correctness (especially regarding multi-process concurrency), and the ability to write to any implementation of Directory (and not just the file system).
In addition to the permanently-stored Lucene index, efficiency dictates that we also keep an in-memory cache of recently seen or all categories, so that we do not need to go back to disk for every category addition to see which ordinal this category already has, if any. A {@link TaxonomyWriterCache} object determines the specific caching algorithmused.
This class offers some hooks for extending classes to control the {@link IndexWriter} instance that is used. See {@link #openLuceneIndex} and{@link #closeLuceneIndex()} . @lucene.experimental

Examples of org.apache.lucene.facet.taxonomy.lucene.LuceneTaxonomyWriter

    other methods (which we have already tested above).
   */
  @Test
  public void testReaderParentArray() throws Exception {
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new LuceneTaxonomyWriter(indexDir);
    fillTaxonomy(tw);
    tw.close();
    TaxonomyReader tr = new LuceneTaxonomyReader(indexDir);
    int[] parents = tr.getParentArray();
    assertEquals(tr.getSize(), parents.length);
    for (int i=0; i<tr.getSize(); i++) {
      assertEquals(tr.getParent(i), parents[i]);

View Full Code Here

Examples of org.apache.lucene.facet.taxonomy.lucene.LuceneTaxonomyWriter

   * below further tests on data refresh etc.
   */
  @Test
  public void testChildrenArrays() throws Exception {
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new LuceneTaxonomyWriter(indexDir);
    fillTaxonomy(tw);
    tw.close();
    TaxonomyReader tr = new LuceneTaxonomyReader(indexDir);
    ChildrenArrays ca = tr.getChildrenArrays();
    int[] youngestChildArray = ca.getYoungestChildArray();
    assertEquals(tr.getSize(), youngestChildArray.length);
    int[] olderSiblingArray = ca.getOlderSiblingArray();

View Full Code Here

Examples of org.apache.lucene.facet.taxonomy.lucene.LuceneTaxonomyWriter

   * of categories like we did in the above test.
   */
  @Test
  public void testChildrenArraysInvariants() throws Exception {
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new LuceneTaxonomyWriter(indexDir);
    fillTaxonomy(tw);
    tw.close();
    TaxonomyReader tr = new LuceneTaxonomyReader(indexDir);
    ChildrenArrays ca = tr.getChildrenArrays();
    int[] youngestChildArray = ca.getYoungestChildArray();
    assertEquals(tr.getSize(), youngestChildArray.length);
    int[] olderSiblingArray = ca.getOlderSiblingArray();

View Full Code Here

Examples of org.apache.lucene.facet.taxonomy.lucene.LuceneTaxonomyWriter

   * @throws IOException
   */
  @Test
  public void testTokensDefaultParams() throws IOException {
    Directory directory = newDirectory();
    TaxonomyWriter taxonomyWriter = new LuceneTaxonomyWriter(
        directory);
    DefaultFacetIndexingParams indexingParams = new DefaultFacetIndexingParams();
    CategoryTokenizer tokenizer = new CategoryTokenizer(
        new CategoryAttributesStream(categoryContainer),
        indexingParams);


    // count the number of tokens
    Set<String> categoryTerms = new HashSet<String>();
    for (int i = 0; i < initialCatgeories.length; i++) {
      categoryTerms.add(initialCatgeories[i]
          .toString(indexingParams.getFacetDelimChar()));
    }


    int nTokens;
    for (nTokens = 0; tokenizer.incrementToken(); nTokens++) {
      if (!categoryTerms.remove(tokenizer.termAttribute.toString())) {
        fail("Unexpected term: " + tokenizer.termAttribute.toString());
      }
    }
    assertTrue("all category terms should have been found", categoryTerms
        .isEmpty());


    // should be 6 - all categories and parents
    assertEquals("Wrong number of tokens", 3, nTokens);


    taxonomyWriter.close();
    directory.close();
  }

View Full Code Here

Examples of org.apache.lucene.facet.taxonomy.lucene.LuceneTaxonomyWriter

   * @throws IOException
   */
  @Test
  public void testLongCategoryPath() throws IOException {
    Directory directory = newDirectory();
    TaxonomyWriter taxonomyWriter = new LuceneTaxonomyWriter(
        directory);


    List<CategoryPath> longCategory = new ArrayList<CategoryPath>();
    longCategory.add(new CategoryPath("one", "two", "three", "four",
        "five", "six", "seven"));


    DefaultFacetIndexingParams indexingParams = new DefaultFacetIndexingParams();
    CategoryTokenizer tokenizer = new CategoryTokenizer(
        new CategoryAttributesStream(new CategoryAttributesIterable(
            longCategory)), indexingParams);


    // count the number of tokens
    String categoryTerm = longCategory.get(0).toString(
        indexingParams.getFacetDelimChar());


    assertTrue("Missing token", tokenizer.incrementToken());
    if (!categoryTerm.equals(tokenizer.termAttribute.toString())) {
      fail("Unexpected term: " + tokenizer.termAttribute.toString());
    }


    assertFalse("Unexpected token", tokenizer.incrementToken());


    taxonomyWriter.close();
    directory.close();
  }

View Full Code Here

Examples of org.apache.lucene.facet.taxonomy.lucene.LuceneTaxonomyWriter

   * Test how getChildrenArrays() deals with the taxonomy's growth:
   */
  @Test
  public void testChildrenArraysGrowth() throws Exception {
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new LuceneTaxonomyWriter(indexDir);
    tw.addCategory(new CategoryPath("hi", "there"));
    tw.commit();
    TaxonomyReader tr = new LuceneTaxonomyReader(indexDir);
    ChildrenArrays ca = tr.getChildrenArrays();
    assertEquals(3, tr.getSize());
    assertEquals(3, ca.getOlderSiblingArray().length);
    assertEquals(3, ca.getYoungestChildArray().length);
    assertTrue(Arrays.equals(new int[] { 1, 2, -1 }, ca.getYoungestChildArray()));
    assertTrue(Arrays.equals(new int[] { -1, -1, -1 }, ca.getOlderSiblingArray()));
    tw.addCategory(new CategoryPath("hi", "ho"));
    tw.addCategory(new CategoryPath("hello"));
    tw.commit();
    // Before refresh, nothing changed..
    ChildrenArrays newca = tr.getChildrenArrays();
    assertSame(newca, ca); // we got exactly the same object
    assertEquals(3, tr.getSize());
    assertEquals(3, ca.getOlderSiblingArray().length);
    assertEquals(3, ca.getYoungestChildArray().length);
    // After the refresh, things change:
    tr.refresh();
    ca = tr.getChildrenArrays();
    assertEquals(5, tr.getSize());
    assertEquals(5, ca.getOlderSiblingArray().length);
    assertEquals(5, ca.getYoungestChildArray().length);
    assertTrue(Arrays.equals(new int[] { 4, 3, -1, -1, -1 }, ca.getYoungestChildArray()));
    assertTrue(Arrays.equals(new int[] { -1, -1, -1, 2, 1 }, ca.getOlderSiblingArray()));
    tw.close();
    tr.close();
    indexDir.close();
  }

View Full Code Here

Examples of org.apache.lucene.facet.taxonomy.lucene.LuceneTaxonomyWriter

  @Test
  @Ignore
  public void testTaxonomyReaderRefreshRaces() throws Exception {
    // compute base child arrays - after first chunk, and after the other
    Directory indexDirBase =  newDirectory();
    TaxonomyWriter twBase = new LuceneTaxonomyWriter(indexDirBase);
    twBase.addCategory(new CategoryPath("a", "0"));
    final CategoryPath abPath = new CategoryPath("a", "b");
    twBase.addCategory(abPath);
    twBase.commit();
    TaxonomyReader trBase = new LuceneTaxonomyReader(indexDirBase);


    final ChildrenArrays ca1 = trBase.getChildrenArrays();
    
    final int abOrd = trBase.getOrdinal(abPath);
    final int abYoungChildBase1 = ca1.getYoungestChildArray()[abOrd]; 
    
    for (int i=0; i < 1<<10; i++) { //1024 facets
      twBase.addCategory(new CategoryPath("a", "b", Integer.toString(i)));
    }
    twBase.commit();
    
    trBase.refresh();
    
    final ChildrenArrays ca2 = trBase.getChildrenArrays();
    final int abYoungChildBase2 = ca2.getYoungestChildArray()[abOrd];

View Full Code Here

Examples of org.apache.lucene.facet.taxonomy.lucene.LuceneTaxonomyWriter

    taxoDir = newDirectory();
    // preparations - index, taxonomy, content
    RandomIndexWriter writer = new RandomIndexWriter(random, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, 
        new MockAnalyzer(random, MockTokenizer.KEYWORD, false)));
    
    TaxonomyWriter taxoWriter = new LuceneTaxonomyWriter(taxoDir);
    
    EnhancementsDocumentBuilder builder = new EnhancementsDocumentBuilder(
        taxoWriter, new DefaultEnhancementsIndexingParams(
            new AssociationEnhancement()));
    
    // index documents, 50% have only 'b' and all have 'a'
    for (int i = 0; i < 100; i++) {
      Document doc = new Document();
      CategoryContainer container = new CategoryContainer();
      container.addCategory(aint, new AssociationIntProperty(2));
      container.addCategory(afloat, new AssociationFloatProperty(0.5f));
      if (i % 2 == 0) { // 50
        container.addCategory(bint, new AssociationIntProperty(3));
        container.addCategory(bfloat, new AssociationFloatProperty(0.2f));
      }
      builder.setCategories(container).build(doc);
      writer.addDocument(doc);
    }
    
    taxoWriter.close();
    reader = writer.getReader();
    writer.close();
  }

View Full Code Here

Examples of org.apache.lucene.facet.taxonomy.lucene.LuceneTaxonomyWriter


  private void populateIndex(FacetIndexingParams iParams, Directory indexDir,
      Directory taxoDir) throws Exception {
    RandomIndexWriter writer = new RandomIndexWriter(random, indexDir, 
        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.KEYWORD, false)));
    TaxonomyWriter taxoWriter = new LuceneTaxonomyWriter(taxoDir);


    for (CategoryPath[] categories : perDocCategories) {
      writer.addDocument(new CategoryDocumentBuilder(taxoWriter, iParams)
          .setCategoryPaths(Arrays.asList(categories)).build(
              new Document()));


    }
    taxoWriter.commit();
    writer.commit();
    taxoWriter.close();
    writer.close();
  }

View Full Code Here

Examples of org.apache.lucene.facet.taxonomy.lucene.LuceneTaxonomyWriter

    dir = newDirectory();
    RandomIndexWriter writer = new RandomIndexWriter(random, dir, 
        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.KEYWORD, false)));
    
    taxoDir = newDirectory();
    TaxonomyWriter taxoWriter = new LuceneTaxonomyWriter(taxoDir);
    
    for (int i = 0; i < 100; i++) {
      ArrayList<CategoryPath> paths = new ArrayList<CategoryPath>();
      Document doc = new Document();
      if (i % 2 == 0) { // 50
        doc.add(new Field("content", "foo", Store.NO, Index.ANALYZED));
      }
      if (i % 3 == 0) { // 33
        doc.add(new Field("content", "bar", Store.NO, Index.ANALYZED));
      }
      if (i % 4 == 0) { // 25
        paths.add(new CategoryPath("a"));
      }
      if (i % 5 == 0) { // 20
        paths.add(new CategoryPath("b"));
      }
      CategoryDocumentBuilder builder = new CategoryDocumentBuilder(taxoWriter);
      builder.setCategoryPaths(paths).build(doc);
      writer.addDocument(doc);
    }
    
    taxoWriter.close();
    reader = writer.getReader();
    writer.close();
    
    taxo = new LuceneTaxonomyReader(taxoDir);
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.