Examples of LuceneTaxonomyWriter

org.apache.lucene.facet.taxonomy.lucene.LuceneTaxonomyWriter
{@link TaxonomyWriter} which uses a Lucene index to store the taxonomyinformation on disk, and keeps an additional in-memory cache of some or all categories.
By using a Lucene index to store the information on disk, rather than some specialized file format, we get for "free" Lucene's correctness (especially regarding multi-process concurrency), and the ability to write to any implementation of Directory (and not just the file system).
In addition to the permanently-stored Lucene index, efficiency dictates that we also keep an in-memory cache of recently seen or all categories, so that we do not need to go back to disk for every category addition to see which ordinal this category already has, if any. A {@link TaxonomyWriterCache} object determines the specific caching algorithmused.
This class offers some hooks for extending classes to control the {@link IndexWriter} instance that is used. See {@link #openLuceneIndex} and{@link #closeLuceneIndex()} . @lucene.experimental

Examples of org.apache.lucene.facet.taxonomy.lucene.LuceneTaxonomyWriter

    Directory[][] dirs = getDirs();
    // create and open an index writer
    RandomIndexWriter iw = new RandomIndexWriter(random, dirs[0][0], newIndexWriterConfig(
        TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
    // create and open a taxonomy writer
    TaxonomyWriter tw = new LuceneTaxonomyWriter(dirs[0][1], OpenMode.CREATE);


    /**
     * Configure with no custom counting lists
     */
    PerDimensionIndexingParams iParams = new PerDimensionIndexingParams();


    seedIndex(iw, tw, iParams);


    IndexReader ir = iw.getReader();
    tw.commit();


    // prepare index reader and taxonomy.
    TaxonomyReader tr = new LuceneTaxonomyReader(dirs[0][1]);


    // prepare searcher to search against
    IndexSearcher searcher = newSearcher(ir);


    FacetsCollector facetsCollector = performSearch(iParams, tr, ir,
        searcher);


    // Obtain facets results and hand-test them
    assertCorrectResults(facetsCollector);


    TermDocs td = ir.termDocs(new Term("$facets", "$fulltree$"));
    assertTrue(td.next());


    tr.close();
    ir.close();
    searcher.close();
    iw.close();
    tw.close();
    IOUtils.close(dirs[0]);
  }

View Full Code Here

Examples of org.apache.lucene.facet.taxonomy.lucene.LuceneTaxonomyWriter

    Directory[][] dirs = getDirs();
    // create and open an index writer
    RandomIndexWriter iw = new RandomIndexWriter(random, dirs[0][0], newIndexWriterConfig(
        TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
    // create and open a taxonomy writer
    TaxonomyWriter tw = new LuceneTaxonomyWriter(dirs[0][1],
        OpenMode.CREATE);


    PerDimensionIndexingParams iParams = new PerDimensionIndexingParams();
    iParams.addCategoryListParams(new CategoryPath("Author"),
        new CategoryListParams(new Term("$author", "Authors")));
    seedIndex(iw, tw, iParams);


    IndexReader ir = iw.getReader();
    tw.commit();


    // prepare index reader and taxonomy.
    TaxonomyReader tr = new LuceneTaxonomyReader(dirs[0][1]);


    // prepare searcher to search against
    IndexSearcher searcher = newSearcher(ir);


    FacetsCollector facetsCollector = performSearch(iParams, tr, ir,
        searcher);


    // Obtain facets results and hand-test them
    assertCorrectResults(facetsCollector);


    assertPostingListExists("$facets", "$fulltree$", ir);
    assertPostingListExists("$author", "Authors", ir);


    tr.close();
    ir.close();
    searcher.close();
    iw.close();
    tw.close();
    IOUtils.close(dirs[0]);
  }

View Full Code Here

Examples of org.apache.lucene.facet.taxonomy.lucene.LuceneTaxonomyWriter

    Directory[][] dirs = getDirs();
    // create and open an index writer
    RandomIndexWriter iw = new RandomIndexWriter(random, dirs[0][0], newIndexWriterConfig(
        TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
    // create and open a taxonomy writer
    TaxonomyWriter tw = new LuceneTaxonomyWriter(dirs[0][1],
        OpenMode.CREATE);


    PerDimensionIndexingParams iParams = new PerDimensionIndexingParams();
    iParams.addCategoryListParams(new CategoryPath("Band"),
        new CategoryListParams(new Term("$music", "Bands")));
    iParams.addCategoryListParams(new CategoryPath("Composer"),
        new CategoryListParams(new Term("$music", "Composers")));
    seedIndex(iw, tw, iParams);


    IndexReader ir = iw.getReader();
    tw.commit();


    // prepare index reader and taxonomy.
    TaxonomyReader tr = new LuceneTaxonomyReader(dirs[0][1]);


    // prepare searcher to search against
    IndexSearcher searcher = newSearcher(ir);


    FacetsCollector facetsCollector = performSearch(iParams, tr, ir,
        searcher);


    // Obtain facets results and hand-test them
    assertCorrectResults(facetsCollector);


    assertPostingListExists("$facets", "$fulltree$", ir);
    assertPostingListExists("$music", "Bands", ir);
    assertPostingListExists("$music", "Composers", ir);


    tr.close();
    ir.close();
    searcher.close();
    iw.close();
    tw.close();
    IOUtils.close(dirs[0]);
  }

View Full Code Here

Examples of org.apache.lucene.facet.taxonomy.lucene.LuceneTaxonomyWriter

    Directory[][] dirs = getDirs();
    // create and open an index writer
    RandomIndexWriter iw = new RandomIndexWriter(random, dirs[0][0], newIndexWriterConfig(
        TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
    // create and open a taxonomy writer
    TaxonomyWriter tw = new LuceneTaxonomyWriter(dirs[0][1], OpenMode.CREATE);


    PerDimensionIndexingParams iParams = new PerDimensionIndexingParams();
    iParams.addCategoryListParams(new CategoryPath("Band"),
        new CategoryListParams(new Term("$bands", "Bands")));
    iParams.addCategoryListParams(new CategoryPath("Composer"),
        new CategoryListParams(new Term("$composers", "Composers")));
    seedIndex(iw, tw, iParams);


    IndexReader ir = iw.getReader();
    tw.commit();


    // prepare index reader and taxonomy.
    TaxonomyReader tr = new LuceneTaxonomyReader(dirs[0][1]);


    // prepare searcher to search against
    IndexSearcher searcher = newSearcher(ir);


    FacetsCollector facetsCollector = performSearch(iParams, tr, ir,
        searcher);


    // Obtain facets results and hand-test them
    assertCorrectResults(facetsCollector);
    assertPostingListExists("$facets", "$fulltree$", ir);
    assertPostingListExists("$bands", "Bands", ir);
    assertPostingListExists("$composers", "Composers", ir);
    tr.close();
    ir.close();
    searcher.close();
    iw.close();
    tw.close();
    IOUtils.close(dirs[0]);
  }

View Full Code Here

Examples of org.apache.lucene.facet.taxonomy.lucene.LuceneTaxonomyWriter

    Directory[][] dirs = getDirs();
    // create and open an index writer
    RandomIndexWriter iw = new RandomIndexWriter(random, dirs[0][0], newIndexWriterConfig(
        TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
    // create and open a taxonomy writer
    TaxonomyWriter tw = new LuceneTaxonomyWriter(dirs[0][1],
        OpenMode.CREATE);


    PerDimensionIndexingParams iParams = new PerDimensionIndexingParams();
    iParams.addCategoryListParams(new CategoryPath("Band"),
        new CategoryListParams(new Term("$music", "music")));
    iParams.addCategoryListParams(new CategoryPath("Composer"),
        new CategoryListParams(new Term("$music", "music")));
    iParams.addCategoryListParams(new CategoryPath("Author"),
        new CategoryListParams(new Term("$literature", "Authors")));


    seedIndex(iw, tw, iParams);


    IndexReader ir = iw.getReader();
    tw.commit();


    // prepare index reader and taxonomy.
    TaxonomyReader tr = new LuceneTaxonomyReader(dirs[0][1]);


    // prepare searcher to search against
    IndexSearcher searcher = newSearcher(ir);


    FacetsCollector facetsCollector = performSearch(iParams, tr, ir,
        searcher);


    // Obtain facets results and hand-test them
    assertCorrectResults(facetsCollector);
    assertPostingListExists("$music", "music", ir);
    assertPostingListExists("$literature", "Authors", ir);


    tr.close();
    ir.close();
    searcher.close();
    iw.close();
    tw.close();
    IOUtils.close(dirs[0]);
  }

View Full Code Here

Examples of org.apache.lucene.facet.taxonomy.lucene.LuceneTaxonomyWriter

      String[] docTexts, CategoryPath[][] cPaths) throws Exception {
    // create and open an index writer
    IndexWriter iw = new IndexWriter(indexDir, new IndexWriterConfig(
        ExampleUtils.EXAMPLE_VER, SimpleUtils.analyzer).setOpenMode(OpenMode.CREATE));
    // create and open a taxonomy writer
    LuceneTaxonomyWriter taxo = new LuceneTaxonomyWriter(taxoDir, OpenMode.CREATE);
    index(iw, taxo, iParams, docTitles, docTexts, cPaths);
  }

View Full Code Here

Examples of org.apache.lucene.facet.taxonomy.lucene.LuceneTaxonomyWriter

  
  @Test
  public void testCommitUserData() throws Exception {
    // Verifies that committed data is retrievable
    Directory dir = newDirectory();
    LuceneTaxonomyWriter ltw = new LuceneTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, new NoOpCache());
    assertFalse(IndexReader.indexExists(dir));
    ltw.commit(); // first commit, so that an index will be created
    ltw.addCategory(new CategoryPath("a"));
    ltw.addCategory(new CategoryPath("b"));
    Map <String, String> userCommitData = new HashMap<String, String>();
    userCommitData.put("testing", "1 2 3");
    ltw.commit(userCommitData);
    ltw.close();
    IndexReader r = IndexReader.open(dir);
    assertEquals("2 categories plus root should have been committed to the underlying directory", 3, r.numDocs());
    Map <String, String> readUserCommitData = r.getCommitUserData();
    assertTrue("wrong value extracted from commit data", 
        "1 2 3".equals(readUserCommitData.get("testing")));

View Full Code Here

Examples of org.apache.lucene.facet.taxonomy.lucene.LuceneTaxonomyWriter

public class TestAddTaxonomies extends LuceneTestCase {


  @Test
  public void test1() throws Exception {
    Directory dir1 = newDirectory();
    LuceneTaxonomyWriter tw1 = new LuceneTaxonomyWriter(dir1);
    tw1.addCategory(new CategoryPath("Author", "Mark Twain"));
    tw1.addCategory(new CategoryPath("Animals", "Dog"));
    Directory dir2 = newDirectory();
    LuceneTaxonomyWriter tw2 = new LuceneTaxonomyWriter(dir2);
    tw2.addCategory(new CategoryPath("Author", "Rob Pike"));
    tw2.addCategory(new CategoryPath("Aardvarks", "Bob"));
    tw2.close();
    Directory dir3 = newDirectory();
    LuceneTaxonomyWriter tw3 = new LuceneTaxonomyWriter(dir3);
    tw3.addCategory(new CategoryPath("Author", "Zebra Smith"));
    tw3.addCategory(new CategoryPath("Aardvarks", "Bob"));
    tw3.addCategory(new CategoryPath("Aardvarks", "Aaron"));
    tw3.close();


    MemoryOrdinalMap[] maps = new MemoryOrdinalMap[2];
    maps[0] = new MemoryOrdinalMap();
    maps[1] = new MemoryOrdinalMap();

View Full Code Here

Examples of org.apache.lucene.facet.taxonomy.lucene.LuceneTaxonomyWriter

    Directory copydirs[] = new Directory[ntaxonomies];


    for (int i=0; i<ntaxonomies; i++) {
      dirs[i] = newDirectory();
      copydirs[i] = newDirectory();
      LuceneTaxonomyWriter tw = new LuceneTaxonomyWriter(dirs[i]);
      LuceneTaxonomyWriter copytw = new LuceneTaxonomyWriter(copydirs[i]);
      for (int j=0; j<ncats; j++) {
        String cat = Integer.toString(random.nextInt(range));
        tw.addCategory(new CategoryPath("a",cat));
        copytw.addCategory(new CategoryPath("a",cat));
      }
      // System.err.println("Taxonomy "+i+": "+tw.getSize());
      tw.close();
      copytw.close();
    }


    LuceneTaxonomyWriter tw = new LuceneTaxonomyWriter(dirs[0]);
    Directory otherdirs[] = new Directory[ntaxonomies-1];
    System.arraycopy(dirs, 1, otherdirs, 0, ntaxonomies-1);


    OrdinalMap[] maps = new OrdinalMap[ntaxonomies-1];
    if (ntaxonomies>1) {
      for (int i=0; i<ntaxonomies-1; i++) {
        if (disk) {
          // TODO: use a LTC tempfile
          maps[i] = new DiskOrdinalMap(new File(System.getProperty("java.io.tmpdir"),
              "tmpmap"+i));
        } else {
          maps[i] = new MemoryOrdinalMap();
        }
      }
    }


    tw.addTaxonomies(otherdirs, maps);
    // System.err.println("Merged axonomy: "+tw.getSize());
    tw.close();


    // Check that all original categories in the main taxonomy remain in
    // unchanged, and the rest of the taxonomies are completely unchanged.
    for (int i=0; i<ntaxonomies; i++) {
      TaxonomyReader tr = new LuceneTaxonomyReader(dirs[i]);

View Full Code Here

Examples of org.apache.lucene.facet.taxonomy.lucene.LuceneTaxonomyWriter

  public void testDefaultSettings() throws Exception {
    FacetSearchParams fsp = new FacetSearchParams();
    assertEquals("unexpected default facet indexing params class", DefaultFacetIndexingParams.class.getName(), fsp.getFacetIndexingParams().getClass().getName());
    assertEquals("no facet requests should be added by default", 0, fsp.getFacetRequests().size());
    Directory dir = newDirectory();
    new LuceneTaxonomyWriter(dir).close();
    TaxonomyReader tr = new LuceneTaxonomyReader(dir);
    assertEquals("unexpected partition offset for 0 categories", 1, PartitionsUtils.partitionOffset(fsp, 1, tr));
    assertEquals("unexpected partition size for 0 categories", 1, PartitionsUtils.partitionSize(fsp,tr));
    tr.close();
    dir.close();

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.