Package org.apache.lucene.facet.taxonomy.directory

Examples of org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter


    Directory taxoDir = newDirectory();
    writer = new RandomIndexWriter(random(), dir);

    // Writes facet ords to a separate directory from the
    // main index:
    taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);

    // Reused across documents, to add the necessary facet
    // fields:
    facetFields = new FacetFields(taxoWriter);
View Full Code Here


          assertEquals("field", name);
          return sim;
        }
      });
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);
    FacetFields facetFields = new FacetFields(taxoWriter);     

    Document doc = new Document();
    doc.add(newTextField("field", "text", Field.Store.NO));
    facetFields.addFields(doc, Collections.singletonList(new CategoryPath("a/path", '/')));
    writer.addDocument(doc);
    writer.close();
    taxoWriter.close();
    dir.close();
    taxoDir.close();
  }
View Full Code Here

  public void testAllParents() throws Exception {
    Directory dir = newDirectory();
    Directory taxoDir = newDirectory();
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
    DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);

    CategoryListParams clp = new CategoryListParams("$facets") {
        @Override
        public OrdinalPolicy getOrdinalPolicy(String fieldName) {
          return OrdinalPolicy.ALL_PARENTS;
        }
      };
    FacetIndexingParams fip = new FacetIndexingParams(clp);

    FacetFields facetFields = new FacetFields(taxoWriter, fip);

    Document doc = new Document();
    doc.add(newTextField("field", "text", Field.Store.NO));
    facetFields.addFields(doc, Collections.singletonList(new CategoryPath("a/path", '/')));
    writer.addDocument(doc);

    // NRT open
    IndexSearcher searcher = newSearcher(writer.getReader());
    writer.close();

    // NRT open
    TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);
    taxoWriter.close();
   
    FacetSearchParams fsp = new FacetSearchParams(fip,
                                                  new CountFacetRequest(new CategoryPath("a", '/'), 10));

    // Aggregate the facet counts:
View Full Code Here

  public void testLabelWithDelimiter() throws Exception {
    Directory dir = newDirectory();
    Directory taxoDir = newDirectory();
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
    DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);

    FacetFields facetFields = new FacetFields(taxoWriter);

    Document doc = new Document();
    doc.add(newTextField("field", "text", Field.Store.NO));
    BytesRef br = new BytesRef(new byte[] {(byte) 0xee, (byte) 0x92, (byte) 0xaa, (byte) 0xef, (byte) 0x9d, (byte) 0x89});
    facetFields.addFields(doc, Collections.singletonList(new CategoryPath("dim/" + br.utf8ToString(), '/')));
    try {
      writer.addDocument(doc);
    } catch (IllegalArgumentException iae) {
      // expected
    }
    writer.close();
    taxoWriter.close();
    dir.close();
    taxoDir.close();
  }
View Full Code Here

    assumeTrue("default Codec doesn't support huge BinaryDocValues", _TestUtil.fieldSupportsHugeBinaryDocValues(CategoryListParams.DEFAULT_FIELD));
    Directory dir = newDirectory();
    Directory taxoDir = newDirectory();
    IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
    DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);
   
    FacetFields facetFields = new FacetFields(taxoWriter);
   
    int numLabels = _TestUtil.nextInt(random(), 40000, 100000);
   
    Document doc = new Document();
    doc.add(newTextField("field", "text", Field.Store.NO));
    List<CategoryPath> paths = new ArrayList<CategoryPath>();
    for (int i = 0; i < numLabels; i++) {
      paths.add(new CategoryPath("dim", "" + i));
    }
    facetFields.addFields(doc, paths);
    writer.addDocument(doc);
   
    // NRT open
    IndexSearcher searcher = newSearcher(writer.getReader());
    writer.close();
   
    // NRT open
    TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);
    taxoWriter.close();
   
    FacetSearchParams fsp = new FacetSearchParams(new CountFacetRequest(new CategoryPath("dim"), Integer.MAX_VALUE));
   
    // Aggregate the facet counts:
    FacetsCollector c = FacetsCollector.create(fsp, searcher.getIndexReader(), taxoReader);
View Full Code Here

    Random r = random();
    RandomIndexWriter writer = new RandomIndexWriter(r, dir,
        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(r, MockTokenizer.KEYWORD, false)));
   
    taxoDir = newDirectory();
    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
   
    for (int i = 0; i < 100; i++) {
      ArrayList<CategoryPath> paths = new ArrayList<CategoryPath>();
      Document doc = new Document();
      if (i % 2 == 0) { // 50
        doc.add(new TextField("content", "foo", Field.Store.NO));
      }
      if (i % 3 == 0) { // 33
        doc.add(new TextField("content", "bar", Field.Store.NO));
      }
      if (i % 4 == 0) { // 25
        if (r.nextBoolean()) {
          paths.add(new CategoryPath("a/1", '/'));
        } else {
          paths.add(new CategoryPath("a/2", '/'));
        }
      }
      if (i % 5 == 0) { // 20
        paths.add(new CategoryPath("b"));
      }
      FacetFields facetFields = new FacetFields(taxoWriter);
      if (paths.size() > 0) {
        facetFields.addFields(doc, paths);
      }
      writer.addDocument(doc);
    }
   
    taxoWriter.close();
    reader = writer.getReader();
    writer.close();
   
    taxo = new DirectoryTaxonomyReader(taxoDir);
  }
View Full Code Here

    // LUCENE-5303: OrdinalsCache used the ThreadLocal BinaryDV instead of reader.getCoreCacheKey().
    Directory indexDir = newDirectory();
    Directory taxoDir = newDirectory();
    IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
    IndexWriter writer = new IndexWriter(indexDir, conf);
    DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
    FacetFields facetFields = new FacetFields(taxoWriter);
   
    Document doc = new Document();
    facetFields.addFields(doc, Arrays.asList(new CategoryPath("A", "1")));
    writer.addDocument(doc);
    doc = new Document();
    facetFields.addFields(doc, Arrays.asList(new CategoryPath("A", "2")));
    writer.addDocument(doc);
    writer.close();
    taxoWriter.close();
   
    final DirectoryReader reader = DirectoryReader.open(indexDir);
    Thread[] threads = new Thread[3];
    for (int i = 0; i < threads.length; i++) {
      threads[i] = new Thread("CachedOrdsThread-" + i) {
View Full Code Here

   * Test how getChildrenArrays() deals with the taxonomy's growth:
   */
  @Test
  public void testChildrenArraysGrowth() throws Exception {
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    tw.addCategory(new CategoryPath("hi", "there"));
    tw.commit();
    TaxonomyReader tr = new DirectoryTaxonomyReader(indexDir);
    ParallelTaxonomyArrays ca = tr.getParallelTaxonomyArrays();
    assertEquals(3, tr.getSize());
    assertEquals(3, ca.siblings().length);
    assertEquals(3, ca.children().length);
    assertTrue(Arrays.equals(new int[] { 1, 2, -1 }, ca.children()));
    assertTrue(Arrays.equals(new int[] { -1, -1, -1 }, ca.siblings()));
    tw.addCategory(new CategoryPath("hi", "ho"));
    tw.addCategory(new CategoryPath("hello"));
    tw.commit();
    // Before refresh, nothing changed..
    ParallelTaxonomyArrays newca = tr.getParallelTaxonomyArrays();
    assertSame(newca, ca); // we got exactly the same object
    assertEquals(3, tr.getSize());
    assertEquals(3, ca.siblings().length);
    assertEquals(3, ca.children().length);
    // After the refresh, things change:
    TaxonomyReader newtr = TaxonomyReader.openIfChanged(tr);
    assertNotNull(newtr);
    tr.close();
    tr = newtr;
    ca = tr.getParallelTaxonomyArrays();
    assertEquals(5, tr.getSize());
    assertEquals(5, ca.siblings().length);
    assertEquals(5, ca.children().length);
    assertTrue(Arrays.equals(new int[] { 4, 3, -1, -1, -1 }, ca.children()));
    assertTrue(Arrays.equals(new int[] { -1, -1, -1, 2, 1 }, ca.siblings()));
    tw.close();
    tr.close();
    indexDir.close();
  }
View Full Code Here

  // Test that getParentArrays is valid when retrieved during refresh
  @Test
  public void testTaxonomyReaderRefreshRaces() throws Exception {
    // compute base child arrays - after first chunk, and after the other
    Directory indexDirBase = newDirectory();
    TaxonomyWriter twBase = new DirectoryTaxonomyWriter(indexDirBase);
    twBase.addCategory(new CategoryPath("a", "0"));
    final CategoryPath abPath = new CategoryPath("a", "b");
    twBase.addCategory(abPath);
    twBase.commit();
    TaxonomyReader trBase = new DirectoryTaxonomyReader(indexDirBase);

    final ParallelTaxonomyArrays ca1 = trBase.getParallelTaxonomyArrays();
   
    final int abOrd = trBase.getOrdinal(abPath);
    final int abYoungChildBase1 = ca1.children()[abOrd];
   
    final int numCategories = atLeast(800);
    for (int i = 0; i < numCategories; i++) {
      twBase.addCategory(new CategoryPath("a", "b", Integer.toString(i)));
    }
    twBase.close();
   
    TaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(trBase);
    assertNotNull(newTaxoReader);
    trBase.close();
    trBase = newTaxoReader;
View Full Code Here

  private void assertConsistentYoungestChild(final CategoryPath abPath,
      final int abOrd, final int abYoungChildBase1, final int abYoungChildBase2, final int retry, int numCategories)
      throws Exception {
    SlowRAMDirectory indexDir = new SlowRAMDirectory(-1, null); // no slowness for intialization
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    tw.addCategory(new CategoryPath("a", "0"));
    tw.addCategory(abPath);
    tw.commit();
   
    final DirectoryTaxonomyReader tr = new DirectoryTaxonomyReader(indexDir);
    for (int i = 0; i < numCategories; i++) {
      final CategoryPath cp = new CategoryPath("a", "b", Integer.toString(i));
      tw.addCategory(cp);
      assertEquals("Ordinal of "+cp+" must be invalid until Taxonomy Reader was refreshed", TaxonomyReader.INVALID_ORDINAL, tr.getOrdinal(cp));
    }
    tw.close();
   
    final AtomicBoolean stop = new AtomicBoolean(false);
    final Throwable[] error = new Throwable[] { null };
    final int retrieval[] = { 0 };
   
View Full Code Here

TOP

Related Classes of org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.