Package org.apache.lucene.facet.taxonomy

Examples of org.apache.lucene.facet.taxonomy.TaxonomyWriter


    IndexAccessor indexAccessor = null;
    IndexWriter indexWriter = null;
    IndexReader indexReader = null;
    TaxonomyAccessor taxonomyAccessor = null;
    TaxonomyWriter taxonomyWriter = null;
    LuceneIndexUpdateChecker luceneIndexUpdateChecker = null;
    boolean finishedIndexJobSuccessfull = false;
    boolean finishedIndexJobWithError = false;

    try {
View Full Code Here


        pair.searchDir = newDirectory();
        pair.taxoDir = newDirectory();
      }
     
      RandomIndexWriter iw = new RandomIndexWriter(random(), pair.searchDir, getIndexWriterConfig(getAnalyzer()));
      TaxonomyWriter taxo = new DirectoryTaxonomyWriter(pair.taxoDir, OpenMode.CREATE);
     
      populateIndex(iw, taxo, fip);
     
      // commit changes (taxonomy prior to search index for consistency)
      taxo.commit();
      iw.commit();
      taxo.close();
      iw.close();
     
      dirsPerPartitionSize.put(Integer.valueOf(partitionSize), pair);
    }
   
View Full Code Here

  }

  private void index100Docs(Directory indexDir, Directory taxoDir, FacetIndexingParams fip) throws IOException {
    IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
    IndexWriter w = new IndexWriter(indexDir, iwc);
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);
   
    FacetFields facetFields = new FacetFields(tw, fip);
    for (int i = 0; i < 100; i++) {
      Document doc = new Document();
      CategoryPath cp = new CategoryPath("root",Integer.toString(i / 10), Integer.toString(i));
View Full Code Here

  private static final String[] CATEGORIES = new String[] { "a/b", "c/d", "a/e", "a/d", "c/g", "c/z", "b/a", "1/2", "b/c" };

  private void index(Directory indexDir, Directory taxoDir) throws IOException {
    IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
    FacetFields facetFields = new FacetFields(taxoWriter);
   
    for (String cat : CATEGORIES) {
      Document doc = new Document();
      facetFields.addFields(doc, Collections.singletonList(new CategoryPath(cat, '/')));
View Full Code Here

    Directory indexDir = newDirectory();
    Directory taxoDir = newDirectory();

    // Create our index/taxonomy writers
    IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
    FacetIndexingParams iParams = FacetIndexingParams.DEFAULT;

    // Add a facet to the index
    addFacets(iParams, indexWriter, taxoWriter, "a", "b");

    // Commit Changes
    indexWriter.commit();
    taxoWriter.commit();

    // Open readers
    DirectoryReader indexReader = DirectoryReader.open(indexDir);
    TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
View Full Code Here

    Directory indexDir = newDirectory();
    Directory taxoDir = newDirectory();
   
    // Create our index/taxonomy writers
    IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
    FacetIndexingParams iParams = new FacetIndexingParams() {
      @Override
      public int getPartitionSize() {
        return 2;
      }
    };
    // Add a facet to the index
    addFacets(iParams, indexWriter, taxoWriter, "a", "b");
    // Commit Changes
    indexWriter.commit();
    taxoWriter.commit();

    DirectoryReader indexReader = DirectoryReader.open(indexDir);
    TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);

    // Create TFC and write cache to disk
    File outputFile = _TestUtil.createTempFile("test", "tmp", TEMP_DIR);
    TFC.store(outputFile, indexReader, taxoReader, iParams);
   
    // Make the taxonomy grow without touching the index
    for (int i = 0; i < 10; i++) {
      taxoWriter.addCategory(new CategoryPath("foo", Integer.toString(i)));
    }
    taxoWriter.commit();
    TaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(taxoReader);
    assertNotNull(newTaxoReader);
    taxoReader.close();
    taxoReader = newTaxoReader;
View Full Code Here

    Directory taxoDir1 = newDirectory(), taxoDir2 = newDirectory();
   
    // Create our index/taxonomy writers
    IndexWriter indexWriter1 = new IndexWriter(indexDir1, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
    IndexWriter indexWriter2 = new IndexWriter(indexDir2, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
    TaxonomyWriter taxoWriter1 = new DirectoryTaxonomyWriter(taxoDir1);
    TaxonomyWriter taxoWriter2 = new DirectoryTaxonomyWriter(taxoDir2);
    FacetIndexingParams iParams = FacetIndexingParams.DEFAULT;

    // Add a facet to the index
    addFacets(iParams, indexWriter1, taxoWriter1, "a", "b");
    addFacets(iParams, indexWriter1, taxoWriter1, "d", "e");
    // Commit Changes
    indexWriter1.commit();
    indexWriter2.commit();
    taxoWriter1.commit();
    taxoWriter2.commit();

    // Open two readers
    DirectoryReader indexReader1 = DirectoryReader.open(indexDir1);
    DirectoryReader indexReader2 = DirectoryReader.open(indexDir2);
    TaxonomyReader taxoReader1 = new DirectoryTaxonomyReader(taxoDir1);
View Full Code Here

    taxoDir = newDirectory();
    // preparations - index, taxonomy, content
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(TEST_VERSION_CURRENT,
        new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)));
   
    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
   
    AssociationsFacetFields assocFacetFields = new AssociationsFacetFields(taxoWriter);
   
    // index documents, 50% have only 'b' and all have 'a'
    for (int i = 0; i < 110; i++) {
      Document doc = new Document();
      CategoryAssociationsContainer associations = new CategoryAssociationsContainer();
      // every 11th document is added empty, this used to cause the association
      // aggregators to go into an infinite loop
      if (i % 11 != 0) {
        associations.setAssociation(aint, new CategoryIntAssociation(2));
        associations.setAssociation(afloat, new CategoryFloatAssociation(0.5f));
        if (i % 2 == 0) { // 50
          associations.setAssociation(bint, new CategoryIntAssociation(3));
          associations.setAssociation(bfloat, new CategoryFloatAssociation(0.2f));
        }
      }
      assocFacetFields.addFields(doc, associations);
      writer.addDocument(doc);
    }
   
    taxoWriter.close();
    reader = writer.getReader();
    writer.close();
  }
View Full Code Here

    initCache();

    Directory indexDir = newDirectory();
    Directory taxoDir = newDirectory();
    IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
   
    FacetIndexingParams iParams = new FacetIndexingParams() {
      @Override
      public int getPartitionSize() {
        return partitionSize;
View Full Code Here

    Random random = random();
    IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
    conf.setMaxBufferedDocs(2); // force few segments
    conf.setMergePolicy(NoMergePolicy.COMPOUND_FILES); // avoid merges so that we're left with few segments
    IndexWriter indexWriter = new IndexWriter(indexDir, conf);
    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
   
    FacetFields facetFields = new PayloadFacetFields(taxoWriter, fip);
   
    HashMap<String,Integer> expectedCounts = new HashMap<String,Integer>(DIMENSIONS.length);
    int numDocs = atLeast(10);
    for (int i = 0; i < numDocs; i++) {
      Document doc = new Document();
      int numCategories = random.nextInt(3) + 1;
      ArrayList<CategoryPath> categories = new ArrayList<CategoryPath>(numCategories);
      HashSet<String> docDimensions = new HashSet<String>();
      while (numCategories-- > 0) {
        String dim = DIMENSIONS[random.nextInt(DIMENSIONS.length)];
        // we should only increment the expected count by 1 per document
        docDimensions.add(dim);
        categories.add(new CategoryPath(dim, Integer.toString(i), Integer.toString(numCategories)));
      }
      facetFields.addFields(doc, categories);
      doc.add(new StringField("docid", Integer.toString(i), Store.YES));
      doc.add(new TextField("foo", "content" + i, Store.YES));
      indexWriter.addDocument(doc);

      // update expected count per dimension
      for (String dim : docDimensions) {
        Integer val = expectedCounts.get(dim);
        if (val == null) {
          expectedCounts.put(dim, Integer.valueOf(1));
        } else {
          expectedCounts.put(dim, Integer.valueOf(val.intValue() + 1));
        }
      }
     
      if (random.nextDouble() < 0.2) { // add some documents that will be deleted
        doc = new Document();
        doc.add(new StringField("del", "key", Store.NO));
        facetFields.addFields(doc, Collections.singletonList(new CategoryPath("dummy")));
        indexWriter.addDocument(doc);
      }
    }
   
    indexWriter.commit();
    taxoWriter.commit();

    // delete the docs that were marked for deletion. note that the 'dummy'
    // category is not removed from the taxonomy, so must account for it when we
    // verify the migrated index.
    indexWriter.deleteDocuments(new Term("del", "key"));
View Full Code Here

TOP

Related Classes of org.apache.lucene.facet.taxonomy.TaxonomyWriter

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.