Package org.apache.lucene.facet.index.streaming

Examples of org.apache.lucene.facet.index.streaming.CategoryTokenizer

176
177
178
179
180
181
182
183
184
185
186
      // CategoryParentsStream, followed by CategoryListTokenizer and
      // CategoryTokenizer composition (the ordering of the last two is
      // not mandatory).
      CategoryParentsStream parentsStream = (CategoryParentsStream) getParentsStream(categoryAttributesStream);
      CategoryListTokenizer categoryListTokenizer = getCategoryListTokenizer(parentsStream);
      CategoryTokenizer stream = getCategoryTokenizer(categoryListTokenizer);

      // Finally creating a suitable field with stream and adding it to a
      // master field-list, used during the build process (see
      // super.build())
      fieldList.add(new Field(e.getKey(), stream));
View Full Code Here
279
280
281
282
283
284
285
   *         tokens.
   * @throws IOException
   */
  protected CategoryTokenizer getCategoryTokenizer(TokenStream categoryStream)
      throws IOException {
    return new CategoryTokenizer(categoryStream, indexingParams);
  }
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
  public void testTokensDefaultParams() throws IOException {
    Directory directory = newDirectory();
    TaxonomyWriter taxonomyWriter = new LuceneTaxonomyWriter(
        directory);
    DefaultFacetIndexingParams indexingParams = new DefaultFacetIndexingParams();
    CategoryTokenizer tokenizer = new CategoryTokenizer(
        new CategoryAttributesStream(categoryContainer),
        indexingParams);

    // count the number of tokens
    Set<String> categoryTerms = new HashSet<String>();
    for (int i = 0; i < initialCatgeories.length; i++) {
      categoryTerms.add(initialCatgeories[i]
          .toString(indexingParams.getFacetDelimChar()));
    }

    int nTokens;
    for (nTokens = 0; tokenizer.incrementToken(); nTokens++) {
      if (!categoryTerms.remove(tokenizer.termAttribute.toString())) {
        fail("Unexpected term: " + tokenizer.termAttribute.toString());
      }
    }
    assertTrue("all category terms should have been found", categoryTerms
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
    List<CategoryPath> longCategory = new ArrayList<CategoryPath>();
    longCategory.add(new CategoryPath("one", "two", "three", "four",
        "five", "six", "seven"));

    DefaultFacetIndexingParams indexingParams = new DefaultFacetIndexingParams();
    CategoryTokenizer tokenizer = new CategoryTokenizer(
        new CategoryAttributesStream(new CategoryAttributesIterable(
            longCategory)), indexingParams);

    // count the number of tokens
    String categoryTerm = longCategory.get(0).toString(
        indexingParams.getFacetDelimChar());

    assertTrue("Missing token", tokenizer.incrementToken());
    if (!categoryTerm.equals(tokenizer.termAttribute.toString())) {
      fail("Unexpected term: " + tokenizer.termAttribute.toString());
    }

    assertFalse("Unexpected token", tokenizer.incrementToken());

    taxonomyWriter.close();
    directory.close();
  }
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
  public void testTokensDefaultParams() throws IOException {
    Directory directory = newDirectory();
    TaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter(
        directory);
    DefaultFacetIndexingParams indexingParams = new DefaultFacetIndexingParams();
    CategoryTokenizer tokenizer = new CategoryTokenizer(
        new CategoryAttributesStream(categoryContainer),
        indexingParams);

    // count the number of tokens
    Set<String> categoryTerms = new HashSet<String>();
    for (int i = 0; i < initialCatgeories.length; i++) {
      categoryTerms.add(initialCatgeories[i]
          .toString(indexingParams.getFacetDelimChar()));
    }

    int nTokens;
    for (nTokens = 0; tokenizer.incrementToken(); nTokens++) {
      if (!categoryTerms.remove(tokenizer.termAttribute.toString())) {
        fail("Unexpected term: " + tokenizer.termAttribute.toString());
      }
    }
    assertTrue("all category terms should have been found", categoryTerms
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
    List<CategoryPath> longCategory = new ArrayList<CategoryPath>();
    longCategory.add(new CategoryPath("one", "two", "three", "four",
        "five", "six", "seven"));

    DefaultFacetIndexingParams indexingParams = new DefaultFacetIndexingParams();
    CategoryTokenizer tokenizer = new CategoryTokenizer(
        new CategoryAttributesStream(new CategoryAttributesIterable(
            longCategory)), indexingParams);

    // count the number of tokens
    String categoryTerm = longCategory.get(0).toString(
        indexingParams.getFacetDelimChar());

    assertTrue("Missing token", tokenizer.incrementToken());
    if (!categoryTerm.equals(tokenizer.termAttribute.toString())) {
      fail("Unexpected term: " + tokenizer.termAttribute.toString());
    }

    assertFalse("Unexpected token", tokenizer.incrementToken());

    taxonomyWriter.close();
    directory.close();
  }
View Full Code Here
175
176
177
178
179
180
181
182
183
184
185
      // CategoryParentsStream, followed by CategoryListTokenizer and
      // CategoryTokenizer composition (the ordering of the last two is
      // not mandatory).
      CategoryParentsStream parentsStream = (CategoryParentsStream) getParentsStream(categoryAttributesStream);
      CategoryListTokenizer categoryListTokenizer = getCategoryListTokenizer(parentsStream);
      CategoryTokenizer stream = getCategoryTokenizer(categoryListTokenizer);

      // Finally creating a suitable field with stream and adding it to a
      // master field-list, used during the build process (see
      // super.build())
      fieldList.add(new Field(e.getKey(), stream));
View Full Code Here
278
279
280
281
282
283
284
   *         tokens.
   * @throws IOException
   */
  protected CategoryTokenizer getCategoryTokenizer(TokenStream categoryStream)
      throws IOException {
    return new CategoryTokenizer(categoryStream, indexingParams);
  }
View Full Code Here
TOP

Related Classes of org.apache.lucene.facet.index.streaming.CategoryTokenizer

  • org.apache.lucene.facet.index.CategoryDocumentBuilder
  • org.apache.lucene.facet.index.streaming.CategoryTokenizerTest
  • org.apache.lucene.facet.taxonomy.CategoryPath
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.