Examples of StandardTokenizerFactory

Examples of org.apache.lucene.analysis.standard.StandardTokenizerFactory

35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
  /**
   * Test IndicNormalizationFilterFactory
   */
  public void testIndicNormalizer() throws Exception {
    Reader reader = new StringReader("ত্‍ अाैर");
    StandardTokenizerFactory factory = new StandardTokenizerFactory();
    factory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
    IndicNormalizationFilterFactory filterFactory = new IndicNormalizationFilterFactory();
    filterFactory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
    Map<String, String> args = Collections.emptyMap();
    factory.init(args);
    filterFactory.init(args);
    Tokenizer tokenizer = factory.create(reader);
    TokenStream stream = filterFactory.create(tokenizer);
    assertTokenStreamContents(stream, new String[] { "ৎ", "और" });
  }
View Full Code Here

Examples of org.apache.lucene.analysis.standard.StandardTokenizerFactory

52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
  /**
   * Test HindiNormalizationFilterFactory
   */
  public void testHindiNormalizer() throws Exception {
    Reader reader = new StringReader("क़िताब");
    StandardTokenizerFactory factory = new StandardTokenizerFactory();
    factory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
    IndicNormalizationFilterFactory indicFilterFactory = new IndicNormalizationFilterFactory();
    HindiNormalizationFilterFactory hindiFilterFactory = new HindiNormalizationFilterFactory();
    hindiFilterFactory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
    Map<String, String> args = Collections.emptyMap();
    factory.init(args);
    hindiFilterFactory.init(args);
    Tokenizer tokenizer = factory.create(reader);
    TokenStream stream = indicFilterFactory.create(tokenizer);
    stream = hindiFilterFactory.create(stream);
    assertTokenStreamContents(stream, new String[] {"किताब"});
  }
View Full Code Here

Examples of org.apache.lucene.analysis.standard.StandardTokenizerFactory

71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
  /**
   * Test HindiStemFilterFactory
   */
  public void testStemmer() throws Exception {
    Reader reader = new StringReader("किताबें");
    StandardTokenizerFactory factory = new StandardTokenizerFactory();
    factory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
    IndicNormalizationFilterFactory indicFilterFactory = new IndicNormalizationFilterFactory();
    HindiNormalizationFilterFactory hindiFilterFactory = new HindiNormalizationFilterFactory();
    HindiStemFilterFactory stemFactory = new HindiStemFilterFactory();
    stemFactory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
    Map<String, String> args = Collections.emptyMap();
    factory.init(args);
    stemFactory.init(args);
    Tokenizer tokenizer = factory.create(reader);
    TokenStream stream = indicFilterFactory.create(tokenizer);
    stream = hindiFilterFactory.create(stream);
    stream = stemFactory.create(stream);
    assertTokenStreamContents(stream, new String[] {"किताब"});
  }
View Full Code Here

Examples of org.apache.lucene.analysis.standard.StandardTokenizerFactory

50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
  /**
   * Test ArabicNormalizationFilterFactory
   */
  public void testNormalizer() throws Exception {
    Reader reader = new StringReader("الذين مَلكت أيمانكم");
    StandardTokenizerFactory factory = new StandardTokenizerFactory();
    factory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
    ArabicNormalizationFilterFactory filterFactory = new ArabicNormalizationFilterFactory();
    filterFactory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
    Map<String, String> args = Collections.emptyMap();
    factory.init(args);
    filterFactory.init(args);
    Tokenizer tokenizer = factory.create(reader);
    TokenStream stream = filterFactory.create(tokenizer);
    assertTokenStreamContents(stream, new String[] {"الذين", "ملكت", "ايمانكم"});
  }
View Full Code Here

Examples of org.apache.lucene.analysis.standard.StandardTokenizerFactory

67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
  /**
   * Test ArabicStemFilterFactory
   */
  public void testStemmer() throws Exception {
    Reader reader = new StringReader("الذين مَلكت أيمانكم");
    StandardTokenizerFactory factory = new StandardTokenizerFactory();
    factory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
    ArabicNormalizationFilterFactory normFactory = new ArabicNormalizationFilterFactory();
    normFactory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
    ArabicStemFilterFactory stemFactory = new ArabicStemFilterFactory();
    Map<String, String> args = Collections.emptyMap();
    factory.init(args);
    normFactory.init(args);
    Tokenizer tokenizer = factory.create(reader);
    TokenStream stream = normFactory.create(tokenizer);
    stream = stemFactory.create(stream);
    assertTokenStreamContents(stream, new String[] {"ذين", "ملكت", "ايمانكم"});
  }
View Full Code Here

Examples of org.apache.lucene.analysis.standard.StandardTokenizerFactory

87
88
89
90
91
92
93
94
95
96
97
98
   * Test PersianCharFilterFactory
   */
  public void testPersianCharFilter() throws Exception {
    Reader reader = new StringReader("می‌خورد");
    PersianCharFilterFactory charfilterFactory = new PersianCharFilterFactory();
    StandardTokenizerFactory tokenizerFactory = new StandardTokenizerFactory();
    tokenizerFactory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
    Map<String, String> args = Collections.emptyMap();
    tokenizerFactory.init(args);
    TokenStream stream = tokenizerFactory.create(charfilterFactory.create(reader));
    assertTokenStreamContents(stream, new String[] { "می", "خورد" });
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.