Examples of KeywordMarkerFilter

Examples of org.apache.lucene.analysis.KeywordMarkerFilter

56
57
58
59
60
61
62
  public boolean isIgnoreCase() {
    return ignoreCase;
  }

  public TokenStream create(TokenStream input) {
    return protectedWords == null ? input : new KeywordMarkerFilter(input, protectedWords);
  }
View Full Code Here

Examples of org.apache.lucene.analysis.KeywordMarkerFilter

56
57
58
59
60
61
62
63
  private CharArraySet protectedWords = null;

  public TokenFilter create(TokenStream input) {
    if (protectedWords != null)
      input = new KeywordMarkerFilter(input, protectedWords);
    return new SnowballFilter(input, new org.tartarus.snowball.ext.EnglishStemmer());
  }
View Full Code Here

Examples of org.apache.lucene.analysis.KeywordMarkerFilter

83
84
85
86
87
88
89
90
    } catch (Exception e) {
      throw new RuntimeException("Error instantiating stemmer for language " + language + "from class " +stemClass, e);
    }

    if (protectedWords != null)
      input = new KeywordMarkerFilter(input, protectedWords);
    return new SnowballFilter(input, program);
  }
View Full Code Here

Examples of org.apache.lucene.analysis.KeywordMarkerFilter

218
219
220
221
222
223
224
225
    Tokenizer source = new StandardTokenizer(matchVersion, reader);
    TokenStream result = new LowerCaseFilter(matchVersion, source);
    result = new StandardFilter(matchVersion, result);
    result = new StopFilter(matchVersion, result, stopwords);
    if(excltable != null && !excltable.isEmpty())
      result = new KeywordMarkerFilter(result, excltable);
    return new TokenStreamComponents(source, new BrazilianStemFilter(result));
  }
View Full Code Here

Examples of org.apache.lucene.analysis.KeywordMarkerFilter

142
143
144
145
146
147
148
149
150
      result = new ElisionFilter(matchVersion, result, DEFAULT_ARTICLES);
    }
    result = new LowerCaseFilter(matchVersion, result);
    result = new StopFilter(matchVersion, result, stopwords);
    if(!stemExclusionSet.isEmpty())
      result = new KeywordMarkerFilter(result, stemExclusionSet);
    result = new SnowballFilter(result, new ItalianStemmer());
    return new TokenStreamComponents(source, result);
  }
View Full Code Here

Examples of org.apache.lucene.analysis.KeywordMarkerFilter

48
49
50
51
52
53
54
        }
        keywordLookup = new CharArraySet(Version.LUCENE_32, rules, ignoreCase);
    }

    @Override public TokenStream create(TokenStream tokenStream) {
        return new KeywordMarkerFilter(tokenStream, keywordLookup);
    }
View Full Code Here

Examples of org.apache.lucene.analysis.KeywordMarkerFilter

120
121
122
123
124
125
126
127
128
    final Tokenizer source = new StandardTokenizer(matchVersion, reader);
    TokenStream result = new StandardFilter(matchVersion, source);
    result = new LowerCaseFilter(matchVersion, result);
    result = new StopFilter(matchVersion, result, stopwords);
    if(!stemExclusionSet.isEmpty())
      result = new KeywordMarkerFilter(result, stemExclusionSet);
    result = new LatvianStemFilter(result);
    return new TokenStreamComponents(source, result);
  }
View Full Code Here

Examples of org.apache.lucene.analysis.KeywordMarkerFilter

46
47
48
49
50
51
52
53
54
  public void testWithKeywordAttribute() throws IOException {
    CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, 1, true);
    set.add("fischen");
    GermanStemFilter filter = new GermanStemFilter(
        new KeywordMarkerFilter(new LowerCaseTokenizer(TEST_VERSION_CURRENT, new StringReader(
            "Fischen Trinken")), set));
    assertTokenStreamContents(filter, new String[] { "fischen", "trink" });
  }
View Full Code Here

Examples of org.apache.lucene.analysis.KeywordMarkerFilter

58
59
60
61
62
63
64
65
66
67
    set.add("fischen");
    CharArraySet set1 = new CharArraySet(TEST_VERSION_CURRENT, 1, true);
    set1.add("trinken");
    set1.add("fischen");
    GermanStemFilter filter = new GermanStemFilter(
        new KeywordMarkerFilter(new LowerCaseTokenizer(TEST_VERSION_CURRENT, new StringReader(
            "Fischen Trinken")), set));
    filter.setExclusionSet(set1);
    assertTokenStreamContents(filter, new String[] { "fischen", "trinken" });
  }
View Full Code Here

Examples of org.apache.lucene.analysis.KeywordMarkerFilter

275
276
277
278
279
280
281
282
283
  }
 
  public void testWithKeywordAttribute() throws IOException {
    CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, 1, true);
    set.add("hole");
    CzechStemFilter filter = new CzechStemFilter(new KeywordMarkerFilter(
        new MockTokenizer(new StringReader("hole desek"), MockTokenizer.WHITESPACE, false), set));
    assertTokenStreamContents(filter, new String[] { "hole", "desk" });
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.