Package org.apache.lucene.analysis.miscellaneous

Examples of org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter

183
184
185
186
187
188
189
190
191
192
193
    final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("chevaux"), false);
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
        TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
        return new TokenStreamComponents(source, new FrenchLightStemFilter(sink));
      }
    };
    checkOneTerm(a, "chevaux", "chevaux");
  }
View Full Code Here
142
143
144
145
146
147
148
149
150
    result = s;
    result = new ElisionFilter(result, DEFAULT_ARTICLES);
    result = new IrishLowerCaseFilter(result);
    result = new StopFilter(matchVersion, result, stopwords);
    if(!stemExclusionSet.isEmpty())
      result = new SetKeywordMarkerFilter(result, stemExclusionSet);
    result = new SnowballFilter(result, new IrishStemmer());
    return new TokenStreamComponents(source, result);
  }
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
    final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("sekretæren"), false);
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
        TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
        return new TokenStreamComponents(source, new NorwegianMinimalStemFilter(sink));
      }
    };
    checkOneTerm(a, "sekretæren", "sekretæren");
  }
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
    final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("sekretæren"), false);
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
        TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
        return new TokenStreamComponents(source, new NorwegianLightStemFilter(sink));
      }
    };
    checkOneTerm(a, "sekretæren", "sekretæren");
  }
View Full Code Here
61
62
63
64
65
66
67
68
    assertTokenStreamContents(filter, new String[]{"lucene", "lucen", "is", "awesome"}, new int[] {1, 0, 1, 1});
   
    // assert with keywork marker
    tokenizer = new MockTokenizer(new StringReader("lucene is awesome"), MockTokenizer.WHITESPACE, true);
    CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, Arrays.asList("Lucene"), true);
    filter = new HunspellStemFilter(new SetKeywordMarkerFilter(tokenizer, set), DICTIONARY);
    assertTokenStreamContents(filter, new String[]{"lucene", "is", "awesome"}, new int[] {1, 1, 1});
  }
View Full Code Here
144
145
146
147
148
149
150
151
152
    result = s;
    result = new ElisionFilter(result, DEFAULT_ARTICLES);
    result = new IrishLowerCaseFilter(result);
    result = new StopFilter(matchVersion, result, stopwords);
    if(!stemExclusionSet.isEmpty())
      result = new SetKeywordMarkerFilter(result, stemExclusionSet);
    result = new SnowballFilter(result, new IrishStemmer());
    return new TokenStreamComponents(source, result);
  }
View Full Code Here
121
122
123
124
125
126
127
128
129
    final Tokenizer source = new StandardTokenizer(matchVersion, reader);
    TokenStream result = new StandardFilter(matchVersion, source);
    result = new LowerCaseFilter(matchVersion, result);
    result = new StopFilter(matchVersion, result, stopwords);
    if(!stemExclusionSet.isEmpty())
      result = new SetKeywordMarkerFilter(result, stemExclusionSet);
    result = new LatvianStemFilter(result);
    return new TokenStreamComponents(source, result);
  }
View Full Code Here
78
79
80
81
82
83
84
85
    } catch (Exception e) {
      throw new RuntimeException("Error instantiating stemmer for language " + language + "from class " + stemClass, e);
    }

    if (protectedWords != null)
      input = new SetKeywordMarkerFilter(input, protectedWords);
    return new SnowballFilter(input, program);
  }
View Full Code Here
122
123
124
125
126
127
128
129
130
    final Tokenizer source = new StandardTokenizer(matchVersion, reader);
    TokenStream result = new StandardFilter(matchVersion, source);
    result = new LowerCaseFilter(matchVersion, result);
    result = new StopFilter(matchVersion, result, stopwords);
    if(!stemExclusionSet.isEmpty())
      result = new SetKeywordMarkerFilter(result, stemExclusionSet);
    result = new BulgarianStemFilter(result);
    return new TokenStreamComponents(source, result);
  }
View Full Code Here
123
124
125
126
127
128
129
130
131
    final Tokenizer source = new StandardTokenizer(matchVersion, reader);
    TokenStream result = new StandardFilter(matchVersion, source);
    result = new LowerCaseFilter(matchVersion, result);
    result = new StopFilter(matchVersion, result, stopwords);
    if(!stemExclusionSet.isEmpty())
      result = new SetKeywordMarkerFilter(result, stemExclusionSet);
    result = new SnowballFilter(result, new SwedishStemmer());
    return new TokenStreamComponents(source, result);
  }
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter

  • org.apache.lucene.analysis.ar.ArabicAnalyzer
  • org.apache.lucene.analysis.ar.TestArabicStemFilter
  • org.apache.lucene.analysis.bg.BulgarianAnalyzer
  • org.apache.lucene.analysis.bg.TestBulgarianStemmer
  • org.apache.lucene.analysis.br.BrazilianAnalyzer
  • org.apache.lucene.analysis.br.TestBrazilianStemmer
  • org.apache.lucene.analysis.ca.CatalanAnalyzer
  • org.apache.lucene.analysis.ckb.SoraniAnalyzer
  • org.apache.lucene.analysis.cz.CzechAnalyzer
  • org.apache.lucene.analysis.cz.TestCzechStemmer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.