Package org.apache.lucene.analysis.cn.smart

Examples of org.apache.lucene.analysis.cn.smart.WordTokenFilter

106
107
108
109
110
111
112
113
114
115
116
    this.stopWords = stopWords;
  }

  public TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream result = new SentenceTokenizer(reader);
    result = new WordTokenFilter(result);
    // result = new LowerCaseFilter(result);
    // LowerCaseFilter is not needed, as SegTokenFilter lowercases Basic Latin text.
    // The porter stemming is too strict, this is not a bug, this is a feature:)
    result = new PorterStemFilter(result);
    if (stopWords != null) {
View Full Code Here
129
130
131
132
133
134
135
136
137
138
139
    SavedStreams streams = (SavedStreams) getPreviousTokenStream();
    if (streams == null) {
      streams = new SavedStreams();
      setPreviousTokenStream(streams);
      streams.tokenStream = new SentenceTokenizer(reader);
      streams.filteredTokenStream = new WordTokenFilter(streams.tokenStream);
      streams.filteredTokenStream = new PorterStemFilter(streams.filteredTokenStream);
      if (stopWords != null) {
        streams.filteredTokenStream = new StopFilter(streams.filteredTokenStream, stopWords, false);
      }
    } else {
View Full Code Here
43
44
45
46
47
48
49
    }
  }
 
  @Override
  public TokenFilter create(TokenStream input) {
      return new WordTokenFilter(input);
  }
View Full Code Here
136
137
138
139
140
141
142
143
144
145
146
  }

  @Override
  public TokenStreamComponents createComponents(String fieldName, Reader reader) {
    Tokenizer tokenizer = new SentenceTokenizer(reader);
    TokenStream result = new WordTokenFilter(tokenizer);
    // result = new LowerCaseFilter(result);
    // LowerCaseFilter is not needed, as SegTokenFilter lowercases Basic Latin text.
    // The porter stemming is too strict, this is not a bug, this is a feature:)
    result = new PorterStemFilter(result);
    if (!stopWords.isEmpty()) {
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
  }

  @Override
  public TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream result = new SentenceTokenizer(reader);
    result = new WordTokenFilter(result);
    // result = new LowerCaseFilter(result);
    // LowerCaseFilter is not needed, as SegTokenFilter lowercases Basic Latin text.
    // The porter stemming is too strict, this is not a bug, this is a feature:)
    result = new PorterStemFilter(result);
    if (!stopWords.isEmpty()) {
View Full Code Here
170
171
172
173
174
175
176
177
178
179
180
    SavedStreams streams = (SavedStreams) getPreviousTokenStream();
    if (streams == null) {
      streams = new SavedStreams();
      setPreviousTokenStream(streams);
      streams.tokenStream = new SentenceTokenizer(reader);
      streams.filteredTokenStream = new WordTokenFilter(streams.tokenStream);
      streams.filteredTokenStream = new PorterStemFilter(streams.filteredTokenStream);
      if (!stopWords.isEmpty()) {
        streams.filteredTokenStream = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                                                     streams.filteredTokenStream, stopWords, false);
      }
View Full Code Here
30
31
32
33
34
35
36
* <code>words="org/apache/lucene/analysis/cn/smart/stopwords.txt"</code>
* @lucene.experimental
*/
public class SmartChineseWordTokenFilterFactory extends BaseTokenFilterFactory {
  public TokenFilter create(TokenStream input) {
      return new WordTokenFilter(input);
  }
View Full Code Here
32
33
34
35
36
37
38
* @lucene.experimental
*/
public class SmartChineseWordTokenFilterFactory extends TokenFilterFactory {
  @Override
  public TokenFilter create(TokenStream input) {
      return new WordTokenFilter(input);
  }
View Full Code Here
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
                log.error(message,e);
                throw new EngineException(this, ci, message, e);
            }
        }
        //now the tokens
        TokenStream tokens = new WordTokenFilter(new AnalyzedTextSentenceTokenizer(at));
        try {
          tokens.reset();
            while(tokens.incrementToken()){
                OffsetAttribute offset = tokens.addAttribute(OffsetAttribute.class);
                Token t = at.addToken(offset.startOffset(), offset.endOffset());
                log.trace("detected {}",t);
            }
        } catch (IOException e) {
            String message = String.format("IOException while reading from "
View Full Code Here
151
152
153
154
155
156
157
158
159
160
161
    this.matchVersion = matchVersion;
  }

  public TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream result = new SentenceTokenizer(reader);
    result = new WordTokenFilter(result);
    // result = new LowerCaseFilter(result);
    // LowerCaseFilter is not needed, as SegTokenFilter lowercases Basic Latin text.
    // The porter stemming is too strict, this is not a bug, this is a feature:)
    result = new PorterStemFilter(result);
    if (stopWords != null) {
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.cn.smart.WordTokenFilter

  • org.apache.lucene.analysis.cn.smart.hhmm.SegToken
  • org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer
  • org.apache.lucene.analysis.cn.smart.SmartChineseWordTokenFilterFactory
  • org.apache.solr.analysis.SmartChineseWordTokenFilterFactory
  • org.apache.stanbol.enhancer.engines.smartcn.impl.SmartcnTokenizerEngine
  • org.carrot2.text.linguistic.lucene.ChineseTokenizerAdapter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.