Examples of StopFilter

org.apache.lucene.analysis.StopFilter
Removes stop words from a token stream.
org.apache.lucene.analysis.core.StopFilter
Removes stop words from a token stream.
You must specify the required {@link Version}compatibility when creating StopFilter:
- As of 3.1, StopFilter correctly handles Unicode 4.0 supplementary characters in stopwords and position increments are preserved

uk.ac.ucl.panda.utility.parser.StopFilter
Removes stop words from a token stream.

Examples of org.apache.lucene.analysis.StopFilter

      if (streams == null) {
        streams = new SavedStreams();
        streams.source = new StandardTokenizer(matchVersion, reader);
        streams.result = new StandardFilter(streams.source);
        streams.result = new LowerCaseFilter(streams.result);
        streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                                        streams.result, stoptable);
        setPreviousTokenStream(streams);
      } else {
        streams.source.reset(reader);
      }

View Full Code Here

Examples of org.apache.lucene.analysis.StopFilter

  @Override
  public TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream ts = new StandardTokenizer(matchVersion, reader);
    ts = new StandardFilter(ts);
    ts = new ThaiWordFilter(ts);
    ts = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                        ts, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
    return ts;
  }

View Full Code Here

Examples of org.apache.lucene.analysis.StopFilter

    if (streams == null) {
      streams = new SavedStreams();
      streams.source = new StandardTokenizer(matchVersion, reader);
      streams.result = new StandardFilter(streams.source);
      streams.result = new ThaiWordFilter(streams.result);
      streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                                      streams.result, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
      setPreviousTokenStream(streams);
    } else {
      streams.source.reset(reader);
      streams.result.reset(); // reset the ThaiWordFilter's state

View Full Code Here

Examples of org.apache.lucene.analysis.StopFilter

    } catch (IOException e) {
      result = delegate.tokenStream(fieldName, reader);
    }
    HashSet<String> stopWords = stopWordsPerField.get(fieldName);
    if (stopWords != null) {
      result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                              result, stopWords);
    }
    return result;
  }

View Full Code Here

Examples of org.apache.lucene.analysis.StopFilter

      streams.wrapped = delegate.reusableTokenStream(fieldName, reader);


      /* if there are any stopwords for the field, save the stopfilter */
      HashSet<String> stopWords = stopWordsPerField.get(fieldName);
      if (stopWords != null)
        streams.withStopFilter = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                                                streams.wrapped, stopWords);
      else
        streams.withStopFilter = streams.wrapped;


    } else {
      /*
       * an entry for this field exists, verify the wrapped stream has not
       * changed. if it has not, reuse it, otherwise wrap the new stream.
       */
      TokenStream result = delegate.reusableTokenStream(fieldName, reader);
      if (result == streams.wrapped) {
        /* the wrapped analyzer reused the stream */
        streams.withStopFilter.reset();
      } else {
        /*
         * the wrapped analyzer did not. if there are any stopwords for the
         * field, create a new StopFilter around the new stream
         */
        streams.wrapped = result;
        HashSet<String> stopWords = stopWordsPerField.get(fieldName);
        if (stopWords != null)
          streams.withStopFilter = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                                                  streams.wrapped, stopWords);
        else
          streams.withStopFilter = streams.wrapped;
      }
    }

View Full Code Here

Examples of org.apache.lucene.analysis.StopFilter

  @Override
  public TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream result = new StandardTokenizer(matchVersion, reader);
    result = new StandardFilter(result);
    result = new LowerCaseFilter(result);
    result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                            result, stopSet);
    result = new GermanStemFilter(result, exclusionSet);
    return result;
  }

View Full Code Here

Examples of org.apache.lucene.analysis.StopFilter

    if (streams == null) {
      streams = new SavedStreams();
      streams.source = new StandardTokenizer(matchVersion, reader);
      streams.result = new StandardFilter(streams.source);
      streams.result = new LowerCaseFilter(streams.result);
      streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                                      streams.result, stopSet);
      streams.result = new GermanStemFilter(streams.result, exclusionSet);
      setPreviousTokenStream(streams);
    } else {
      streams.source.reset(reader);

View Full Code Here

Examples of org.apache.lucene.analysis.StopFilter

public class TwitterAnalyzer extends Analyzer {
  private DoubleMetaphone filter = new DoubleMetaphone();
  
  @Override
  public TokenStream tokenStream(String fieldName, Reader reader) {
    final TokenStream result = new PorterStemFilter(new StopFilter(
        true, new StandardTokenizer(Version.LUCENE_CURRENT, reader),
        StandardAnalyzer.STOP_WORDS_SET));
    
    TermAttribute termAtt = (TermAttribute) result
        .addAttribute(TermAttribute.class);

View Full Code Here

Examples of org.apache.lucene.analysis.StopFilter

  public TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream result = new StandardTokenizer(
        Version.LUCENE_CURRENT, reader);
    result = new LowerCaseFilter(result);
    result = new LengthFilter(result, 3, 50);
    result = new StopFilter(true, result, StandardAnalyzer.STOP_WORDS_SET);
    result = new PorterStemFilter(result);
    return result;
  }

View Full Code Here

Examples of org.apache.lucene.analysis.StopFilter

  @Override
  public TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream result = new StandardTokenizer(Version.LUCENE_CURRENT, reader);
    result = new StandardFilter(result);
    result = new LowerCaseFilter(result);
    result = new StopFilter(true, result, StandardAnalyzer.STOP_WORDS_SET);
    
    TermAttribute termAtt = (TermAttribute) result.addAttribute(TermAttribute.class);
    StringBuilder buf = new StringBuilder();
    try {
      while (result.incrementToken()) {

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.