Examples of SetKeywordMarkerFilter

org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter
Marks terms as keywords via the {@link KeywordAttribute}. Each token contained in the provided set is marked as a keyword by setting {@link KeywordAttribute#setKeyword(boolean)} to true.

Examples of org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter

    final Tokenizer source = new StandardTokenizer(matchVersion, reader);
    TokenStream result = new StandardFilter(matchVersion, source);
    result = new LowerCaseFilter(matchVersion, result);
    result = new StopFilter(matchVersion, result, stopwords);
    if(!stemExclusionSet.isEmpty())
      result = new SetKeywordMarkerFilter(result, stemExclusionSet);
    result = new SnowballFilter(result, new RomanianStemmer());
    return new TokenStreamComponents(source, result);
  }

View Full Code Here

Examples of org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter

    final Tokenizer source = new StandardTokenizer(matchVersion, reader);
    TokenStream result = new StandardFilter(matchVersion, source);
    result = new LowerCaseFilter(matchVersion, result);
    result = new StopFilter(matchVersion, result, stopwords);
    if(!stemExclusionSet.isEmpty())
      result = new SetKeywordMarkerFilter(result, stemExclusionSet);
    if (matchVersion.onOrAfter(Version.LUCENE_3_6)) {
      result = new PortugueseLightStemFilter(result);
    } else {
      result = new SnowballFilter(result, new PortugueseStemmer());
    }

View Full Code Here

Examples of org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter

      Reader reader) {
    final Tokenizer source = new StandardTokenizer(matchVersion, reader);
    TokenStream result = new StandardFilter(matchVersion, source);
    result = new LowerCaseFilter(matchVersion, result);
    result = new StopFilter( matchVersion, result, stopwords);
    result = new SetKeywordMarkerFilter(result, exclusionSet);
    if (matchVersion.onOrAfter(Version.LUCENE_3_6)) {
      result = new GermanNormalizationFilter(result);
      result = new GermanLightStemFilter(result);
    } else if (matchVersion.onOrAfter(Version.LUCENE_3_1)) {
      result = new SnowballFilter(result, new German2Stemmer());

View Full Code Here

Examples of org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter

      if (matchVersion.onOrAfter(Version.LUCENE_3_1)) {
        final Tokenizer source = new StandardTokenizer(matchVersion, reader);
        TokenStream result = new StandardFilter(matchVersion, source);
        result = new LowerCaseFilter(matchVersion, result);
        result = new StopFilter(matchVersion, result, stopwords);
        if (!stemExclusionSet.isEmpty()) result = new SetKeywordMarkerFilter(
            result, stemExclusionSet);
        result = new SnowballFilter(result, new org.tartarus.snowball.ext.RussianStemmer());
        return new TokenStreamComponents(source, result);
      } else {
        final Tokenizer source = new RussianLetterTokenizer(matchVersion, reader);
        TokenStream result = new LowerCaseFilter(matchVersion, source);
        result = new StopFilter(matchVersion, result, stopwords);
        if (!stemExclusionSet.isEmpty()) result = new SetKeywordMarkerFilter(
          result, stemExclusionSet);
        result = new SnowballFilter(result, new org.tartarus.snowball.ext.RussianStemmer());
        return new TokenStreamComponents(source, result);
      }
    }

View Full Code Here

Examples of org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter

    final Tokenizer source = new StandardTokenizer(matchVersion, reader);
    TokenStream result = new StandardFilter(matchVersion, source);
    result = new LowerCaseFilter(matchVersion, result);
    result = new StopFilter(matchVersion, result, stopwords);
    if(!stemExclusionSet.isEmpty())
      result = new SetKeywordMarkerFilter(result, stemExclusionSet);
    result = new SnowballFilter(result, new ArmenianStemmer());
    return new TokenStreamComponents(source, result);
  }

View Full Code Here

Examples of org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter

        final CharArraySet keywords = new CharArraySet(version, 1, false);
        keywords.add("liście");


        final Tokenizer src = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
        TokenStream result = new StandardFilter(TEST_VERSION_CURRENT, src);
        result = new SetKeywordMarkerFilter(result, keywords);
        result = new MorfologikFilter(result, TEST_VERSION_CURRENT); 


        return new TokenStreamComponents(src, result);
      }
    };

View Full Code Here

Examples of org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter

    final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("コーヒー"), false);
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
        TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
        return new TokenStreamComponents(source, new JapaneseKatakanaStemFilter(sink));
      }
    };
    checkOneTerm(a, "コーヒー", "コーヒー");
  }

View Full Code Here

Examples of org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter

    CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, 1, true);
    set.add("строеве");
    MockTokenizer tokenStream = new MockTokenizer(new StringReader("строевете строеве"), MockTokenizer.WHITESPACE, false);


    BulgarianStemFilter filter = new BulgarianStemFilter(
        new SetKeywordMarkerFilter(tokenStream, set));
    assertTokenStreamContents(filter, new String[] { "строй", "строеве" });
  }

View Full Code Here

Examples of org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter

    final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("sängerinnen"), false);
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
        TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
        return new TokenStreamComponents(source, new GermanLightStemFilter(sink));
      }
    };
    checkOneTerm(a, "sängerinnen", "sängerinnen");
  }

View Full Code Here

Examples of org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter

  }
  
  public void testWithKeywordAttribute() throws IOException {
    CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, 1, true);
    set.add("hole");
    CzechStemFilter filter = new CzechStemFilter(new SetKeywordMarkerFilter(
        new MockTokenizer(new StringReader("hole desek"), MockTokenizer.WHITESPACE, false), set));
    assertTokenStreamContents(filter, new String[] { "hole", "desk" });
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.