Examples of StandardTokenizer

de.susebox.jtopas.StandardTokenizer

This is the mainstream {@link Tokenizer}. It implements the {@link Tokenizer}interface in a straightforward approach without too specialized parse optimizations.
Beside the {@link Tokenizer} interface, the class StandardTokenizerprovides some basic features for cascading (nested) tokenizers. Consider the usual HTML pages found today in the WWW. Most of them are a mixture of regular HTML, cascading style sheets (CSS) and embedded JavaScript. These different languages use different syntaxes, so one needs varous tokenizers on the same input stream.
This {@link Tokenizer} implementation is not synchronized. Take care when usingwith multible threads.
@see Tokenizer @see TokenizerProperties @author Heiko Blau
it.unibz.instasearch.indexing.tokenizers.standard.StandardTokenizer
A grammar-based tokenizer constructed with JavaCC.
This should be a good tokenizer for most European-language documents:
- Splits words at punctuation characters, removing punctuation. However, a dot that's not followed by whitespace is considered part of a token.
- Splits words at hyphens, unless there's a number in the token, in which case the whole token is interpreted as a product number and is not split.
- Recognizes email addresses and internet hostnames as one token.
Many applications have specific tokenizer needs. If this tokenizer does not suit your application, please consider copying this source code directory to your project and maintaining your own grammar-based tokenizer.
org.apache.lucene.analysis.standard.StandardTokenizer
s.apache.org/jira/browse/LUCENE-1068">LUCENE-1608

Examples of org.apache.lucene.analysis.standard.StandardTokenizer

    }
    
    SavedStreams streams = (SavedStreams) getPreviousTokenStream();
    if (streams == null) {
      streams = new SavedStreams();
      streams.source = new StandardTokenizer(matchVersion, reader);
      streams.result = new StandardFilter(streams.source);
      streams.result = new ThaiWordFilter(streams.result);
      streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                                      streams.result, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
      setPreviousTokenStream(streams);

Examples of org.apache.lucene.analysis.standard.StandardTokenizer

    }
    
    SavedStreams streams = (SavedStreams) getPreviousTokenStream();
    if (streams == null) {
      streams = new SavedStreams();
      streams.source = new StandardTokenizer(matchVersion, reader);
      streams.result = new StandardFilter(streams.source);
      streams.result = new LowerCaseFilter(streams.result);
      streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                                      streams.result, stopSet);
      streams.result = new GermanStemFilter(streams.result, exclusionSet);

Examples of org.apache.lucene.analysis.standard.StandardTokenizer

    public MultiAnalyzer() {
    }


    @Override
    public TokenStream tokenStream(String fieldName, Reader reader) {
      TokenStream result = new StandardTokenizer(Version.LUCENE_CURRENT, reader);
      result = new TestFilter(result);
      result = new LowerCaseFilter(result);
      return result;
    }

Examples of org.apache.lucene.analysis.standard.StandardTokenizer

    public PosIncrementAnalyzer() {
    }


    @Override
    public TokenStream tokenStream(String fieldName, Reader reader) {
      TokenStream result = new StandardTokenizer(Version.LUCENE_CURRENT, reader);
      result = new TestPosIncrementFilter(result);
      result = new LowerCaseFilter(result);
      return result;
    }

Examples of org.apache.lucene.analysis.standard.StandardTokenizer

   *         {@link StandardFilter}, {@link LowerCaseFilter}, {@link StopFilter}, and
   *         {@link GermanStemFilter}
   */
  @Override
  public TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream result = new StandardTokenizer(matchVersion, reader);
    result = new StandardFilter(result);
    result = new LowerCaseFilter(result);
    result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                            result, stopSet);
    result = new GermanStemFilter(result, exclusionSet);

Examples of org.apache.lucene.analysis.standard.StandardTokenizer

 */
public class TestElision extends BaseTokenStreamTestCase {


  public void testElision() throws Exception {
    String test = "Plop, juste pour voir l'embrouille avec O'brian. M'enfin.";
    Tokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(test));
    CharArraySet articles = new CharArraySet(TEST_VERSION_CURRENT, asSet("l", "M"), false);
    TokenFilter filter = new ElisionFilter(tokenizer, articles);
    List<String> tas = filter(filter);
    assertEquals("embrouille", tas.get(4));
    assertEquals("O'brian", tas.get(6));

Examples of org.apache.lucene.analysis.standard.StandardTokenizer

      @Override
      protected TokenStreamComponents createComponents(String field, Reader reader) {
        final CharArraySet keywords = new CharArraySet(version, 1, false);
        keywords.add("liście");


        final Tokenizer src = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
        TokenStream result = new StandardFilter(TEST_VERSION_CURRENT, src);
        result = new SetKeywordMarkerFilter(result, keywords);
        result = new MorfologikFilter(result, TEST_VERSION_CURRENT); 


        return new TokenStreamComponents(src, result);

Examples of org.apache.lucene.analysis.standard.StandardTokenizer

  private DoubleMetaphone filter = new DoubleMetaphone();
  
  @Override
  public TokenStream tokenStream(String fieldName, Reader reader) {
    final TokenStream result = new PorterStemFilter(new StopFilter(
        true, new StandardTokenizer(Version.LUCENE_CURRENT, reader),
        StandardAnalyzer.STOP_WORDS_SET));
    
    TermAttribute termAtt = (TermAttribute) result
        .addAttribute(TermAttribute.class);
    StringBuilder buf = new StringBuilder();

Examples of org.apache.lucene.analysis.standard.StandardTokenizer

public class MyAnalyzer extends Analyzer {
  
  @SuppressWarnings("deprecation")
  @Override
  public TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream result = new StandardTokenizer(
        Version.LUCENE_CURRENT, reader);
    result = new LowerCaseFilter(result);
    result = new LengthFilter(result, 3, 50);
    result = new StopFilter(true, result, StandardAnalyzer.STOP_WORDS_SET);
    result = new PorterStemFilter(result);

Examples of org.apache.lucene.analysis.standard.StandardTokenizer

  
  private final Pattern alphabets = Pattern.compile("[a-z]+");
  
  @Override
  public TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream result = new StandardTokenizer(Version.LUCENE_CURRENT, reader);
    result = new StandardFilter(result);
    result = new LowerCaseFilter(result);
    result = new StopFilter(true, result, StandardAnalyzer.STOP_WORDS_SET);
    
    TermAttribute termAtt = (TermAttribute) result.addAttribute(TermAttribute.class);
    StringBuilder buf = new StringBuilder();
    try {
      while (result.incrementToken()) {
        if (termAtt.termLength() < 3) continue;
        String word = new String(termAtt.termBuffer(), 0, termAtt.termLength());
        Matcher m = alphabets.matcher(word);
        
        if (m.matches()) {