Examples of StandardTokenizer

de.susebox.jtopas.StandardTokenizer

This is the mainstream {@link Tokenizer}. It implements the {@link Tokenizer}interface in a straightforward approach without too specialized parse optimizations.
Beside the {@link Tokenizer} interface, the class StandardTokenizerprovides some basic features for cascading (nested) tokenizers. Consider the usual HTML pages found today in the WWW. Most of them are a mixture of regular HTML, cascading style sheets (CSS) and embedded JavaScript. These different languages use different syntaxes, so one needs varous tokenizers on the same input stream.
This {@link Tokenizer} implementation is not synchronized. Take care when usingwith multible threads.
@see Tokenizer @see TokenizerProperties @author Heiko Blau
it.unibz.instasearch.indexing.tokenizers.standard.StandardTokenizer
A grammar-based tokenizer constructed with JavaCC.
This should be a good tokenizer for most European-language documents:
- Splits words at punctuation characters, removing punctuation. However, a dot that's not followed by whitespace is considered part of a token.
- Splits words at hyphens, unless there's a number in the token, in which case the whole token is interpreted as a product number and is not split.
- Recognizes email addresses and internet hostnames as one token.
Many applications have specific tokenizer needs. If this tokenizer does not suit your application, please consider copying this source code directory to your project and maintaining your own grammar-based tokenizer.
org.apache.lucene.analysis.standard.StandardTokenizer
s.apache.org/jira/browse/LUCENE-1068">LUCENE-1608

Examples of de.susebox.jtopas.StandardTokenizer

   */
  private long tokenizeFile(String message, File file, boolean useReaderSource) throws Throwable {
    System.out.println(message);


    TokenizerSource   source    = null;
    StandardTokenizer tokenizer = new StandardTokenizer(_properties);
    
    try {
      if (useReaderSource) {
        source = new ReaderSource(file);
      } else {
        source = new CharArraySource(readFile(file));
      }
      tokenizer.setSource(source);
      return tokenize(tokenizer);
    } finally {
      if (useReaderSource && source != null) {
        ((ReaderSource)source).close();
      }
      tokenizer.close();
    }
  }

Examples of it.unibz.instasearch.indexing.tokenizers.standard.StandardTokenizer

  
  @Override
  public TokenStream tokenStream(String fieldName, Reader reader)
  {
    if( Field.CONTENTS.toString().equals(fieldName) ) {
      TokenStream result = new StandardTokenizer(reader); // splits at ". ", "-"
      
      result = new WordSplitTokenizer(result);   // non-alphanumerics
      result = new DotSplitTokenizer(result);   // com.package.names
      result = new CamelCaseTokenizer(result);   // CamelCaseIdentifiers

Examples of org.apache.lucene.analysis.standard.StandardTokenizer

 public StandardBgramAnalyzer() { }
 
 public TokenStream tokenStream (String fieldName, Reader reader)
 {
   TokenStream ts = (extractEntities) ? 
     new EntFilter ( new BgramFilter( new LowerCaseFilter( new StandardFilter( new StandardTokenizer(reader) ) ) ) ):
                     new BgramFilter( new LowerCaseFilter( new StandardFilter( new StandardTokenizer(reader) ) ) );
   return (ts);
 }

Examples of org.apache.lucene.analysis.standard.StandardTokenizer

   *         filtered with {@link StandardFilter}, {@link StopFilter}, 
   *         {@link FrenchStemFilter} and {@link LowerCaseFilter}
   */
  @Override
  public final TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream result = new StandardTokenizer(matchVersion, reader);
    result = new StandardFilter(result);
    result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                            result, stoptable);
    result = new FrenchStemFilter(result, excltable);
    // Convert to lowercase after stemming!

Examples of org.apache.lucene.analysis.standard.StandardTokenizer

  public TokenStream reusableTokenStream(String fieldName, Reader reader)
      throws IOException {
    SavedStreams streams = (SavedStreams) getPreviousTokenStream();
    if (streams == null) {
      streams = new SavedStreams();
      streams.source = new StandardTokenizer(matchVersion, reader);
      streams.result = new StandardFilter(streams.source);
      streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                                      streams.result, stoptable);
      streams.result = new FrenchStemFilter(streams.result, excltable);
      // Convert to lowercase after stemming!

Examples of org.apache.lucene.analysis.standard.StandardTokenizer

   *   filtered with {@link StandardFilter}, {@link StopFilter}, 
   *   and {@link DutchStemFilter}
   */
  @Override
  public TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream result = new StandardTokenizer(matchVersion, reader);
    result = new StandardFilter(result);
    result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                            result, stoptable);
    result = new DutchStemFilter(result, excltable, stemdict);
    return result;

Examples of org.apache.lucene.analysis.standard.StandardTokenizer

    }
    
    SavedStreams streams = (SavedStreams) getPreviousTokenStream();
    if (streams == null) {
      streams = new SavedStreams();
      streams.source = new StandardTokenizer(matchVersion, reader);
      streams.result = new StandardFilter(streams.source);
      streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                                      streams.result, stoptable);
      streams.result = new DutchStemFilter(streams.result, excltable, stemdict);
      setPreviousTokenStream(streams);

Examples of org.apache.lucene.analysis.standard.StandardTokenizer

   *       {@link LowerCaseFilter}, {@link StandardFilter}, {@link StopFilter}, and 
   *          {@link BrazilianStemFilter}.
   */
  @Override
  public final TokenStream tokenStream(String fieldName, Reader reader) {
                TokenStream result = new StandardTokenizer( matchVersion, reader );
    result = new LowerCaseFilter( result );
    result = new StandardFilter( result );
    result = new StopFilter( StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                                         result, stoptable );
    result = new BrazilianStemFilter( result, excltable );

Examples of org.apache.lucene.analysis.standard.StandardTokenizer

    public TokenStream reusableTokenStream(String fieldName, Reader reader)
      throws IOException {
      SavedStreams streams = (SavedStreams) getPreviousTokenStream();
      if (streams == null) {
        streams = new SavedStreams();
        streams.source = new StandardTokenizer(matchVersion, reader);
        streams.result = new LowerCaseFilter(streams.source);
        streams.result = new StandardFilter(streams.result);
        streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
                                        streams.result, stoptable);
        streams.result = new BrazilianStemFilter(streams.result, excltable);

Examples of org.apache.lucene.analysis.standard.StandardTokenizer

    public MultiAnalyzer() {
    }


    @Override
    public TokenStream tokenStream(String fieldName, Reader reader) {
      TokenStream result = new StandardTokenizer(Version.LUCENE_CURRENT, reader);
      result = new TestFilter(result);
      result = new LowerCaseFilter(result);
      return result;
    }