Package org.apache.lucene.analysis.ngram

Examples of org.apache.lucene.analysis.ngram.NGramTokenFilter


    minGramSize = (minArg != null ? Integer.parseInt(minArg)
        : NGramTokenFilter.DEFAULT_MIN_NGRAM_SIZE);
  }

  public NGramTokenFilter create(TokenStream input) {
    return new NGramTokenFilter(input, minGramSize, maxGramSize);
  }
View Full Code Here


      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, reader, 2, 94);
        //TokenStream stream = new SopTokenFilter(tokenizer);
        TokenStream stream = new ShingleFilter(tokenizer, 5);
        //stream = new SopTokenFilter(stream);
        stream = new NGramTokenFilter(TEST_VERSION_CURRENT, stream, 55, 83);
        //stream = new SopTokenFilter(stream);
        return new TokenStreamComponents(tokenizer, stream);
     
    };
    checkRandomData(random(), analyzer, 2000);
View Full Code Here

      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, reader, 2, 94);
        //TokenStream stream = new SopTokenFilter(tokenizer);
        TokenStream stream = new ShingleFilter(tokenizer, 5);
        //stream = new SopTokenFilter(stream);
        stream = new NGramTokenFilter(TEST_VERSION_CURRENT, stream, 55, 83);
        //stream = new SopTokenFilter(stream);
        return new TokenStreamComponents(tokenizer, stream);
     
    };
    checkRandomData(random(), analyzer, 2000);
View Full Code Here

    minGramSize = (minArg != null ? Integer.parseInt(minArg)
        : NGramTokenFilter.DEFAULT_MIN_NGRAM_SIZE);
  }

  public NGramTokenFilter create(TokenStream input) {
    return new NGramTokenFilter(input, minGramSize, maxGramSize);
  }
View Full Code Here

            @Override public String name() {
                return "nGram";
            }

            @Override public TokenStream create(TokenStream tokenStream) {
                return new NGramTokenFilter(tokenStream);
            }
        }));

        tokenFilterFactories.put("ngram", new PreBuiltTokenFilterFactoryFactory(new TokenFilterFactory() {
            @Override public String name() {
                return "ngram";
            }

            @Override public TokenStream create(TokenStream tokenStream) {
                return new NGramTokenFilter(tokenStream);
            }
        }));

        tokenFilterFactories.put("edgeNGram", new PreBuiltTokenFilterFactoryFactory(new TokenFilterFactory() {
            @Override public String name() {
View Full Code Here

        this.minGram = settings.getAsInt("min_gram", NGramTokenFilter.DEFAULT_MIN_NGRAM_SIZE);
        this.maxGram = settings.getAsInt("max_gram", NGramTokenFilter.DEFAULT_MAX_NGRAM_SIZE);
    }

    @Override public TokenStream create(TokenStream tokenStream) {
        return new NGramTokenFilter(tokenStream, minGram, maxGram);
    }
View Full Code Here

   /*
    * (non-Javadoc)
    * @see org.apache.lucene.analysis.Analyzer#tokenStream(java.lang.String, java.io.Reader)
    */
   public TokenStream tokenStream(String fieldName, Reader reader) {
       return new NGramTokenFilter(new UpperCaseFilter(new SentenceTokenizer(reader)), minTokenLength, maxTokenLength);
   }
View Full Code Here

        : NGramTokenFilter.DEFAULT_MIN_NGRAM_SIZE);
  }

  @Override
  public NGramTokenFilter create(TokenStream input) {
    return new NGramTokenFilter(input, minGramSize, maxGramSize);
  }
View Full Code Here

      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, reader, 2, 94);
        //TokenStream stream = new SopTokenFilter(tokenizer);
        TokenStream stream = new ShingleFilter(tokenizer, 5);
        //stream = new SopTokenFilter(stream);
        stream = new NGramTokenFilter(TEST_VERSION_CURRENT, stream, 55, 83);
        //stream = new SopTokenFilter(stream);
        return new TokenStreamComponents(tokenizer, stream);
     
    };
    checkRandomData(random(), analyzer, 2000);
View Full Code Here

    loader.assertAwareCompatibility( clazz, new NGramFilterFactory() );
    loader.assertAwareCompatibility( clazz, new KeywordTokenizerFactory() );
   
    // Make sure it throws an error for invalid objects
    Object[] invalid = new Object[] {
        new NGramTokenFilter( null ),
        "hello"new Float( 12.3f ),
        new LukeRequestHandler(),
        new JSONResponseWriter()
    };
    for( Object obj : invalid ) {
View Full Code Here

TOP

Related Classes of org.apache.lucene.analysis.ngram.NGramTokenFilter

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.