Examples of MockTokenizer

org.apache.lucene.analysis.MockTokenizer
Tokenizer for testing.
This tokenizer is a replacement for {@link #WHITESPACE}, {@link #SIMPLE}, and {@link #KEYWORD}tokenizers. If you are writing a component such as a TokenFilter, its a great idea to test it wrapping this tokenizer instead for extra checks. This tokenizer has the following behavior:
- An internal state-machine is used for checking consumer consistency. These checks can be disabled with {@link #setEnableChecks(boolean)}.
- For convenience, optionally lowercases terms that it outputs.

Examples of org.apache.lucene.analysis.MockTokenizer

    assertTokenStreamContents(ts, new String[0], new int[]{}, new int[]{}, 5);
  }


  public void testNonBMPChar() throws Exception {
    CharFilter cs = new MappingCharFilter( normMap, new StringReader( UnicodeUtil.newString(new int[] {0x1D122}, 0, 1) ) );
    TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
    assertTokenStreamContents(ts, new String[]{"fclef"}, new int[]{0}, new int[]{2}, 2);
  }

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer

    assertTokenStreamContents(ts, new String[]{"fclef"}, new int[]{0}, new int[]{2}, 2);
  }


  public void testFullWidthChar() throws Exception {
    CharFilter cs = new MappingCharFilter( normMap, new StringReader( "\uff01") );
    TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
    assertTokenStreamContents(ts, new String[]{"full-width-exclamation"}, new int[]{0}, new int[]{1}, 1);
  }

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer

  //   aa,20,22 =>    a,20,22
  //
  public void testTokenStream() throws Exception {
    String testString = "h i j k ll cccc bbb aa";
    CharFilter cs = new MappingCharFilter( normMap, new StringReader( testString ) );
    TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
    assertTokenStreamContents(ts,
      new String[]{"i","i","jj","kkk","llll","cc","b","a"},
      new int[]{0,2,4,6,8,11,16,20},
      new int[]{1,3,5,7,10,15,19,22},
      testString.length()

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer

  //    h,8,9 => i,8,9
  public void testChained() throws Exception {
    String testString = "aaaa ll h";
    CharFilter cs = new MappingCharFilter( normMap,
        new MappingCharFilter( normMap, new StringReader( testString ) ) );
    TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
    assertTokenStreamContents(ts,
      new String[]{"a","llllllll","i"},
      new int[]{0,5,8},
      new int[]{4,7,9},
      testString.length()

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer

  public void testRandom() throws Exception {
    Analyzer analyzer = new Analyzer() {


      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
        return new TokenStreamComponents(tokenizer, tokenizer);
      }


      @Override
      protected Reader initReader(String fieldName, Reader reader) {

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer

    final NormalizeCharMap map = builder.build();


    Analyzer analyzer = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
        return new TokenStreamComponents(tokenizer, tokenizer);
      }


      @Override
      protected Reader initReader(String fieldName, Reader reader) {

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer

    for (int i = 0; i < numIterations; i++) {
      final NormalizeCharMap map = randomMap();
      Analyzer analyzer = new Analyzer() {
        @Override
        protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
          Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
          return new TokenStreamComponents(tokenizer, tokenizer);
        }


        @Override
        protected Reader initReader(String fieldName, Reader reader) {

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer

    StringReader reader = new StringReader("rowery bilety");
    Map<String,String> initParams = new HashMap<String,String>();
    initParams.put(MorfologikFilterFactory.DICTIONARY_SCHEMA_ATTRIBUTE, "morfologik");
    initParams.put("luceneMatchVersion", TEST_VERSION_CURRENT.toString());
    MorfologikFilterFactory factory = new MorfologikFilterFactory(initParams);
    TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
    stream = factory.create(stream);
    assertTokenStreamContents(stream, new String[] {"rower", "bilet"});
  }

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer

public class DelimitedPayloadTokenFilterTest extends LuceneTestCase {


  public void testPayloads() throws Exception {
    String test = "The quick|JJ red|JJ fox|NN jumped|VB over the lazy|JJ brown|JJ dogs|NN";
    DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter
      (new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false), 
       DelimitedPayloadTokenFilter.DEFAULT_DELIMITER, new IdentityEncoder());
    CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
    PayloadAttribute payAtt = filter.getAttribute(PayloadAttribute.class);
    filter.reset();
    assertTermEquals("The", filter, termAtt, payAtt, null);

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer


  public void testNext() throws Exception {


    String test = "The quick|JJ red|JJ fox|NN jumped|VB over the lazy|JJ brown|JJ dogs|NN";
    DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter
      (new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false), 
       DelimitedPayloadTokenFilter.DEFAULT_DELIMITER, new IdentityEncoder());
    filter.reset();
    assertTermEquals("The", filter, null);
    assertTermEquals("quick", filter, "JJ".getBytes("UTF-8"));
    assertTermEquals("red", filter, "JJ".getBytes("UTF-8"));

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.