Examples of MockTokenizer

org.apache.lucene.analysis.MockTokenizer
Tokenizer for testing.
This tokenizer is a replacement for {@link #WHITESPACE}, {@link #SIMPLE}, and {@link #KEYWORD}tokenizers. If you are writing a component such as a TokenFilter, its a great idea to test it wrapping this tokenizer instead for extra checks. This tokenizer has the following behavior:
- An internal state-machine is used for checking consumer consistency. These checks can be disabled with {@link #setEnableChecks(boolean)}.
- For convenience, optionally lowercases terms that it outputs.

Examples of org.apache.lucene.analysis.MockTokenizer

    assertTokenStreamContents(ts, new String[] { "a", "e" });
  }
  
  public void testMultiWordSynonyms() throws Exception {
    Reader reader = new StringReader("a e");
    TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
    stream = tokenFilterFactory("Synonym", TEST_VERSION_CURRENT,
        new StringMockResourceLoader("a b c,d"),
        "synonyms", "synonyms.txt").create(stream);
    // This fails because ["e","e"] is the value of the token stream
    assertTokenStreamContents(stream, new String[] { "a", "e" });

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer

    assertEquals( 'x', buf[0]) ;
  }


  public void testNothingChange() throws Exception {
    CharFilter cs = new MappingCharFilter( normMap, new StringReader( "x" ) );
    TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
    assertTokenStreamContents(ts, new String[]{"x"}, new int[]{0}, new int[]{1}, 1);
  }

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer

    assertTokenStreamContents(ts, new String[]{"x"}, new int[]{0}, new int[]{1}, 1);
  }


  public void test1to1() throws Exception {
    CharFilter cs = new MappingCharFilter( normMap, new StringReader( "h" ) );
    TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
    assertTokenStreamContents(ts, new String[]{"i"}, new int[]{0}, new int[]{1}, 1);
  }

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer

    assertTokenStreamContents(ts, new String[]{"i"}, new int[]{0}, new int[]{1}, 1);
  }


  public void test1to2() throws Exception {
    CharFilter cs = new MappingCharFilter( normMap, new StringReader( "j" ) );
    TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
    assertTokenStreamContents(ts, new String[]{"jj"}, new int[]{0}, new int[]{1}, 1);
  }

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer

    assertTokenStreamContents(ts, new String[]{"jj"}, new int[]{0}, new int[]{1}, 1);
  }


  public void test1to3() throws Exception {
    CharFilter cs = new MappingCharFilter( normMap, new StringReader( "k" ) );
    TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
    assertTokenStreamContents(ts, new String[]{"kkk"}, new int[]{0}, new int[]{1}, 1);
  }

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer

    assertTokenStreamContents(ts, new String[]{"kkk"}, new int[]{0}, new int[]{1}, 1);
  }


  public void test2to4() throws Exception {
    CharFilter cs = new MappingCharFilter( normMap, new StringReader( "ll" ) );
    TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
    assertTokenStreamContents(ts, new String[]{"llll"}, new int[]{0}, new int[]{2}, 2);
  }

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer

    assertTokenStreamContents(ts, new String[]{"llll"}, new int[]{0}, new int[]{2}, 2);
  }


  public void test2to1() throws Exception {
    CharFilter cs = new MappingCharFilter( normMap, new StringReader( "aa" ) );
    TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
    assertTokenStreamContents(ts, new String[]{"a"}, new int[]{0}, new int[]{2}, 2);
  }

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer

    assertTokenStreamContents(ts, new String[]{"a"}, new int[]{0}, new int[]{2}, 2);
  }


  public void test3to1() throws Exception {
    CharFilter cs = new MappingCharFilter( normMap, new StringReader( "bbb" ) );
    TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
    assertTokenStreamContents(ts, new String[]{"b"}, new int[]{0}, new int[]{3}, 3);
  }

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer

    assertTokenStreamContents(ts, new String[]{"b"}, new int[]{0}, new int[]{3}, 3);
  }


  public void test4to2() throws Exception {
    CharFilter cs = new MappingCharFilter( normMap, new StringReader( "cccc" ) );
    TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
    assertTokenStreamContents(ts, new String[]{"cc"}, new int[]{0}, new int[]{4}, 4);
  }

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer

    assertTokenStreamContents(ts, new String[]{"cc"}, new int[]{0}, new int[]{4}, 4);
  }


  public void test5to0() throws Exception {
    CharFilter cs = new MappingCharFilter( normMap, new StringReader( "empty" ) );
    TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
    assertTokenStreamContents(ts, new String[0], new int[]{}, new int[]{}, 5);
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.