Examples of MockTokenizer

org.apache.lucene.analysis.MockTokenizer
Tokenizer for testing.
This tokenizer is a replacement for {@link #WHITESPACE}, {@link #SIMPLE}, and {@link #KEYWORD}tokenizers. If you are writing a component such as a TokenFilter, its a great idea to test it wrapping this tokenizer instead for extra checks. This tokenizer has the following behavior:
- An internal state-machine is used for checking consumer consistency. These checks can be disabled with {@link #setEnableChecks(boolean)}.
- For convenience, optionally lowercases terms that it outputs.

Examples of org.apache.lucene.analysis.MockTokenizer

    Directory dir = newDirectory();
    // note this test explicitly disables payloads
    final Analyzer analyzer = new Analyzer() {
      @Override
      public TokenStream tokenStream(String fieldName, Reader reader) {
        return new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
      }
    };
    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).setRAMBufferSizeMB(1.0).setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH).setMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH));
    w.setInfoStream(VERBOSE ? System.out : null);
    Document doc = new Document();

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer

    final int codeLen = _TestUtil.nextInt(random(), 1, 8);
    Analyzer a = new Analyzer() {


      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
        return new TokenStreamComponents(tokenizer, new DoubleMetaphoneFilter(tokenizer, codeLen, false));
      }
      
    };
    checkRandomData(random(), a, 1000 * RANDOM_MULTIPLIER);
    
    Analyzer b = new Analyzer() {


      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
        return new TokenStreamComponents(tokenizer, new DoubleMetaphoneFilter(tokenizer, codeLen, true));
      }
      
    };
    checkRandomData(random(), b, 1000 * RANDOM_MULTIPLIER);

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer

    }
  }
  
  static void assertAlgorithm(String algName, String inject, String input,
      String[] expected) throws Exception {
    Tokenizer tokenizer = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
    Map<String,String> args = new HashMap<String,String>();
    args.put("encoder", algName);
    args.put("inject", inject);
    PhoneticFilterFactory factory = new PhoneticFilterFactory(args);
    factory.inform(new ClasspathResourceLoader(factory.getClass()));

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer


public class TestDoubleMetaphoneFilterFactory extends BaseTokenStreamTestCase {


  public void testDefaults() throws Exception {
    DoubleMetaphoneFilterFactory factory = new DoubleMetaphoneFilterFactory(new HashMap<String, String>());
    TokenStream inputStream = new MockTokenizer(new StringReader("international"), MockTokenizer.WHITESPACE, false);


    TokenStream filteredStream = factory.create(inputStream);
    assertEquals(DoubleMetaphoneFilter.class, filteredStream.getClass());
    assertTokenStreamContents(filteredStream, new String[] { "international", "ANTR" });
  }

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer

    Map<String,String> parameters = new HashMap<String,String>();
    parameters.put("inject", "false");
    parameters.put("maxCodeLength", "8");
    DoubleMetaphoneFilterFactory factory = new DoubleMetaphoneFilterFactory(parameters);


    TokenStream inputStream = new MockTokenizer(new StringReader("international"), MockTokenizer.WHITESPACE, false);


    TokenStream filteredStream = factory.create(inputStream);
    assertEquals(DoubleMetaphoneFilter.class, filteredStream.getClass());
    assertTokenStreamContents(filteredStream, new String[] { "ANTRNXNL" });
  }

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer


/** Simple tests for {@link BeiderMorseFilterFactory} */
public class TestBeiderMorseFilterFactory extends BaseTokenStreamTestCase {
  public void testBasics() throws Exception {
    BeiderMorseFilterFactory factory = new BeiderMorseFilterFactory(new HashMap<String,String>());
    TokenStream ts = factory.create(new MockTokenizer(new StringReader("Weinberg"), MockTokenizer.WHITESPACE, false));
    assertTokenStreamContents(ts,
        new String[] { "vDnbirk", "vanbirk", "vinbirk", "wDnbirk", "wanbirk", "winbirk" },
        new int[] { 0, 0, 0, 0, 0, 0 },
        new int[] { 8, 8, 8, 8, 8, 8 },
        new int[] { 1, 0, 0, 0, 0, 0 });

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer

  
  public void testLanguageSet() throws Exception {
    Map<String,String> args = new HashMap<String,String>();
    args.put("languageSet", "polish");
    BeiderMorseFilterFactory factory = new BeiderMorseFilterFactory(args);
    TokenStream ts = factory.create(new MockTokenizer(new StringReader("Weinberg"), MockTokenizer.WHITESPACE, false));
    assertTokenStreamContents(ts,
        new String[] { "vDmbYrk", "vDmbirk", "vambYrk", "vambirk", "vimbYrk", "vimbirk" },
        new int[] { 0, 0, 0, 0, 0, 0 },
        new int[] { 8, 8, 8, 8, 8, 8 },
        new int[] { 1, 0, 0, 0, 0, 0 });

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer

  public void testOptions() throws Exception {
    Map<String,String> args = new HashMap<String,String>();
    args.put("nameType", "ASHKENAZI");
    args.put("ruleType", "EXACT");
    BeiderMorseFilterFactory factory = new BeiderMorseFilterFactory(args);
    TokenStream ts = factory.create(new MockTokenizer(new StringReader("Weinberg"), MockTokenizer.WHITESPACE, false));
    assertTokenStreamContents(ts,
        new String[] { "vajnberk" },
        new int[] { 0 },
        new int[] { 8 },
        new int[] { 1 });

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer


  static private Analyzer newTestAnalyzer() {
    return new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
        return new TokenStreamComponents(tokenizer, tokenizer);
      }


      @Override
      protected Reader initReader(String fieldName, Reader reader) {

View Full Code Here

Examples of org.apache.lucene.analysis.MockTokenizer

    List<String> rules = new ArrayList<String>();
    rules.add("a b c,d");
    SlowSynonymMap synMap = new SlowSynonymMap(true);
    SlowSynonymFilterFactory.parseRules(rules, synMap, "=>", ",", true, null);


    SlowSynonymFilter ts = new SlowSynonymFilter(new MockTokenizer(new StringReader("a e"), MockTokenizer.WHITESPACE, false), synMap);
    // This fails because ["e","e"] is the value of the token stream
    assertTokenStreamContents(ts, new String[] { "a", "e" });
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.