Examples of TeeTokenFilter

Examples of org.apache.lucene.analysis.TeeTokenFilter

39
40
41
42
43
44
45
46
47
48
49
50
51
52
  }

  public void test() throws IOException {
    TokenRangeSinkTokenizer rangeToks = new TokenRangeSinkTokenizer(2, 4);
    String test = "The quick red fox jumped over the lazy brown dogs";
    TeeTokenFilter tee = new TeeTokenFilter(new WhitespaceTokenizer(new StringReader(test)), rangeToks);
    Token tok = null;
    int count = 0;
    while ((tok = tee.next()) != null){
      assertTrue("tok is null and it shouldn't be", tok != null);
      count++;
    }
    assertTrue(count + " does not equal: " + 10, count == 10);
    assertTrue("rangeToks Size: " + rangeToks.getTokens().size() + " is not: " + 2, rangeToks.getTokens().size() == 2);
View Full Code Here

Examples of org.apache.lucene.analysis.TeeTokenFilter

40
41
42
43
44
45
46
47
48
49
50
51
52
53
  }

  public void test() throws IOException {
    DateRecognizerSinkTokenizer sink = new DateRecognizerSinkTokenizer(new SimpleDateFormat("MM/dd/yyyy"));
    String test = "The quick red fox jumped over the lazy brown dogs on 7/11/2006  The dogs finally reacted on 7/12/2006";
    TeeTokenFilter tee = new TeeTokenFilter(new WhitespaceTokenizer(new StringReader(test)), sink);
    Token tok = null;
    int count = 0;
    while ((tok = tee.next()) != null){
      assertTrue("tok is null and it shouldn't be", tok != null);
      if (tok.termBuffer()[0] == '7'){
        assertTrue(tok.type() + " is not equal to " + DateRecognizerSinkTokenizer.DATE_TYPE,
                tok.type().equals(DateRecognizerSinkTokenizer.DATE_TYPE) == true);
      }
View Full Code Here

Examples of org.apache.lucene.analysis.TeeTokenFilter

39
40
41
42
43
44
45
46
47
48
49
50
51
52
  }

  public void test() throws IOException {
    TokenRangeSinkTokenizer rangeToks = new TokenRangeSinkTokenizer(2, 4);
    String test = "The quick red fox jumped over the lazy brown dogs";
    TeeTokenFilter tee = new TeeTokenFilter(new WhitespaceTokenizer(new StringReader(test)), rangeToks);
    int count = 0;
    final Token reusableToken = new Token();
    for (Token nextToken = tee.next(reusableToken); nextToken != null; nextToken = tee.next(reusableToken)) {
      assertTrue("nextToken is null and it shouldn't be", nextToken != null);
      count++;
    }
    assertTrue(count + " does not equal: " + 10, count == 10);
    assertTrue("rangeToks Size: " + rangeToks.getTokens().size() + " is not: " + 2, rangeToks.getTokens().size() == 2);
View Full Code Here

Examples of org.apache.lucene.analysis.TeeTokenFilter

40
41
42
43
44
45
46
47
48
49
50
51
52
53
  }

  public void test() throws IOException {
    DateRecognizerSinkTokenizer sink = new DateRecognizerSinkTokenizer(new SimpleDateFormat("MM/dd/yyyy"));
    String test = "The quick red fox jumped over the lazy brown dogs on 7/11/2006  The dogs finally reacted on 7/12/2006";
    TeeTokenFilter tee = new TeeTokenFilter(new WhitespaceTokenizer(new StringReader(test)), sink);
    int count = 0;
    final Token reusableToken = new Token();
    for (Token nextToken = tee.next(reusableToken); nextToken != null; nextToken = tee.next(reusableToken)) {
      assertTrue("nextToken is null and it shouldn't be", nextToken != null);
      if (nextToken.termBuffer()[0] == '7'){
        assertTrue(nextToken.type() + " is not equal to " + DateRecognizerSinkTokenizer.DATE_TYPE,
                nextToken.type().equals(DateRecognizerSinkTokenizer.DATE_TYPE) == true);
      }
View Full Code Here

Examples of org.apache.lucene.analysis.TeeTokenFilter

43
44
45
46
47
48
49
50
51
52
53
54
55
56
  public void test() throws IOException {
    TokenTypeSinkTokenizer sink = new TokenTypeSinkTokenizer("D");
    String test = "The quick red fox jumped over the lazy brown dogs";

    TeeTokenFilter ttf = new TeeTokenFilter(new WordTokenFilter(new WhitespaceTokenizer(new StringReader(test))), sink);
    boolean seenDogs = false;
    final Token reusableToken = new Token();
    for (Token nextToken = ttf.next(reusableToken); nextToken != null; nextToken = ttf.next(reusableToken)) {
      if (nextToken.term().equals("dogs")) {
        seenDogs = true;
        assertTrue(nextToken.type() + " is not equal to " + "D", nextToken.type().equals("D") == true);
      } else {
        assertTrue(nextToken.type() + " is not null and it should be", nextToken.type().equals("word"));
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.