Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Token.termBuffer()

55
56
57
58
59
60
61
62
63
64
65
    TokenStream stream = new IterTokenStream("Kuczewski");
    TokenStream filter = new DoubleMetaphoneFilter(stream, 4, false);

    Token token = filter.next(new Token());
    assertEquals(4, token.termLength());
    assertEquals("KSSK", new String(token.termBuffer(), 0, token.termLength()));

    token = filter.next(new Token());
    assertEquals(4, token.termLength());
    assertEquals("KXFS", new String(token.termBuffer(), 0, token.termLength()));
    assertNull(filter.next(new Token()));
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
    assertEquals(4, token.termLength());
    assertEquals("KSSK", new String(token.termBuffer(), 0, token.termLength()));

    token = filter.next(new Token());
    assertEquals(4, token.termLength());
    assertEquals("KXFS", new String(token.termBuffer(), 0, token.termLength()));
    assertNull(filter.next(new Token()));
  }

  public void testSize8FalseInject() throws Exception {
    TokenStream stream = new IterTokenStream("international");
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
    TokenStream stream = new IterTokenStream("international");
    TokenStream filter = new DoubleMetaphoneFilter(stream, 8, false);

    Token token = filter.next(new Token());
    assertEquals(8, token.termLength());
    assertEquals("ANTRNXNL", new String(token.termBuffer(), 0, token
        .termLength()));

    assertNull(filter.next(new Token()));
  }
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
        new String[] { "12345", "#$%@#^%&" });
    TokenStream filter = new DoubleMetaphoneFilter(stream, 8, true);

    Token token = filter.next(new Token());
    assertEquals(5, token.termLength());
    assertEquals("12345", new String(token.termBuffer(), 0, token.termLength()));

    token = filter.next(new Token());
    assertEquals(8, token.termLength());
    assertEquals("#$%@#^%&", new String(token.termBuffer(), 0, token
        .termLength()));
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
    assertEquals(5, token.termLength());
    assertEquals("12345", new String(token.termBuffer(), 0, token.termLength()));

    token = filter.next(new Token());
    assertEquals(8, token.termLength());
    assertEquals("#$%@#^%&", new String(token.termBuffer(), 0, token
        .termLength()));
  }

  public void testNonConvertableStringsWithoutInject() throws Exception {
    TokenStream stream = new IterTokenStream(
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
    assertEquals(DoubleMetaphoneFilter.class, filteredStream.getClass());

    Token token = filteredStream.next(new Token());
    assertEquals(13, token.termLength());
    assertEquals("international", new String(token.termBuffer(), 0, token
        .termLength()));

    token = filteredStream.next(new Token());
    assertEquals(4, token.termLength());
    assertEquals("ANTR", new String(token.termBuffer(), 0, token.termLength()));
View Full Code Here
41
42
43
44
45
46
47
48
49
50
51
    assertEquals("international", new String(token.termBuffer(), 0, token
        .termLength()));

    token = filteredStream.next(new Token());
    assertEquals(4, token.termLength());
    assertEquals("ANTR", new String(token.termBuffer(), 0, token.termLength()));

    assertNull(filteredStream.next(new Token()));
  }

  public void testSettingSizeAndInject() throws Exception {
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
    assertEquals(DoubleMetaphoneFilter.class, filteredStream.getClass());

    Token token = filteredStream.next(new Token());
    assertEquals(8, token.termLength());
    assertEquals("ANTRNXNL", new String(token.termBuffer(), 0, token
        .termLength()));

    assertNull(filteredStream.next(new Token()));
  }
}
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
      }

      // common case fast-path of first token not matching anything
      Token firstTok = nextTok(target);
      if (firstTok == null) return null;
      SynonymMap result = map.submap!=null ? map.submap.get(firstTok.termBuffer(), 0, firstTok.termLength()) : null;
      if (result == null) return firstTok;

      // OK, we matched a token, so find the longest match.

      matched = new LinkedList<Token>();
View Full Code Here
180
181
182
183
184
185
186
187
188
189
190
    if (map.submap != null) {
      Token tok = nextTok();
      if (tok != null) {
        // check for positionIncrement!=1?  if>1, should not match, if==0, check multiple at this level?
        SynonymMap subMap = map.submap.get(tok.termBuffer(), 0, tok.termLength());

        if (subMap != null) {
          // recurse
          result = match(subMap);
        }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.