Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Token.termLength()

68
69
70
71
72
73
74
75
76
77
78
  private void assertTokEq(List<Token> a, List<Token> b, boolean checkOff) {
    int pos=0;
    for (Iterator iter = a.iterator(); iter.hasNext();) {
      Token tok = (Token)iter.next();
      pos += tok.getPositionIncrement();
      if (!tokAt(b, new String(tok.termBuffer(), 0, tok.termLength()), pos
              , checkOff ? tok.startOffset() : -1
              , checkOff ? tok.endOffset() : -1
              ))
      {
        fail(a + "!=" + b);
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
  public boolean tokAt(List<Token> lst, String val, int tokPos, int startOff, int endOff) {
    int pos=0;
    for (Iterator iter = lst.iterator(); iter.hasNext();) {
      Token tok = (Token)iter.next();
      pos += tok.getPositionIncrement();
      if (pos==tokPos && new String(tok.termBuffer(), 0, tok.termLength()).equals(val)
          && (startOff==-1 || tok.startOffset()==startOff)
          && (endOff  ==-1 || tok.endOffset()  ==endOff  )
           )
      {
        return true;
View Full Code Here
35
36
37
38
39
40
41
42
43
44
45
  public static class AB_Q_Stream extends BufferedTokenStream {
    public AB_Q_Stream(TokenStream input) {super(input);}
    protected Token process(Token t) throws IOException {
      if ("A".equals(new String(t.termBuffer(), 0, t.termLength()))) {
        Token t2 = read();
        if (t2!=null && "B".equals(new String(t2.termBuffer(), 0, t2.termLength()))) t.setTermText("Q");
        if (t2!=null) pushBack(t2);
      }
      return t;
    }
  }
View Full Code Here
27
28
29
30
31
32
33
34
35
36
37
  public void testSize4FalseInject() throws Exception {
    TokenStream stream = new IterTokenStream("international");
    TokenStream filter = new DoubleMetaphoneFilter(stream, 4, false);

    Token token = filter.next(new Token());
    assertEquals(4, token.termLength());
    assertEquals("ANTR", new String(token.termBuffer(), 0, token.termLength()));

    assertNull(filter.next(new Token()));
  }
View Full Code Here
28
29
30
31
32
33
34
35
36
37
38
    TokenStream stream = new IterTokenStream("international");
    TokenStream filter = new DoubleMetaphoneFilter(stream, 4, false);

    Token token = filter.next(new Token());
    assertEquals(4, token.termLength());
    assertEquals("ANTR", new String(token.termBuffer(), 0, token.termLength()));

    assertNull(filter.next(new Token()));
  }

  public void testSize4TrueInject() throws Exception {
View Full Code Here
38
39
40
41
42
43
44
45
46
47
48
  public void testSize4TrueInject() throws Exception {
    TokenStream stream = new IterTokenStream("international");
    TokenStream filter = new DoubleMetaphoneFilter(stream, 4, true);

    Token token = filter.next(new Token());
    assertEquals(13, token.termLength());
    assertEquals("international", new String(token.termBuffer(), 0, token
        .termLength()));

    token = filter.next(new Token());
    assertEquals(4, token.termLength());
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
    TokenStream stream = new IterTokenStream("international");
    TokenStream filter = new DoubleMetaphoneFilter(stream, 4, true);

    Token token = filter.next(new Token());
    assertEquals(13, token.termLength());
    assertEquals("international", new String(token.termBuffer(), 0, token
        .termLength()));

    token = filter.next(new Token());
    assertEquals(4, token.termLength());
    assertEquals("ANTR", new String(token.termBuffer(), 0, token.termLength()));
View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
    assertEquals(13, token.termLength());
    assertEquals("international", new String(token.termBuffer(), 0, token
        .termLength()));

    token = filter.next(new Token());
    assertEquals(4, token.termLength());
    assertEquals("ANTR", new String(token.termBuffer(), 0, token.termLength()));

    assertNull(filter.next(new Token()));
  }
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
    assertEquals("international", new String(token.termBuffer(), 0, token
        .termLength()));

    token = filter.next(new Token());
    assertEquals(4, token.termLength());
    assertEquals("ANTR", new String(token.termBuffer(), 0, token.termLength()));

    assertNull(filter.next(new Token()));
  }

  public void testAlternateInjectFalse() throws Exception {
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
  public void testAlternateInjectFalse() throws Exception {
    TokenStream stream = new IterTokenStream("Kuczewski");
    TokenStream filter = new DoubleMetaphoneFilter(stream, 4, false);

    Token token = filter.next(new Token());
    assertEquals(4, token.termLength());
    assertEquals("KSSK", new String(token.termBuffer(), 0, token.termLength()));

    token = filter.next(new Token());
    assertEquals(4, token.termLength());
    assertEquals("KXFS", new String(token.termBuffer(), 0, token.termLength()));
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.