Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Token.endOffset()

78
79
80
81
82
83
84
85
86
87
88
    public boolean incrementToken() throws IOException {
      clearAttributes();
      Token t = next();
      if (t != null) {
        termAtt.setTermBuffer(t.termBuffer(), 0, t.termLength());
        offsetAtt.setOffset(t.startOffset(), t.endOffset());
        posIncrAtt.setPositionIncrement(t.getPositionIncrement());
        typeAtt.setType(t.type());
      }    
      return t != null;
    }
View Full Code Here
245
246
247
248
249
250
251
252
253
254
255
                    TermVectorOffsetInfo[] tmp = info;
                    info = new TermVectorOffsetInfo[tmp.length + 1];
                    System.arraycopy(tmp, 0, info, 0, tmp.length);
                }
                info[info.length - 1] = new TermVectorOffsetInfo(
                        t.startOffset(), t.endOffset());
                termMap.put(termText, info);
            }
        } catch (IOException e) {
            // should never happen, we are reading from a string
        }
View Full Code Here
242
243
244
245
246
247
248
249
250
251
252
                    TermVectorOffsetInfo[] tmp = info;
                    info = new TermVectorOffsetInfo[tmp.length + 1];
                    System.arraycopy(tmp, 0, info, 0, tmp.length);
                }
                info[info.length - 1] = new TermVectorOffsetInfo(
                        t.startOffset(), t.endOffset());
                termMap.put(termText, info);
            }
        } catch (IOException e) {
            // should never happen, we are reading from a string
        }
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
  public final boolean incrementToken() throws IOException {
    clearAttributes();
    Token token = nextToken(reusableToken);
    if(token != null) {
      termAtt.copyBuffer(token.buffer(), 0, token.length());
      offsetAtt.setOffset(token.startOffset(), token.endOffset());
      typeAtt.setType(token.type());
      return true;
    } else {
      end();
      return false;
View Full Code Here
187
188
189
190
191
192
193
194
195
196
197
          return false;
        }
        Token token = tokens[currentToken++];
        clearAttributes();
        termAtt.setEmpty().append(token);
        offsetAtt.setOffset(token.startOffset(), token.endOffset());
        BytesRef payload = token.getPayload();
        if (payload != null) {
          payloadAtt.setPayload(payload);
        }
        posincAtt
View Full Code Here
187
188
189
190
191
192
193
194
195
196
197
          return false;
        }
        Token token = tokens[currentToken++];
        clearAttributes();
        termAtt.setEmpty().append(token);
        offsetAtt.setOffset(token.startOffset(), token.endOffset());
        BytesRef payload = token.getPayload();
        if (payload != null) {
          payloadAtt.setPayload(payload);
        }
        posincAtt
View Full Code Here
121
122
123
124
125
126
127
128
129
130
131
      final Token next = this.tokensAtCurrentPosition.next();
      clearAttributes();
      termAttribute.setEmpty().append(next);
      positionIncrementAttribute.setPositionIncrement(next
          .getPositionIncrement());
      offsetAttribute.setOffset(next.startOffset(), next.endOffset());
      payloadAttribute.setPayload(next.getPayload());
      return true;
    }
    return false;
  }
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
          public boolean incrementToken() {
            if (toks.hasNext()) {
              clearAttributes();
              Token tok = toks.next();
              termAtt.setEmpty().append(tok);
              offsetAtt.setOffset(tok.startOffset(), tok.endOffset());
              posIncAtt.setPositionIncrement(tok.getPositionIncrement());
              return true;
            } else {
              return false;
            }
View Full Code Here
340
341
342
343
344
345
346
347
348
349
350
            for(Token token : expected) {
              int pos = Integer.parseInt(token.type());
              //System.out.println("        pos=" + pos);
              assertEquals(pos, docsAndPositionsAndOffsets.nextPosition());
              assertEquals(token.startOffset(), docsAndPositionsAndOffsets.startOffset());
              assertEquals(token.endOffset(), docsAndPositionsAndOffsets.endOffset());
            }
          }
        }
      }       
      // TODO: test advance:
View Full Code Here
180
181
182
183
184
185
186
187
188
189
190
                break;
              }
            }
           
            if(lastToken != null)
              offset += lastToken.endOffset() + 1;
           
          } finally {
            stream.close();
          }
        }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.