Examples of org.apache.lucene.analysis.Token.endOffset()

org.apache.lucene.analysis.Token.endOffset()
Returns this Token's ending offset, one greater than the position of the last character corresponding to this token in the source text. The length of the token in the source text is (endOffset - startOffset).

            if (currentToken >= tokens.length) {
              return false;
            }
            Token token = tokens[currentToken++];
            termAtt.setTermBuffer(token.term());
            offsetAtt.setOffset(token.startOffset(), token.endOffset());
            return true;
          }
        }      
        //code to reconstruct the original sequence of Tokens
        String[] terms=tpv.getTerms();

View Full Code Here

      } else
        while ((token = this.input.next(reusableToken)) != null) {
          if (token.term().equals("phrase")) {
            inPhrase = true;
            savedStart = token.startOffset();
            savedEnd = token.endOffset();
            token.setTermBuffer("phrase1");
            token.setStartOffset(savedStart);
            token.setEndOffset(savedEnd);
            return token;
          } else if (!token.term().equals("stop"))

View Full Code Here


        Token token = null;


        while ((token = ts.next()) != null) {
            System.out.println("ReTokenizeFile.reTokenize(File): " + token.termText() + " " +
                token.startOffset() + " " + token.endOffset() + " " + token.type());
        }


        return file.getAbsolutePath();
    }

View Full Code Here

            {
               TermVectorOffsetInfo[] tmp = info;
               info = new TermVectorOffsetInfo[tmp.length + 1];
               System.arraycopy(tmp, 0, info, 0, tmp.length);
            }
            info[info.length - 1] = new TermVectorOffsetInfo(t.startOffset(), t.endOffset());
            termMap.put(termText, info);
         }
      }
      catch (IOException e)
      {

View Full Code Here

            {
               Token t = tokens.get(i);
               // only replace if word acutally changed
               if (!t.termText().equalsIgnoreCase(suggestions[i]))
               {
                  sb.replace(t.startOffset(), t.endOffset(), suggestions[i]);
               }
            }
            // if suggestion is same as a statement return null
            String result = sb.toString();
            if (statement.equalsIgnoreCase(result))

View Full Code Here

         try
         {
            Token t;
            while ((t = ts.next()) != null)
            {
               String origWord = statement.substring(t.startOffset(), t.endOffset());
               if (t.getPositionIncrement() > 0)
               {
                  words.add(t.termText());
                  tokens.add(t);
               }

View Full Code Here

                    TermVectorOffsetInfo[] tmp = info;
                    info = new TermVectorOffsetInfo[tmp.length + 1];
                    System.arraycopy(tmp, 0, info, 0, tmp.length);
                }
                info[info.length - 1] = new TermVectorOffsetInfo(
                        t.startOffset(), t.endOffset());
                termMap.put(termText, info);
            }
        } catch (IOException e) {
            // should never happen, we are reading from a string
        }

View Full Code Here

    clearAttributes();


    final Token t = next();
    if (t != null) {
      termAtt.setTermBuffer(t.termBuffer(), 0, t.termLength());
      offsetAtt.setOffset(t.startOffset(), t.endOffset());
      posIncrAtt.setPositionIncrement(t.getPositionIncrement());
      typeAtt.setType(t.type());
      return true;
    } else {
      return false;

View Full Code Here

          //
          Token t = tokens[j];
          if (highlight.contains(t.term())) {
            excerpt.addToken(t.term());
            excerpt.add(new Fragment(text.substring(offset, t.startOffset())));
            excerpt.add(new Highlight(text.substring(t.startOffset(),t.endOffset())));
            offset = t.endOffset();
            endToken = Math.min(j + sumContext, tokens.length);
          }
          
          j++;

View Full Code Here

          Token t = tokens[j];
          if (highlight.contains(t.term())) {
            excerpt.addToken(t.term());
            excerpt.add(new Fragment(text.substring(offset, t.startOffset())));
            excerpt.add(new Highlight(text.substring(t.startOffset(),t.endOffset())));
            offset = t.endOffset();
            endToken = Math.min(j + sumContext, tokens.length);
          }
          
          j++;
        }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.