Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Token.copyBuffer()


  private static Token createToken
    (String term, int start, int offset, int positionIncrement)
  {
    Token token = new Token(start, offset);
    token.copyBuffer(term.toCharArray(), 0, term.length());
    token.setPositionIncrement(positionIncrement);
    return token;
  }
 
  /** blast some random strings through the analyzer */
 
View Full Code Here


    try {
      CharTermAttribute term = stream.addAttribute( CharTermAttribute.class );
      stream.reset();
      while ( stream.incrementToken() ) {
        Token token = new Token();
        token.copyBuffer( term.buffer(), 0, term.length() );
        tokenList.add( token );
      }
      stream.end();
    }
    finally {
View Full Code Here

    int pos1 = tok1!=null ? tok1.getPositionIncrement() : 0;
    int pos2 = tok2!=null ? tok2.getPositionIncrement() : 0;
    while(tok1!=null || tok2!=null) {
      while (tok1 != null && (pos1 <= pos2 || tok2==null)) {
        Token tok = new Token(tok1.startOffset(), tok1.endOffset(), tok1.type());
        tok.copyBuffer(tok1.buffer(), 0, tok1.length());
        tok.setPositionIncrement(pos1-pos);
        result.add(tok);
        pos=pos1;
        tok1 = iter1.hasNext() ? iter1.next() : null;
        pos1 += tok1!=null ? tok1.getPositionIncrement() : 0;
View Full Code Here

        tok1 = iter1.hasNext() ? iter1.next() : null;
        pos1 += tok1!=null ? tok1.getPositionIncrement() : 0;
      }
      while (tok2 != null && (pos2 <= pos1 || tok1==null)) {
        Token tok = new Token(tok2.startOffset(), tok2.endOffset(), tok2.type());
        tok.copyBuffer(tok2.buffer(), 0, tok2.length());
        tok.setPositionIncrement(pos2-pos);
        result.add(tok);
        pos=pos2;
        tok2 = iter2.hasNext() ? iter2.next() : null;
        pos2 += tok2!=null ? tok2.getPositionIncrement() : 0;
View Full Code Here

  private static Token createToken
    (String term, int start, int offset, int positionIncrement)
  {
    Token token = new Token(start, offset);
    token.copyBuffer(term.toCharArray(), 0, term.length());
    token.setPositionIncrement(positionIncrement);
    return token;
  }
 
  /** blast some random strings through the analyzer */
 
View Full Code Here

    TokenStream stream = analyzer.tokenStream( field, new StringReader( text ) );
    CharTermAttribute term = stream.addAttribute( CharTermAttribute.class );
    List<Token> tokenList = new ArrayList<Token>();
    while ( stream.incrementToken() ) {
      Token token = new Token();
      token.copyBuffer( term.buffer(), 0, term.length() );
      tokenList.add( token );
    }

    return tokenList.toArray( new Token[tokenList.size()] );
  }
View Full Code Here

    try {
      CharTermAttribute term = stream.addAttribute( CharTermAttribute.class );
      stream.reset();
      while ( stream.incrementToken() ) {
        Token token = new Token();
        token.copyBuffer( term.buffer(), 0, term.length() );
        tokenList.add( token );
      }
      stream.end();
    }
    finally {
View Full Code Here

  private static Token createToken
    (String term, int start, int offset, int positionIncrement)
  {
    Token token = new Token(start, offset);
    token.copyBuffer(term.toCharArray(), 0, term.length());
    token.setPositionIncrement(positionIncrement);
    return token;
  }
}
View Full Code Here

  private static Token createToken
    (String term, int start, int offset, int positionIncrement)
  {
    Token token = new Token(start, offset);
    token.copyBuffer(term.toCharArray(), 0, term.length());
    token.setPositionIncrement(positionIncrement);
    return token;
  }
 
  /** blast some random strings through the analyzer */
 
View Full Code Here

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.