Examples of org.apache.lucene.analysis.Token.termLength()

org.apache.lucene.analysis.Token.termLength()
Return number of valid characters (length of the term) in the termBuffer array.

      }


      // common case fast-path of first token not matching anything
      Token firstTok = nextTok(target);
      if (firstTok == null) return null;
      SynonymMap result = map.submap!=null ? map.submap.get(firstTok.termBuffer(), 0, firstTok.termLength()) : null;
      if (result == null) return firstTok;


      // OK, we matched a token, so find the longest match.


      matched = new LinkedList<Token>();

View Full Code Here


    if (map.submap != null) {
      Token tok = nextTok();
      if (tok != null) {
        // check for positionIncrement!=1?  if>1, should not match, if==0, check multiple at this level?
        SynonymMap subMap = map.submap.get(tok.termBuffer(), 0, tok.termLength());


        if (subMap != null) {
          // recurse
          result = match(subMap);
        }

View Full Code Here

    NamedList<NamedList<Object>> tokens = new NamedList<NamedList<Object>>();
    Token t = null;
    while (((t = tstream.next()) != null)) {
      NamedList<Object> token = new SimpleOrderedMap<Object>();
      tokens.add("token", token);
      token.add("value", new String(t.termBuffer(), 0, t.termLength()));
      token.add("start", t.startOffset());
      token.add("end", t.endOffset());
      token.add("posInc", t.getPositionIncrement());
      token.add("type", t.type());
      //TODO: handle payloads

View Full Code Here

                                Fieldable[] fields = aDoc.getFieldables(FieldNames.PROPERTIES);
                                Token t = new Token();
                                for (Fieldable field : fields) {
                                    // assume properties fields use SingleTokenStream
                                    t = field.tokenStreamValue().next(t);
                                    String value = new String(t.termBuffer(), 0, t.termLength());
                                    if (value.startsWith(namePrefix)) {
                                        // extract value
                                        value = value.substring(namePrefix.length());
                                        // create new named value
                                        Path p = getRelativePath(state, propState);

View Full Code Here

    TokenStream stream = tokenStream(new StringReader(text));
    Token token = new Token();


    try {
      while ((token = stream.next(token)) != null) {
        str.append(token.termBuffer(), 0, token.termLength());
        str.append(" ");
      }
    } catch (Exception e) {
      e.printStackTrace();
    }

View Full Code Here


    if (t == null)
      return null;


    char[] buffer = t.termBuffer();
    final int bufferLength = t.termLength();
    final String type = t.type();


    if (type == APOSTROPHE_TYPE &&      // remove 's
  bufferLength >= 2 &&
        buffer[bufferLength-2] == '\'' &&

View Full Code Here

                        {
                           Fieldable field = fields[k];
                           // assume properties fields use
                           // SingleTokenStream
                           t = field.tokenStreamValue().next(t);
                           String value = new String(t.termBuffer(), 0, t.termLength());
                           if (value.startsWith(namePrefix))
                           {
                              // extract value
                              value = value.substring(namePrefix.length());
                              // create new named value

View Full Code Here

                                Fieldable[] fields = aDoc.getFieldables(FieldNames.PROPERTIES);
                                Token t = new Token();
                                for (Fieldable field : fields) {
                                    // assume properties fields use SingleTokenStream
                                    t = field.tokenStreamValue().next(t);
                                    String value = new String(t.termBuffer(), 0, t.termLength());
                                    if (value.startsWith(namePrefix)) {
                                        // extract value
                                        value = value.substring(namePrefix.length());
                                        // create new named value
                                        Path p = getRelativePath(state, propState);

View Full Code Here

      document = StringEscapeUtils.unescapeHtml(document.replaceFirst("<text xml:space=\"preserve\">", "").replaceAll("</text>", ""));
      TokenStream stream = analyzer.tokenStream(country, new StringReader(document));
      while(true){
        Token token = stream.next();
        if(token==null) break;
        contents.append(token.termBuffer(), 0, token.termLength()).append(' ');
      }
      output.collect(new Text(country.replace(" ","_")), new Text(contents.toString()));
    }
  }

View Full Code Here

      writer.write('\t'); // edit: Inorder to match Hadoop standard
      // TextInputFormat
      Token token = new Token();
      while ((token = ts.next(token)) != null) {
        char[] termBuffer = token.termBuffer();
        int termLen = token.termLength();
        writer.write(termBuffer, 0, termLen);
        writer.write(' ');
      }
    } finally {
      quietClose(reader);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.