Examples of org.apache.lucene.analysis.TokenStream.reset()

Class org.apache.lucene.analysis.TokenStream

Examples of org.apache.lucene.analysis.TokenStream.reset()

org.apache.lucene.analysis.TokenStream.reset()
Resets this stream to the beginning. This is an optional operation, so subclasses may or may not implement this method. {@link #reset()} is not needed forthe standard indexing process. However, if the tokens of a TokenStream are intended to be consumed more than once, it is necessary to implement {@link #reset()}. Note that if your TokenStream caches tokens and feeds them back again after a reset, it is imperative that you clone the tokens when you store them away (on the first pass) as well as when you return them (on future passes after {@link #reset()}).


  private void assertReadings(String input, String... readings) throws IOException {
    TokenStream ts = analyzer.tokenStream("ignored", input);
    try {
      ReadingAttribute readingAtt = ts.addAttribute(ReadingAttribute.class);
      ts.reset();
      for(String reading : readings) {
        assertTrue(ts.incrementToken());
        assertEquals(reading, readingAtt.getReading());
      }
      assertFalse(ts.incrementToken());

View Full Code Here


  private void assertPronunciations(String input, String... pronunciations) throws IOException {
    TokenStream ts = analyzer.tokenStream("ignored", input);
    try {
      ReadingAttribute readingAtt = ts.addAttribute(ReadingAttribute.class);
      ts.reset();
      for(String pronunciation : pronunciations) {
        assertTrue(ts.incrementToken());
        assertEquals(pronunciation, readingAtt.getPronunciation());
      }
      assertFalse(ts.incrementToken());

View Full Code Here

  
  private void assertBaseForms(String input, String... baseForms) throws IOException {
    TokenStream ts = analyzer.tokenStream("ignored", input);
    try {
      BaseFormAttribute baseFormAtt = ts.addAttribute(BaseFormAttribute.class);
      ts.reset();
      for(String baseForm : baseForms) {
        assertTrue(ts.incrementToken());
        assertEquals(baseForm, baseFormAtt.getBaseForm());
      }
      assertFalse(ts.incrementToken());

View Full Code Here


  private void assertInflectionTypes(String input, String... inflectionTypes) throws IOException {
    TokenStream ts = analyzer.tokenStream("ignored", input);
    try {
      InflectionAttribute inflectionAtt = ts.addAttribute(InflectionAttribute.class);
      ts.reset();
      for(String inflectionType : inflectionTypes) {
        assertTrue(ts.incrementToken());
        assertEquals(inflectionType, inflectionAtt.getInflectionType());
      }
      assertFalse(ts.incrementToken());

View Full Code Here

    boolean hasMoreTokens = false;    
    
    TokenStream source = null;
    try {
      source = analyzer.tokenStream(field, queryText);
      source.reset();
      buffer = new CachingTokenFilter(source);
      buffer.reset();


      if (buffer.hasAttribute(TermToBytesRefAttribute.class)) {
        termAtt = buffer.getAttribute(TermToBytesRefAttribute.class);

View Full Code Here

      IOException priorException = null;
      TokenStream ts = a.tokenStream("foo", s);
      try {
        final TermToBytesRefAttribute termAtt = ts.getAttribute(TermToBytesRefAttribute.class);
        final BytesRef termBytes = termAtt.getBytesRef();
        ts.reset();


        int count = 0;
        boolean changed = false;


        while(ts.incrementToken()) {

View Full Code Here

      stream = queryContext.getQueryAnalyzer().reusableTokenStream( fieldName, reader);


      CharTermAttribute termAttribute = stream.addAttribute( CharTermAttribute.class );
      PositionIncrementAttribute positionAttribute = stream.addAttribute( PositionIncrementAttribute.class );


      stream.reset();
      int position = -1; //start at -1 since we apply at least one increment
      List<Term> termsAtSamePosition = null;
      while ( stream.incrementToken() ) {
        int positionIncrement = 1;
        if ( positionAttribute != null ) {

View Full Code Here

          "pass String parameters" );
    }
    Reader reader = new StringReader(localText);
    TokenStream stream = analyzer.reusableTokenStream( fieldName, reader);
    CharTermAttribute attribute = stream.addAttribute( CharTermAttribute.class );
    stream.reset();


    while ( stream.incrementToken() ) {
      if ( attribute.length() > 0 ) {
        String term = new String( attribute.buffer(), 0, attribute.length() );
        terms.add( term );

View Full Code Here

          "pass String parameters" );
    }
    Reader reader = new StringReader(localText);
    TokenStream stream = analyzer.reusableTokenStream( fieldName, reader);
    TermAttribute attribute = stream.addAttribute( TermAttribute.class );
    stream.reset();


    while ( stream.incrementToken() ) {
      if ( attribute.termLength() > 0 ) {
        String term = attribute.term();
        terms.add( term );

View Full Code Here

          "pass String parameters" );
    }
    Reader reader = new StringReader(localText);
    TokenStream stream = analyzer.reusableTokenStream( fieldName, reader);
    TermAttribute attribute = stream.addAttribute( TermAttribute.class );
    stream.reset();


    while ( stream.incrementToken() ) {
      if ( attribute.termLength() > 0 ) {
        String term = attribute.term();
        terms.add( term );

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.