Examples of org.apache.lucene.analysis.standard.StandardAnalyzer.tokenStream()

Package org.apache.lucene.analysis.standard

Class org.apache.lucene.analysis.standard.StandardAnalyzer

Examples of org.apache.lucene.analysis.standard.StandardAnalyzer.tokenStream()

org.apache.lucene.analysis.standard.StandardAnalyzer.tokenStream()
Constructs a {@link StandardTokenizer} filtered by a {@link StandardFilter}, a {@link LowerCaseFilter} and a {@link StopFilter}.

               if(columns.length > 7 )
                 body = columns[6]+" "+columns[7];
               else
                 body = columns[6];
               
             TokenStream ts = analyzer.tokenStream("superColumn", new StringReader(body));
             Token token = null;
             token = ts.next();
             while(token != null)
             {
               if(token.termText() != "")

View Full Code Here

              String superColumnName = merge(superColumnList, " ");
              superColumnList.clear();
              if(importer_.columnFamily.superColumn.tokenize)
              {
                  Analyzer analyzer = new StandardAnalyzer();
                  TokenStream ts = analyzer.tokenStream("superColumn", new StringReader(superColumnName));
                  Token token = null;
                  token = ts.next();
                  while(token != null)
                  {
                    superColumnList.add(token.termText());

View Full Code Here

            throw new IllegalArgumentException
              ("field must have either String or Reader value");


          int position = 0;
          // Tokenize field and add to postingTable
          TokenStream stream = analyzer.tokenStream(fieldName, reader);
          try {
            for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
              position += (nextToken.getPositionIncrement() - 1);
              position++;
              String name = nextToken.term();

View Full Code Here


        int maxNumFragmentsRequired = 3;


        for (int i = 0; i < hits.length(); i++) {
          String text = hits.doc(i).get(FIELD_NAME);
          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
          Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream, HighlighterTest.this, false);


          highlighter.setTextFragmenter(new SimpleFragmenter(40));


          String highlightedText = highlighter.getBestFragments(tokenStream, text,

View Full Code Here

    int maxNumFragmentsRequired = 3;


    for (int i = 0; i < hits.length(); i++)
    {
      String text = hits.doc(i).get(FIELD_NAME);
      TokenStream tokenStream=analyzer.tokenStream(FIELD_NAME,new StringReader(text));


      String highlightedText = highlighter.getBestFragments(tokenStream,text,maxNumFragmentsRequired,"...");
      System.out.println(highlightedText);
    }
    //We expect to have zero highlights if the query is multi-terms and is not rewritten!

View Full Code Here


        int maxNumFragmentsRequired = 3;


        for (int i = 0; i < hits.totalHits; i++) {
          String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
          Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream, HighlighterTest.this, false);


          highlighter.setTextFragmenter(new SimpleFragmenter(40));


          String highlightedText = highlighter.getBestFragments(tokenStream, text,

View Full Code Here

      id = xpath.evaluate("//page/id/text()", doc);
      String text = xpath.evaluate("//page/revision/text/text()", doc);


      if (!text.isEmpty()) {
        Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_30);
        TokenStream stream = analyzer.tokenStream(null, new StringReader(text));
        while (stream.incrementToken()) {
          String token = stream.getAttribute(TermAttribute.class).term();


          if (dictionary != null && !dictionary.contains(token))
            continue;

View Full Code Here


        int maxNumFragmentsRequired = 3;


        for (int i = 0; i < hits.totalHits; i++) {
          String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
          Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream, HighlighterTest.this, false);


          highlighter.setTextFragmenter(new SimpleFragmenter(40));


          String highlightedText = highlighter.getBestFragments(tokenStream, text,

View Full Code Here


        int maxNumFragmentsRequired = 3;


        for (int i = 0; i < hits.totalHits; i++) {
          String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
          Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream, HighlighterTest.this, false);


          highlighter.setTextFragmenter(new SimpleFragmenter(40));


          String highlightedText = highlighter.getBestFragments(tokenStream, text,

View Full Code Here


        int maxNumFragmentsRequired = 3;


        for (int i = 0; i < hits.totalHits; i++) {
          String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
          Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream, HighlighterTest.this, false);


          highlighter.setTextFragmenter(new SimpleFragmenter(40));


          String highlightedText = highlighter.getBestFragments(tokenStream, text,

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.