Examples of TermPositions

ivory.core.data.index.TermPositions

A Hadoop {@code Writable} that encodes the position of term occurrences within a document. Termoccurrences are represented as an array of ints, where each int represents a term position. These objects serve as intermediate values in building document-sorted inverted indexes.

In serialized form, term positions are represented as first-order differences (i.e., position gaps or p-gaps) using Gamma encoding. As an example, let's say a term has a term frequency of 5, at token positions [3, 53, 58, 90, 101]. Such an object would be encoded as the following sequence of ints: 3, 50, 5, 32, 11, each of which is expressed using Gamma codes. Every int except the first represents the difference between the previous term position and the current term position.
@author Jimmy Lin
org.apache.lucene.index.TermPositions
TermPositions provides an interface for enumerating the <document, frequency, <position>* > tuples for a term.
The document and frequency are the same as for a TermDocs. The positions portion lists the ordinal positions of each occurrence of a term in a document. @see IndexReader#termPositions
uk.ac.ucl.panda.utility.structure.TermPositions
TermPositions provides an interface for enumerating the <document, frequency, <position>* > tuples for a term.
The document and frequency are the same as for a TermDocs. The positions portion lists the ordinal positions of each occurrence of a term in a document. @see IndexReader#termPositions()

Examples of org.apache.lucene.index.TermPositions

                        new StringReader("a a b c d e a f g h i j a b k k")));
      writer.addDocument(doc);


      IndexReader r = writer.getReader();


      TermPositions tp = r.termPositions(new Term("content", "a"));
      int count = 0;
      assertTrue(tp.next());
      // "a" occurs 4 times
      assertEquals(4, tp.freq());
      int expected;
      if (x == 1) {
        expected = Integer.MAX_VALUE;
      } else {
        expected = 0;
      }
      assertEquals(expected, tp.nextPosition());
      if (x == 1) {
        continue;
      }
      assertEquals(1, tp.nextPosition());
      assertEquals(3, tp.nextPosition());
      assertEquals(6, tp.nextPosition());


      // only one doc has "a"
      assertFalse(tp.next());


      IndexSearcher is = new IndexSearcher(r);
    
      SpanTermQuery stq1 = new SpanTermQuery(new Term("content", "a"));
      SpanTermQuery stq2 = new SpanTermQuery(new Term("content", "k"));

View Full Code Here

Examples of org.apache.lucene.index.TermPositions

    //    don't reset the stream here, the DocumentWriter should do that implicitly
    writer.addDocument(doc);
    writer.close();
    
    IndexReader reader = IndexReader.open(dir);
    TermPositions termPositions = reader.termPositions(new Term("preanalyzed", "term1"));
    assertTrue(termPositions.next());
    assertEquals(1, termPositions.freq());
    assertEquals(0, termPositions.nextPosition());


    termPositions.seek(new Term("preanalyzed", "term2"));
    assertTrue(termPositions.next());
    assertEquals(2, termPositions.freq());
    assertEquals(1, termPositions.nextPosition());
    assertEquals(3, termPositions.nextPosition());
    
    termPositions.seek(new Term("preanalyzed", "term3"));
    assertTrue(termPositions.next());
    assertEquals(1, termPositions.freq());
    assertEquals(2, termPositions.nextPosition());
    reader.close();
    
    // 3) reset stream and consume tokens again
    stream.reset();
    checkTokens(stream);

View Full Code Here

Examples of org.apache.lucene.index.TermPositions

  
    @Override
    public TermPositions termPositions() {
      if (DEBUG) System.err.println("MemoryIndexReader.termPositions");
      
      return new TermPositions() {
  
        private boolean hasNext;
        private int cursor = 0;
        private ArrayIntList current;
        private Term term;

View Full Code Here

Examples of org.apache.lucene.index.TermPositions

    assertEquals(1, reader.docFreq(new Term("content", "term")));
    assertEquals(1, reader.docFreq(new Term("content", "another")));


    // Make sure position is still incremented when
    // massive term is skipped:
    TermPositions tps = reader.termPositions(new Term("content", "another"));
    assertTrue(tps.next());
    assertEquals(1, tps.freq());
    assertEquals(3, tps.nextPosition());


    // Make sure the doc that has the massive term is in
    // the index:
    assertEquals("document with wicked long term should is not in the index!", 2, reader.numDocs());

View Full Code Here

Examples of org.apache.lucene.index.TermPositions

        return null;


      PostingsAndFreq[] postingsFreqs = new PostingsAndFreq[terms.size()];
      for (int i = 0; i < terms.size(); i++) {
        final Term t = terms.get(i);
        TermPositions p = reader.termPositions(t);
        if (p == null)
          return null;
        postingsFreqs[i] = new PostingsAndFreq(p, reader.docFreq(t), positions.get(i).intValue(), t);
      }

View Full Code Here

Examples of org.apache.lucene.index.TermPositions


                        termDocs.seek(termEnum);
                        while (termDocs.next()) {
                            type = PropertyType.UNDEFINED;
                            if (hasPayloads) {
                                TermPositions termPos = (TermPositions) termDocs;
                                termPos.nextPosition();
                                if (termPos.isPayloadAvailable()) {
                                    payload = termPos.getPayload(payload, 0);
                                    type = PropertyMetaData.fromByteArray(payload).getPropertyType();
                                }
                            }
                            setValues++;
                            retArray[termDocs.doc()] = getValue(value, type);

View Full Code Here

Examples of org.apache.lucene.index.TermPositions

      aprioriTermDocs.close();
      testTermDocs.close();


      // compare term positions


      TermPositions testTermPositions = testReader.termPositions(testTermEnum.term());
      TermPositions aprioriTermPositions = aprioriReader.termPositions(aprioriTermEnum.term());


      if (aprioriTermPositions != null) {


        for (int docIndex = 0; docIndex < aprioriReader.maxDoc(); docIndex++) {
          boolean hasNext = aprioriTermPositions.next();
          if (hasNext) {
            assertTrue(testTermPositions.next());


            assertEquals(aprioriTermPositions.freq(), testTermPositions.freq());




            for (int termPositionIndex = 0; termPositionIndex < aprioriTermPositions.freq(); termPositionIndex++) {
              int aprioriPos = aprioriTermPositions.nextPosition();
              int testPos = testTermPositions.nextPosition();


              if (aprioriPos != testPos) {
                assertEquals(aprioriPos, testPos);
              }




              assertEquals(aprioriTermPositions.isPayloadAvailable(), testTermPositions.isPayloadAvailable());
              if (aprioriTermPositions.isPayloadAvailable()) {
                assertEquals(aprioriTermPositions.getPayloadLength(), testTermPositions.getPayloadLength());
                byte[] aprioriPayloads = aprioriTermPositions.getPayload(new byte[aprioriTermPositions.getPayloadLength()], 0);
                byte[] testPayloads = testTermPositions.getPayload(new byte[testTermPositions.getPayloadLength()], 0);
                for (int i = 0; i < aprioriPayloads.length; i++) {
                  assertEquals(aprioriPayloads[i], testPayloads[i]);
                }
              }


            }
          }
        }


        aprioriTermPositions.close();
        testTermPositions.close();


      }
    }

View Full Code Here

Examples of org.apache.lucene.index.TermPositions

    writer.close();
    


    IndexSearcher searcher = new IndexSearcher(store, true);
    
    TermPositions pos = searcher.getIndexReader().termPositions(new Term("field", "1"));
    pos.next();
    // first token should be at position 0
    assertEquals(0, pos.nextPosition());
    
    pos = searcher.getIndexReader().termPositions(new Term("field", "2"));
    pos.next();
    // second token should be at position 2
    assertEquals(2, pos.nextPosition());
    
    PhraseQuery q;
    ScoreDoc[] hits;


    q = new PhraseQuery();

View Full Code Here

Examples of org.apache.lucene.index.TermPositions

                      new StringReader("a a b c d e a f g h i j a b k k")));
    writer.addDocument(doc);


    IndexReader r = writer.getReader();


    TermPositions tp = r.termPositions(new Term("content", "a"));
    int count = 0;
    assertTrue(tp.next());
    // "a" occurs 4 times
    assertEquals(4, tp.freq());
    int expected = 0;
    assertEquals(expected, tp.nextPosition());
    assertEquals(1, tp.nextPosition());
    assertEquals(3, tp.nextPosition());
    assertEquals(6, tp.nextPosition());


    // only one doc has "a"
    assertFalse(tp.next());


    IndexSearcher is = new IndexSearcher(r);
  
    SpanTermQuery stq1 = new SpanTermQuery(new Term("content", "a"));
    SpanTermQuery stq2 = new SpanTermQuery(new Term("content", "k"));

View Full Code Here

Examples of org.apache.lucene.index.TermPositions


      TermPositions[] tps = new TermPositions[termArrays.size()];
      for (int i=0; i<tps.length; i++) {
        Term[] terms = termArrays.get(i);


        TermPositions p;
        if (terms.length > 1)
          p = new MultipleTermPositions(reader, terms);
        else
          p = reader.termPositions(terms[0]);

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.