Examples of TermPositions

ivory.core.data.index.TermPositions

A Hadoop {@code Writable} that encodes the position of term occurrences within a document. Termoccurrences are represented as an array of ints, where each int represents a term position. These objects serve as intermediate values in building document-sorted inverted indexes.

In serialized form, term positions are represented as first-order differences (i.e., position gaps or p-gaps) using Gamma encoding. As an example, let's say a term has a term frequency of 5, at token positions [3, 53, 58, 90, 101]. Such an object would be encoded as the following sequence of ints: 3, 50, 5, 32, 11, each of which is expressed using Gamma codes. Every int except the first represents the difference between the previous term position and the current term position.
@author Jimmy Lin
org.apache.lucene.index.TermPositions
TermPositions provides an interface for enumerating the <document, frequency, <position>* > tuples for a term.
The document and frequency are the same as for a TermDocs. The positions portion lists the ordinal positions of each occurrence of a term in a document. @see IndexReader#termPositions
uk.ac.ucl.panda.utility.structure.TermPositions
TermPositions provides an interface for enumerating the <document, frequency, <position>* > tuples for a term.
The document and frequency are the same as for a TermDocs. The positions portion lists the ordinal positions of each occurrence of a term in a document. @see IndexReader#termPositions()

Examples of org.apache.lucene.index.TermPositions

                        termDocs.seek(term);
                        while (termDocs.next()) {
                            int termPosition = 0;
                            type = PropertyType.UNDEFINED;
                            if (hasPayloads) {
                                TermPositions termPos = (TermPositions) termDocs;
                                termPosition = termPos.nextPosition();
                                if (termPos.isPayloadAvailable()) {
                                    payload = termPos.getPayload(payload, 0);
                                    type = PropertyMetaData.fromByteArray(payload).getPropertyType();
                                }
                            }
                            setValues++;
                            Comparable<?> v = getValue(value, type);

View Full Code Here

Examples of org.apache.lucene.index.TermPositions

                        termDocs.seek(term);
                        while (termDocs.next()) {
                            int termPosition = 0;
                            type = PropertyType.UNDEFINED;
                            if (hasPayloads) {
                                TermPositions termPos = (TermPositions) termDocs;
                                termPosition = termPos.nextPosition();
                                if (termPos.isPayloadAvailable()) {
                                    payload = termPos.getPayload(payload, 0);
                                    type = PropertyMetaData.fromByteArray(payload).getPropertyType();
                                }
                            }
                            setValues++;
                            Comparable<?> v = getValue(value, type);

View Full Code Here

Examples of org.apache.lucene.index.TermPositions


                        termDocs.seek(termEnum);
                        while (termDocs.next()) {
                            type = PropertyType.UNDEFINED;
                            if (hasPayloads) {
                                TermPositions termPos = (TermPositions) termDocs;
                                termPos.nextPosition();
                                if (termPos.isPayloadAvailable()) {
                                    payload = termPos.getPayload(payload, 0);
                                    type = PropertyMetaData.fromByteArray(payload).getPropertyType();
                                }
                            }
                            setValues++;
                            retArray[termDocs.doc()] = getValue(value, type);

View Full Code Here

Examples of org.apache.lucene.index.TermPositions

    }
  
    public TermPositions termPositions() {
      if (DEBUG) System.err.println("MemoryIndexReader.termPositions");
      
      return new TermPositions() {
  
        private boolean hasNext;
        private int cursor = 0;
        private ArrayIntList current;

View Full Code Here

Examples of org.apache.lucene.index.TermPositions

    //    don't reset the stream here, the DocumentWriter should do that implicitly
    writer.addDocument(doc);
    writer.close();
    
    IndexReader reader = IndexReader.open(dir);
    TermPositions termPositions = reader.termPositions(new Term("preanalyzed", "term1"));
    assertTrue(termPositions.next());
    assertEquals(1, termPositions.freq());
    assertEquals(0, termPositions.nextPosition());


    termPositions.seek(new Term("preanalyzed", "term2"));
    assertTrue(termPositions.next());
    assertEquals(2, termPositions.freq());
    assertEquals(1, termPositions.nextPosition());
    assertEquals(3, termPositions.nextPosition());
    
    termPositions.seek(new Term("preanalyzed", "term3"));
    assertTrue(termPositions.next());
    assertEquals(1, termPositions.freq());
    assertEquals(2, termPositions.nextPosition());
    reader.close();
    
    // 3) reset stream and consume tokens again
    stream.reset();
    checkTokens(stream);

View Full Code Here

Examples of org.apache.lucene.index.TermPositions

    }
  
    public TermPositions termPositions() {
      if (DEBUG) System.err.println("MemoryIndexReader.termPositions");
      
      return new TermPositions() {
  
        private boolean hasNext;
        private int cursor = 0;
        private ArrayIntList current;

View Full Code Here

Examples of org.apache.lucene.index.TermPositions

                        termDocs.seek(term);
                        while (termDocs.next()) {
                            int termPosition = 0;
                            type = PropertyType.UNDEFINED;
                            if (hasPayloads) {
                                TermPositions termPos = (TermPositions) termDocs;
                                termPosition = termPos.nextPosition();
                                if (termPos.isPayloadAvailable()) {
                                    payload = termPos.getPayload(payload, 0);
                                    type = PropertyMetaData.fromByteArray(payload).getPropertyType();
                                }
                            }
                            setValues++;
                            Comparable<?> v = getValue(value, type);

View Full Code Here

Examples of org.apache.lucene.index.TermPositions

  }


  protected void loadPayload(Term term) throws IOException
  {
    byte[] payloadBuf = null;
    TermPositions tp = _reader.termPositions();
    tp.seek(term);
    while(tp.next())
    {
      if(tp.freq() > 0)
      {
        tp.nextPosition();
        if(tp.isPayloadAvailable())
        {
          int len = tp.getPayloadLength();
          payloadBuf = tp.getPayload(payloadBuf, 0);
          add(tp.doc(), payloadBuf, len);
        }
      }
    }
    
    // save the last page

View Full Code Here

Examples of org.apache.lucene.index.TermPositions

    }
    
    @Override
    public void load() throws Exception
    {
      TermPositions tp = null;
      byte[] payloadBuffer = new byte[4];        // four bytes for an int
      try
      {
        tp = _reader.termPositions(_sizeTerm);


        if(tp == null) return;
        
        while(tp.next())
        {
          if(tp.freq() > 0)
          {
            tp.nextPosition();
            tp.getPayload(payloadBuffer, 0);
            int len = bytesToInt(payloadBuffer);
            allocate(tp.doc(), Math.min(len, _maxItems), true);
          }
        }
      }
      finally
      {
        if(tp != null) tp.close();
      }
    }

View Full Code Here

Examples of org.apache.lucene.index.TermPositions

    }
    
    @Override
    public void load() throws Exception
    {
      TermPositions tp = null;
      byte[] payloadBuffer = new byte[4];        // four bytes for an int
      try
      {
        tp = _reader.termPositions(_sizeTerm);


        if(tp == null) return;
        
        while(tp.next())
        {
          if(tp.freq() > 0)
          {
            tp.nextPosition();
            tp.getPayload(payloadBuffer, 0);
            int len = bytesToInt(payloadBuffer);
            allocate(tp.doc(), Math.min(len, _maxItems), true);
          }
        }
      }
      finally
      {
        if(tp != null) tp.close();
      }
    }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.