Examples of TermPositions

ivory.core.data.index.TermPositions

A Hadoop {@code Writable} that encodes the position of term occurrences within a document. Termoccurrences are represented as an array of ints, where each int represents a term position. These objects serve as intermediate values in building document-sorted inverted indexes.

In serialized form, term positions are represented as first-order differences (i.e., position gaps or p-gaps) using Gamma encoding. As an example, let's say a term has a term frequency of 5, at token positions [3, 53, 58, 90, 101]. Such an object would be encoded as the following sequence of ints: 3, 50, 5, 32, 11, each of which is expressed using Gamma codes. Every int except the first represents the difference between the previous term position and the current term position.
@author Jimmy Lin
org.apache.lucene.index.TermPositions
TermPositions provides an interface for enumerating the <document, frequency, <position>* > tuples for a term.
The document and frequency are the same as for a TermDocs. The positions portion lists the ordinal positions of each occurrence of a term in a document. @see IndexReader#termPositions
uk.ac.ucl.panda.utility.structure.TermPositions
TermPositions provides an interface for enumerating the <document, frequency, <position>* > tuples for a term.
The document and frequency are the same as for a TermDocs. The positions portion lists the ordinal positions of each occurrence of a term in a document. @see IndexReader#termPositions()

Examples of ivory.core.data.index.TermPositions

      bitsOut = new BitOutputStream(bytesOut);


      Iterator<Map.Entry<Integer, int[]>> it = termPositionsMap.entrySet().iterator();
      Map.Entry<Integer, int[]> posting = it.next();
      int[] positions = posting.getValue();
      TermPositions tp = new TermPositions();
      // Write out the first termid.
      int lastTerm = posting.getKey().intValue();
      bitsOut.writeBinary(32, lastTerm);
      // Write out the tf value.
      bitsOut.writeGamma((short) positions.length);
      tp.set(positions, (short) positions.length);
      // Write out the positions.
      writePositions(bitsOut, tp);


      int curTerm;
      while (it.hasNext()) {
        posting = it.next();
        curTerm = posting.getKey().intValue();
        positions = posting.getValue();
        int tgap = curTerm - lastTerm;
        if (tgap <= 0) {
          throw new RuntimeException("Error: encountered invalid t-gap. termid=" + curTerm);
        }
        // Write out the gap.
        bitsOut.writeGamma(tgap);
        tp.set(positions, (short) positions.length);
        // Write out the tf value.
        bitsOut.writeGamma((short) positions.length);
        // Write out the positions.
        writePositions(bitsOut, tp);
        lastTerm = curTerm;

View Full Code Here

Examples of ivory.core.data.index.TermPositions

    StringBuffer s = new StringBuffer("[");
    try {
      Reader r = this.getReader();
      while (r.hasMoreTerms()) {
        int id = r.nextTerm();
        TermPositions pos = new TermPositions();
        r.getPositions(pos);
        s.append("(" + id + ", " + pos.getTf() + ", " + pos + ")");
      }
      s.append("]");
    } catch (Exception e) {
      e.printStackTrace();
    }

View Full Code Here

Examples of ivory.core.data.index.TermPositions

    StringBuffer s = new StringBuffer("");
    try {
      Reader r = this.getReader();
      while (r.hasMoreTerms()) {
        int id = r.nextTerm();
        TermPositions pos = new TermPositions();
        r.getPositions(pos);
        s.append(String.format("(%d, %d, %s)", map.getTerm(id), pos.getTf(), pos));
      }
      s.append("]");
    } catch (Exception e) {
      e.printStackTrace();
    }

View Full Code Here

Examples of ivory.core.data.index.TermPositions

        }


        numPostings = 0;
        Iterator<TermPositions> iter = values.iterator();
        while (iter.hasNext()) {
          TermPositions positions = iter.next();
          numPostings += positions.getPositions()[0];
        }


        postings.setNumberOfPostings(numPostings);
        return;
      }


      Iterator<TermPositions> iter = values.iterator();
      TermPositions positions = iter.next();
      postings.add(pair.getRightElement(), positions.getTf(), positions);


      if (iter.hasNext()) {
        throw new RuntimeException(
            String.format("Error: values with the same (term, docno): docno=%d, term=%d",
                pair.getRightElement(), curTerm));

View Full Code Here

Examples of ivory.core.data.index.TermPositions

        }


        numPostings = 0;
        Iterator<TermPositions> iter = values.iterator();
        while (iter.hasNext()) {
          TermPositions positions = iter.next();
          numPostings += positions.getPositions()[0];
        }


        postings.setNumberOfPostings(numPostings);
        return;
      }


      Iterator<TermPositions> iter = values.iterator();
      TermPositions positions = iter.next();
      postings.add(pair.getRightElement(), positions.getTf(), positions);


      if (iter.hasNext()) {
        throw new RuntimeException(
            String.format("Error: values with the same (term, docno): docno=%d, term=%d",
                pair.getRightElement(), curTerm));

View Full Code Here

Examples of ivory.core.data.index.TermPositions

      bitsOut = new BitOutputStream(bytesOut);


      Iterator<Map.Entry<Integer, int[]>> it = termPositionsMap.entrySet().iterator();
      Map.Entry<Integer, int[]> posting = it.next();
      int[] positions = posting.getValue();
      TermPositions tp = new TermPositions();
      // Write out the first termid.
      int lastTerm = posting.getKey().intValue();
      bitsOut.writeBinary(32, lastTerm);
      // Write out the tf value.
      bitsOut.writeGamma((short) positions.length);
      tp.set(positions, (short) positions.length);
      // Write out the positions.
      writePositions(bitsOut, tp);


      int curTerm;
      while (it.hasNext()) {
        posting = it.next();
        curTerm = posting.getKey().intValue();
        positions = posting.getValue();
        int tgap = curTerm - lastTerm;
        if (tgap <= 0) {
          throw new RuntimeException("Error: encountered invalid t-gap. termid=" + curTerm);
        }
        // Write out the gap.
        bitsOut.writeGamma(tgap);
        tp.set(positions, (short) positions.length);
        // Write out the tf value.
        bitsOut.writeGamma((short) positions.length);
        // Write out the positions.
        writePositions(bitsOut, tp);
        lastTerm = curTerm;

View Full Code Here

Examples of ivory.core.data.index.TermPositions

    StringBuffer s = new StringBuffer("[");
    try {
      Reader r = this.getReader();
      while (r.hasMoreTerms()) {
        int id = r.nextTerm();
        TermPositions pos = new TermPositions();
        r.getPositions(pos);
        s.append("(" + id + ", " + pos.getTf() + ", " + pos + ")");
      }
      s.append("]");
    } catch (Exception e) {
      e.printStackTrace();
    }

View Full Code Here

Examples of ivory.core.data.index.TermPositions

    StringBuffer s = new StringBuffer("");
    try {
      Reader r = this.getReader();
      while (r.hasMoreTerms()) {
        int id = r.nextTerm();
        TermPositions pos = new TermPositions();
        r.getPositions(pos);
        s.append(String.format("(%d, %d, %s)", map.getTerm(id), pos.getTf(), pos));
      }
      s.append("]");
    } catch (Exception e) {
      e.printStackTrace();
    }

View Full Code Here

Examples of org.apache.lucene.index.TermPositions

    }
    
    @Override
    public void load() throws Exception
    {
      TermPositions tp = null;
      byte[] payloadBuffer = new byte[4];        // four bytes for an int
      try
      {
        tp = _reader.termPositions(_sizeTerm);


        if(tp == null) return;
        
        while(tp.next())
        {
          if(tp.freq() > 0)
          {
            tp.nextPosition();
            tp.getPayload(payloadBuffer, 0);
            int len = bytesToInt(payloadBuffer);
            allocate(tp.doc(), Math.min(len, _maxItems), true);
          }
        }
      }
      finally
      {
        if(tp != null) tp.close();
      }
    }

View Full Code Here

Examples of org.apache.lucene.index.TermPositions

      aprioriTermDocs.close();
      testTermDocs.close();


      // compare term positions


      TermPositions testTermPositions = testReader.termPositions(testTermEnum.term());
      TermPositions aprioriTermPositions = aprioriReader.termPositions(aprioriTermEnum.term());


      if (aprioriTermPositions != null) {


        for (int docIndex = 0; docIndex < aprioriReader.maxDoc(); docIndex++) {
          boolean hasNext = aprioriTermPositions.next();
          if (hasNext) {
            assertTrue(testTermPositions.next());


            assertEquals(aprioriTermPositions.freq(), testTermPositions.freq());




            for (int termPositionIndex = 0; termPositionIndex < aprioriTermPositions.freq(); termPositionIndex++) {
              int aprioriPos = aprioriTermPositions.nextPosition();
              int testPos = testTermPositions.nextPosition();


              if (aprioriPos != testPos) {
                assertEquals(aprioriPos, testPos);
              }




              assertEquals(aprioriTermPositions.isPayloadAvailable(), testTermPositions.isPayloadAvailable());
              if (aprioriTermPositions.isPayloadAvailable()) {
                assertEquals(aprioriTermPositions.getPayloadLength(), testTermPositions.getPayloadLength());
                byte[] aprioriPayloads = aprioriTermPositions.getPayload(new byte[aprioriTermPositions.getPayloadLength()], 0);
                byte[] testPayloads = testTermPositions.getPayload(new byte[testTermPositions.getPayloadLength()], 0);
                for (int i = 0; i < aprioriPayloads.length; i++) {
                  assertEquals(aprioriPayloads[i], testPayloads[i]);
                }
              }


            }
          }
        }


        aprioriTermPositions.close();
        testTermPositions.close();


      }
    }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.