Examples of Payload


Examples of it.unimi.dsi.mg4j.index.payload.Payload

    final String field = properties.getString( Index.PropertyKeys.FIELD, new File( basename.toString() ).getName() );

    if ( termMap != null && termMap.size() != numberOfTerms ) throw new IllegalArgumentException( "The size of the term map (" + termMap.size() + ") is not equal to the number of terms (" + numberOfTerms + ")" );
    if ( prefixMap != null && prefixMap.size() != numberOfTerms ) throw new IllegalArgumentException( "The size of the prefix map (" + prefixMap.size() + ") is not equal to the number of terms (" + numberOfTerms + ")" );

    final Payload payload = (Payload)( properties.containsKey( Index.PropertyKeys.PAYLOADCLASS ) ? Class.forName( properties.getString( Index.PropertyKeys.PAYLOADCLASS ) ).newInstance() : null );
    final Coding frequencyCoding = flags.get( Component.FREQUENCIES );
    final Coding pointerCoding = flags.get( Component.POINTERS );
    final Coding countCoding = flags.get( Component.COUNTS );
    final Coding positionCoding = flags.get( Component.POSITIONS );
   
View Full Code Here

Examples of it.unimi.dsi.mg4j.index.payload.Payload

    if ( sizes != null && documentSizes ) LOGGER.warn( "You are loading both local sizes and a global size file specified by the \"size\" properties, which is usually nonsensical" );

    boolean hasCounts = true;
    boolean hasPositions = true;
    Payload payload = null;
   
    for ( int i = 0; i < localIndex.length; i++ ) {
      hasCounts = hasCounts && localIndex[ i ].hasCounts;
      hasPositions = hasPositions && localIndex[ i ].hasPositions;

      if ( i == 0 ) payload = localIndex[ i ].payload;
      if ( ( payload == null ) != ( localIndex[ i ].payload == null ) || payload != null && ! payload.compatibleWith( localIndex[ i ].payload ) ) throw new IllegalStateException( "The payload specification of index " + localIndex[ 0 ] + " is not compatible with that of index " + localIndex[ i ] );
    }

    // We stem the names of Bloom filters from the index basename.
    BloomFilter[] termFilter = null;
    if ( properties.getBoolean( DocumentalCluster.PropertyKeys.BLOOM ) ) {
View Full Code Here

Examples of it.unimi.dsi.mg4j.index.payload.Payload

              // TODO: write tests for the other case
              if ( allBitStreamIndices ) {
                IndexIterator indexIterator = indexReader[ i ].documents( 0 );
                int pointer = indexIterator.skipTo( currDoc );
                if ( pointer == currDoc ) {
                  Payload payload = indexIterator.payload();
                  if ( ! payload.get().equals( content ) ) LOGGER.error( index[ i ] + ": Document " + currDoc + " has payload " + content + " but the index says " + payload )
                }
                else LOGGER.error( index[ i ] + ": Document " + currDoc + " does not appear in the inverted list of term " + t );
              }
              else {
                IndexIterator indexIterator = indexReader[ i ].documents);
                if ( indexIterator.skipTo( currDoc ) == currDoc ) {
                  if ( ! indexIterator.payload().get().equals( content ) )
                    LOGGER.error( index[ i ] + ": Document " + currDoc + " has payload " + content + " but the index says " + indexIterator.payload().get() );
                }
                else LOGGER.error( index[ i ] + ": Document " + currDoc + " does not appear in the inverted list of term " + t );
              }
            }
            else {
              // text index
              pos = 0;
              termsInDoc[ i ].clear();
              reader = (Reader)content;
              wordReader = document.wordReader( stem || index[ i ].field == null ? indexedField[ i ] : factory.fieldIndex( index[ i ].field ) );
              wordReader.setReader( reader );
              while( wordReader.next( word, nonWord ) ) {
                if ( word.length() == 0 || index[ i ].termProcessor != null && ! index[ i ].termProcessor.processTerm( word ) ) continue;
                if ( ( t = (int)( (BitStreamIndex)index[ i ] ).termMap.getLong( word ) ) == -1 ) LOGGER.error( index[ i ] + ": Could not find term " + word + " in term index" );
                else {
                  if ( index[ i ].hasCounts ) termsInDoc[ i ].put( t, termsInDoc[ i ].get( t ) + 1 );
                  if ( index[ i ].hasPositions ) wordInPos[ i ][ pos++ ] = t;
                }
              }

              if ( allBitStreamIndices ) {
                for( IntIterator x = termsInDoc[ i ].keySet().iterator(); x.hasNext(); ) {
                  t = x.nextInt();

                  IndexIterator indexIterator = indexReader[ i ].documents( t );

                  int pointer = indexIterator.skipTo( currDoc );
                  if ( pointer == currDoc ) {
                    if ( index[ i ].hasCounts ) {
                      int c = indexIterator.count();
                      if ( termsInDoc[ i ].get( t ) !=  c )
                        LOGGER.error( index[ i ] + ": The count for term " + t + " in document " + currDoc + " is " + c + " instead of " + termsInDoc[ i ].get( t ) );
                      else {
                        if ( index[ i ].hasPositions ) {
                          indexIterator.positions( occ[ i ] );

                          for( int j = 0; j < c; j++ )
                            if ( wordInPos[ i ][ occ[ i ][ j ] ] != t
                              LOGGER.error( index[ i ] + ": The occurrence of index " + i + " of term " + t + " (position " + occ[ i ] +") in document " + currDoc + " is occupied instead by term " + wordInPos[ i ][ occ[ i ][ j ] ] );
                        }
                      }
                    }
                  }
                  else LOGGER.error( index[ i ] + ": Document " + currDoc + " does not appear in the inverted list of term " + t + "(skipTo returned " + pointer + ")" );
                }
              }
              else {
                for( IntIterator x = termsInDoc[ i ].keySet().iterator(); x.hasNext(); ) {
                  t = x.nextInt();
                  IndexIterator indexIterator = termLists ? indexReader[ i ].documents( terms[ i ].get( t ) ) : indexReader[ i ].documents( t );

                  if ( indexIterator.skipTo( currDoc ) == currDoc ) {
                    if ( index[ i ].hasCounts ) {
                      int c = indexIterator.count();
                      if ( termsInDoc[ i ].get( t ) !=  c )
                        LOGGER.error( index[ i ] + ": The count for term " + t + " in document " + currDoc + " is " + c + " instead of " + termsInDoc[ i ].get( t ) );
                      else {
                        if ( index[ i ].hasPositions ) {
                          indexIterator.positions( occ[ i ] );

                          for( int j = 0; j < c; j++ )
                            if ( wordInPos[ i ][ occ[ i ][ j ] ] != t
                              LOGGER.error( index[ i ] + ": The occurrence of index " + i + " of term " + t + " (position " + occ[ i ] +") in document " + currDoc + " is occupied instead by term " + wordInPos[ i ][ occ[ i ][ j ] ] );
                        }
                      }
                    }
                  }
                  else LOGGER.error( index[ i ] + ": Document " + currDoc + " does not appear in the inverted list of term " + t );
                }
              }
            }
          }
          docCounter++;
          document.close();
          pl.update();
        }
      }
      else {
        LOGGER.warn( "Random access tests require very slow single-term scanning as not all indices are disk based" );

        it.unimi.dsi.mg4j.document.DocumentIterator documentIterator = documentSequence.iterator();
        Document document;
        Reader reader;
        WordReader wordReader;
       
        final MutableString word = new MutableString(), nonWord = new MutableString();
       
        int docCounter = 0;
       
        while( ( document = documentIterator.nextDocument() ) != null ) {
          currDoc = permutation != null ? permutation[ docCounter ] : docCounter;

          for( i = 0; i < index.length; i++ ) {
            Object content = document.content( stem || index[ i ].field == null ? indexedField[ i ] : factory.fieldIndex( index[ i ].field ) );
            if ( index[ i ].hasPayloads ) {
              if ( allBitStreamIndices ) {
                IndexIterator indexIterator = indexReader[ i ].documents( 0 );
                int pointer = indexIterator.skipTo( currDoc );
                if ( pointer == currDoc ) {
                  Payload payload = indexIterator.payload();
                  if ( ! payload.get().equals( content ) ) LOGGER.error( index[ i ] + ": Document " + currDoc + " has payload " + content + " but the index says " + payload )
                }
                else LOGGER.error( index[ i ] + ": Document " + currDoc + " does not appear in the inverted list of term " + t );
              }
              else {
                IndexIterator indexIterator = indexReader[ i ].documents( "#" );
View Full Code Here

Examples of org.apache.lucene.index.Payload

          buffer[0] = (byte) (size);
          buffer[1] = (byte) (size >> 8);
          buffer[2] = (byte) (size >> 16);
          buffer[3] = (byte) (size >> 24);
          payloadAttr = (PayloadAttribute)addAttribute(PayloadAttribute.class);
          payloadAttr.setPayload(new Payload(buffer));
          termAttr = (TermAttribute)addAttribute(TermAttribute.class);
          termAttr.setTermBuffer(term.text());
          returnToken = true;
        }
View Full Code Here

Examples of org.apache.lucene.index.Payload

      data[--off] = (byte)(id);
      id >>>= 8;
    }
    while(id > 0);
   
    return new Payload(data, off, data.length - off);
  }
View Full Code Here

Examples of org.apache.lucene.index.Payload

      buf[i++] = (byte)(datum >>> 8);
      buf[i++] = (byte)(datum >>> 16);
      buf[i++] = (byte)(datum >>> 24);
    }
   
    _payload = new Payload(buf);
    _returnToken = true;
  }
View Full Code Here

Examples of org.apache.lucene.index.Payload

  private TypeAttribute typeAtt;

  public NumericPayloadTokenFilter(TokenStream input, float payload, String typeMatch) {
    super(input);
    //Need to encode the payload
    thePayload = new Payload(PayloadHelper.encodeFloat(payload));
    this.typeMatch = typeMatch;
    payloadAtt = addAttribute(PayloadAttribute.class);
    typeAtt = addAttribute(TypeAttribute.class);
  }
View Full Code Here

Examples of org.apache.lucene.index.Payload

  public final boolean incrementToken() throws IOException {
    if (input.incrementToken()) {
      byte[] data = new byte[8];
      PayloadHelper.encodeInt(offsetAtt.startOffset(), data, 0);
      PayloadHelper.encodeInt(offsetAtt.endOffset(), data, 4);
      Payload payload = new Payload(data);
      payAtt.setPayload(payload);
      return true;
    } else {
    return false;
    }
View Full Code Here

Examples of org.apache.lucene.index.Payload

  @Override
  public final boolean incrementToken() throws IOException {
    if (input.incrementToken()) {
      String type = typeAtt.type();
      if (type != null && type.equals("") == false) {
        payloadAtt.setPayload(new Payload(type.getBytes("UTF-8")));
      }
      return true;
    } else {
      return false;
    }
View Full Code Here

Examples of org.apache.lucene.index.Payload

    @Override
    public void setWeight(Token token, float weight) {
      if (weight == 1f) {
        token.setPayload(null);
      } else {
        token.setPayload(new Payload(PayloadHelper.encodeFloat(weight)));
      }
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.