Package org.apache.lucene.store

Examples of org.apache.lucene.store.DataInput


    return true;
  }

  @Override
  public boolean load(InputStream input) throws IOException {
    DataInput in = new InputStreamDataInput(input);
    CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_START);
    byte separatorOrig = in.readByte();
    if (separatorOrig != separator) {
      throw new IllegalStateException("separator=" + separator + " is incorrect: original model was built with separator=" + separatorOrig);
    }
    int gramsOrig = in.readVInt();
    if (gramsOrig != grams) {
      throw new IllegalStateException("grams=" + grams + " is incorrect: original model was built with grams=" + gramsOrig);
    }
    totTokens = in.readVLong();

    fst = new FST<Long>(in, PositiveIntOutputs.getSingleton());

    return true;
  }
View Full Code Here


    final boolean isFirstTerm = termState.termBlockOrd == 0;
    final boolean fieldHasPositions = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
    final boolean fieldHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
    final boolean fieldHasPayloads = fieldInfo.hasPayloads();

    final DataInput in = termState.bytesReader;
    if (isFirstTerm) {
      if (termState.docFreq == 1) {
        termState.singletonDocID = in.readVInt();
        termState.docStartFP = 0;
      } else {
        termState.singletonDocID = -1;
        termState.docStartFP = in.readVLong();
      }
      if (fieldHasPositions) {
        termState.posStartFP = in.readVLong();
        if (termState.totalTermFreq > BLOCK_SIZE) {
          termState.lastPosBlockOffset = in.readVLong();
        } else {
          termState.lastPosBlockOffset = -1;
        }
        if ((fieldHasPayloads || fieldHasOffsets) && termState.totalTermFreq >= BLOCK_SIZE) {
          termState.payStartFP = in.readVLong();
        } else {
          termState.payStartFP = -1;
        }
      }
    } else {
      if (termState.docFreq == 1) {
        termState.singletonDocID = in.readVInt();
      } else {
        termState.singletonDocID = -1;
        termState.docStartFP += in.readVLong();
      }
      if (fieldHasPositions) {
        termState.posStartFP += in.readVLong();
        if (termState.totalTermFreq > BLOCK_SIZE) {
          termState.lastPosBlockOffset = in.readVLong();
        } else {
          termState.lastPosBlockOffset = -1;
        }
        if ((fieldHasPayloads || fieldHasOffsets) && termState.totalTermFreq >= BLOCK_SIZE) {
          long delta = in.readVLong();
          if (termState.payStartFP == -1) {
            termState.payStartFP = delta;
          } else {
            termState.payStartFP += delta;
          }
        }
      }
    }

    if (termState.docFreq > BLOCK_SIZE) {
      termState.skipOffset = in.readVLong();
    } else {
      termState.skipOffset = -1;
    }
  }
View Full Code Here

        }
      }
     
      out.close();
      final IndexInput input = dir.openInput("foo", IOContext.DEFAULT);
      final DataInput in = input.clone();
     
      p.copy(input, input.length());
      final PagedBytes.Reader reader = p.freeze(random.nextBoolean());

      final byte[] verify = new byte[numBytes];
      int read = 0;
      while(read < numBytes) {
        if (random().nextInt(10) == 7) {
          verify[read++] = in.readByte();
        } else {
          int chunk = Math.min(random().nextInt(1000), numBytes - read);
          in.readBytes(verify, read, chunk);
          read += chunk;
        }
      }
      assertTrue(Arrays.equals(answer, verify));
View Full Code Here

        }
      }

      final PagedBytes.Reader reader = p.freeze(random.nextBoolean());

      final DataInput in = p.getDataInput();

      final byte[] verify = new byte[numBytes];
      int read = 0;
      while(read < numBytes) {
        if (random().nextInt(10) == 7) {
          verify[read++] = in.readByte();
        } else {
          int chunk = Math.min(random().nextInt(1000), numBytes - read);
          in.readBytes(verify, read, chunk);
          read += chunk;
        }
      }
      assertTrue(Arrays.equals(answer, verify));
View Full Code Here

    return true;
  }

  @Override
  public boolean load(InputStream input) throws IOException {
    DataInput dataIn = new InputStreamDataInput(input);
    try {
      this.fst = new FST<Pair<Long,BytesRef>>(dataIn, new PairOutputs<Long,BytesRef>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()));
      maxAnalyzedPathsForOneInput = dataIn.readVInt();
      hasPayloads = dataIn.readByte() == 1;
    } finally {
      IOUtils.close(input);
    }
    return true;
  }
View Full Code Here

    if (numStoredFields == 0) {
      // nothing to do
      return;
    }

    final DataInput documentInput;
    if (version >= VERSION_BIG_CHUNKS && totalLength >= 2 * chunkSize) {
      assert chunkSize > 0;
      assert offset < chunkSize;

      decompressor.decompress(fieldsStream, chunkSize, offset, Math.min(length, chunkSize - offset), bytes);
      documentInput = new DataInput() {

        int decompressed = bytes.length;

        void fillBuffer() throws IOException {
          assert decompressed <= length;
          if (decompressed == length) {
            throw new EOFException();
          }
          final int toDecompress = Math.min(length - decompressed, chunkSize);
          decompressor.decompress(fieldsStream, toDecompress, 0, toDecompress, bytes);
          decompressed += toDecompress;
        }

        @Override
        public byte readByte() throws IOException {
          if (bytes.length == 0) {
            fillBuffer();
          }
          --bytes.length;
          return bytes.bytes[bytes.offset++];
        }

        @Override
        public void readBytes(byte[] b, int offset, int len) throws IOException {
          while (len > bytes.length) {
            System.arraycopy(bytes.bytes, bytes.offset, b, offset, bytes.length);
            len -= bytes.length;
            offset += bytes.length;
            fillBuffer();
          }
          System.arraycopy(bytes.bytes, bytes.offset, b, offset, len);
          bytes.offset += len;
          bytes.length -= len;
        }

      };
    } else {
      final BytesRef bytes = totalLength <= BUFFER_REUSE_THRESHOLD ? this.bytes : new BytesRef();
      decompressor.decompress(fieldsStream, totalLength, offset, length, bytes);
      assert bytes.length == length;
      documentInput = new ByteArrayDataInput(bytes.bytes, bytes.offset, bytes.length);
    }

    for (int fieldIDX = 0; fieldIDX < numStoredFields; fieldIDX++) {
      final long infoAndBits = documentInput.readVLong();
      final int fieldNumber = (int) (infoAndBits >>> TYPE_BITS);
      final FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber);

      final int bits = (int) (infoAndBits & TYPE_MASK);
      assert bits <= NUMERIC_DOUBLE: "bits=" + Integer.toHexString(bits);
View Full Code Here

      IndexInput in1 = dir.openInput("out.bin", IOContext.DEFAULT);
      byte[] buf = new byte[(int) fp];
      in1.readBytes(buf, 0, (int) fp);
      in1.seek(0L);
      ByteArrayDataInput in2 = new ByteArrayDataInput(buf);
      final DataInput in = random().nextBoolean() ? in1 : in2;
      final BlockPackedReaderIterator it = new BlockPackedReaderIterator(in, PackedInts.VERSION_CURRENT, blockSize, valueCount);
      for (int i = 0; i < valueCount; ) {
        if (random().nextBoolean()) {
          assertEquals("" + i, values[i], it.next());
          ++i;
View Full Code Here

  /**
   * Calls {@link #load(DataInput)} after converting
   * {@link InputStream} to {@link DataInput}
   */
  public boolean load(InputStream input) throws IOException {
    DataInput dataIn = new InputStreamDataInput(input);
    try {
      return load(dataIn);
    } finally {
      IOUtils.close(input);
    }
View Full Code Here

    return true;
  }

  @Override
  public boolean load(InputStream input) throws IOException {
    DataInput dataIn = new InputStreamDataInput(input);
    try {
      this.fst = new FST<Pair<Long,BytesRef>>(dataIn, new PairOutputs<Long,BytesRef>(PositiveIntOutputs.getSingleton(true), ByteSequenceOutputs.getSingleton()));
      maxAnalyzedPathsForOneInput = dataIn.readVInt();
    } finally {
      IOUtils.close(input);
    }
    return true;
  }
View Full Code Here

    final boolean isFirstTerm = termState.termBlockOrd == 0;
    final boolean fieldHasPositions = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
    final boolean fieldHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
    final boolean fieldHasPayloads = fieldInfo.hasPayloads();

    final DataInput in = termState.bytesReader;
    if (isFirstTerm) {
      if (termState.docFreq == 1) {
        termState.singletonDocID = in.readVInt();
        termState.docStartFP = 0;
      } else {
        termState.singletonDocID = -1;
        termState.docStartFP = in.readVLong();
      }
      if (fieldHasPositions) {
        termState.posStartFP = in.readVLong();
        if (termState.totalTermFreq > BLOCK_SIZE) {
          termState.lastPosBlockOffset = in.readVLong();
        } else {
          termState.lastPosBlockOffset = -1;
        }
        if ((fieldHasPayloads || fieldHasOffsets) && termState.totalTermFreq >= BLOCK_SIZE) {
          termState.payStartFP = in.readVLong();
        } else {
          termState.payStartFP = -1;
        }
      }
    } else {
      if (termState.docFreq == 1) {
        termState.singletonDocID = in.readVInt();
      } else {
        termState.singletonDocID = -1;
        termState.docStartFP += in.readVLong();
      }
      if (fieldHasPositions) {
        termState.posStartFP += in.readVLong();
        if (termState.totalTermFreq > BLOCK_SIZE) {
          termState.lastPosBlockOffset = in.readVLong();
        } else {
          termState.lastPosBlockOffset = -1;
        }
        if ((fieldHasPayloads || fieldHasOffsets) && termState.totalTermFreq >= BLOCK_SIZE) {
          long delta = in.readVLong();
          if (termState.payStartFP == -1) {
            termState.payStartFP = delta;
          } else {
            termState.payStartFP += delta;
          }
        }
      }
    }

    if (termState.docFreq > BLOCK_SIZE) {
      termState.skipOffset = in.readVLong();
    } else {
      termState.skipOffset = -1;
    }
  }
View Full Code Here

TOP

Related Classes of org.apache.lucene.store.DataInput

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.