Package org.apache.lucene.store

Examples of org.apache.lucene.store.ChecksumIndexInput

105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
    this.compressionMode = compressionMode;
    final String segment = si.name;
    boolean success = false;
    fieldInfos = fn;
    numDocs = si.getDocCount();
    ChecksumIndexInput indexStream = null;
    try {
      final String indexStreamFN = IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_INDEX_EXTENSION);
      final String fieldsStreamFN = IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION);
      // Load the index into memory
      indexStream = d.openChecksumInput(indexStreamFN, context);
      final String codecNameIdx = formatName + CODEC_SFX_IDX;
      version = CodecUtil.checkHeader(indexStream, codecNameIdx, VERSION_START, VERSION_CURRENT);
      assert CodecUtil.headerLength(codecNameIdx) == indexStream.getFilePointer();
      indexReader = new CompressingStoredFieldsIndexReader(indexStream, si);

      long maxPointer = -1;
     
      if (version >= VERSION_CHECKSUM) {
        maxPointer = indexStream.readVLong();
        CodecUtil.checkFooter(indexStream);
      } else {
        CodecUtil.checkEOF(indexStream);
      }
      indexStream.close();
      indexStream = null;

      // Open the data file and read metadata
      fieldsStream = d.openInput(fieldsStreamFN, context);
      if (version >= VERSION_CHECKSUM) {
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
   
  Lucene49NormsProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
    maxDoc = state.segmentInfo.getDocCount();
    String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
    // read in the entries from the metadata file.
    ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context);
    boolean success = false;
    ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass()));
    try {
      version = CodecUtil.checkHeader(in, metaCodec, VERSION_START, VERSION_CURRENT);
      readFields(in, state.fieldInfos);
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
 
  /** expert: instantiates a new reader */
  Lucene49DocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
    String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
    // read in the entries from the metadata file.
    ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context);
    this.maxDoc = state.segmentInfo.getDocCount();
    boolean success = false;
    try {
      version = CodecUtil.checkHeader(in, metaCodec,
                                      Lucene49DocValuesFormat.VERSION_START,
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
      ver = null;
    }
    lenientFieldInfoCheck = ver == null || !ver.onOrAfter(Version.LUCENE_4_9);
    String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
    // read in the entries from the metadata file.
    ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context);
    this.maxDoc = state.segmentInfo.getDocCount();
    boolean success = false;
    try {
      version = CodecUtil.checkHeader(in, metaCodec,
                                      Lucene45DocValuesFormat.VERSION_START,
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
   
  MemoryDocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
    maxDoc = state.segmentInfo.getDocCount();
    String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
    // read in the entries from the metadata file.
    ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context);
    boolean success = false;
    try {
      version = CodecUtil.checkHeader(in, metaCodec,
                                      VERSION_START,
                                      VERSION_CURRENT);
View Full Code Here
513
514
515
516
517
518
519
520
521
522
523
  @Override
  public void checkIntegrity() throws IOException {
    BytesRef scratch = new BytesRef();
    IndexInput clone = data.clone();
    clone.seek(0);
    ChecksumIndexInput input = new BufferedChecksumIndexInput(clone);
    while(true) {
      SimpleTextUtil.readLine(input, scratch);
      if (scratch.equals(END)) {
        SimpleTextUtil.checkFooter(input);
        break;
View Full Code Here
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
  }

  @Override
  public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
    final String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
    final ChecksumIndexInput in = state.directory.openChecksumInput(fileName, IOContext.READONCE);

    final SortedMap<String,TermsReader> fields = new TreeMap<>();

    try {
      CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_CURRENT);
      while(true) {
        final int termCount = in.readVInt();
        if (termCount == 0) {
          break;
        }
        final TermsReader termsReader = new TermsReader(state.fieldInfos, in, termCount);
        // System.out.println("load field=" + termsReader.field.name);
        fields.put(termsReader.field.name, termsReader);
      }
      CodecUtil.checkFooter(in);
    } finally {
      in.close();
    }

    return new FieldsProducer() {
      @Override
      public Iterator<String> iterator() {
View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
  public FSTOrdTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) throws IOException {
    final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, FSTOrdTermsWriter.TERMS_INDEX_EXTENSION);
    final String termsBlockFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, FSTOrdTermsWriter.TERMS_BLOCK_EXTENSION);

    this.postingsReader = postingsReader;
    ChecksumIndexInput indexIn = null;
    IndexInput blockIn = null;
    boolean success = false;
    try {
      indexIn = state.directory.openChecksumInput(termsIndexFileName, state.context);
      blockIn = state.directory.openInput(termsBlockFileName, state.context);
View Full Code Here
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
      }
    }
  }
 
  private TreeMap<String,Long> readFields(IndexInput in) throws IOException {
    ChecksumIndexInput input = new BufferedChecksumIndexInput(in);
    BytesRef scratch = new BytesRef(10);
    TreeMap<String,Long> fields = new TreeMap<>();
   
    while (true) {
      SimpleTextUtil.readLine(input, scratch);
      if (scratch.equals(END)) {
        SimpleTextUtil.checkFooter(input);
        return fields;
      } else if (StringHelper.startsWith(scratch, FIELD)) {
        String fieldName = new String(scratch.bytes, scratch.offset + FIELD.length, scratch.length - FIELD.length, StandardCharsets.UTF_8);
        fields.put(fieldName, input.getFilePointer());
      }
    }
  }
View Full Code Here
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
    public BloomFilteredFieldsProducer(SegmentReadState state)
        throws IOException {
     
      String bloomFileName = IndexFileNames.segmentFileName(
          state.segmentInfo.name, state.segmentSuffix, BLOOM_EXTENSION);
      ChecksumIndexInput bloomIn = null;
      boolean success = false;
      try {
        bloomIn = state.directory.openChecksumInput(bloomFileName, state.context);
        int version = CodecUtil.checkHeader(bloomIn, BLOOM_CODEC_NAME, VERSION_START, VERSION_CURRENT);
        // // Load the hash function used in the BloomFilter
        // hashFunction = HashFunction.forName(bloomIn.readString());
        // Load the delegate postings format
        PostingsFormat delegatePostingsFormat = PostingsFormat.forName(bloomIn
            .readString());
       
        this.delegateFieldsProducer = delegatePostingsFormat
            .fieldsProducer(state);
        int numBlooms = bloomIn.readInt();
        for (int i = 0; i < numBlooms; i++) {
          int fieldNum = bloomIn.readInt();
          FuzzySet bloom = FuzzySet.deserialize(bloomIn);
          FieldInfo fieldInfo = state.fieldInfos.fieldInfo(fieldNum);
          bloomsByFieldName.put(fieldInfo.name, bloom);
        }
        if (version >= VERSION_CHECKSUM) {
View Full Code Here
TOP

Related Classes of org.apache.lucene.store.ChecksumIndexInput

  • org.apache.lucene.codecs.bloom.BloomFilteringPostingsFormat$BloomFilteredFieldsProducer
  • org.apache.lucene.codecs.CodecUtil
  • org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader
  • org.apache.lucene.codecs.compressing.CompressingTermVectorsReader
  • org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter
  • org.apache.lucene.codecs.lucene40.BitVector
  • org.apache.lucene.codecs.lucene42.Lucene42DocValuesProducer
  • org.apache.lucene.codecs.lucene45.Lucene45DocValuesProducer
  • org.apache.lucene.codecs.lucene46.Lucene46FieldInfosReader
  • org.apache.lucene.codecs.lucene46.Lucene46SegmentInfoReader
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.