long bytes = totalPostings * 8 + totalPayloadBytes;
SegmentWriteState writeState = new SegmentWriteState(null, dir,
segmentInfo, newFieldInfos,
32, null, new IOContext(new FlushInfo(maxDoc, bytes)));
FieldsConsumer fieldsConsumer = codec.postingsFormat().fieldsConsumer(writeState);
for(Map.Entry<String,Map<BytesRef,Long>> fieldEnt : fields.entrySet()) {
String field = fieldEnt.getKey();
Map<BytesRef,Long> terms = fieldEnt.getValue();
FieldInfo fieldInfo = newFieldInfos.fieldInfo(field);
IndexOptions indexOptions = fieldInfo.getIndexOptions();
if (VERBOSE) {
System.out.println("field=" + field + " indexOtions=" + indexOptions);
}
boolean doFreq = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
boolean doPos = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
boolean doPayloads = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 && allowPayloads;
boolean doOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
TermsConsumer termsConsumer = fieldsConsumer.addField(fieldInfo);
long sumTotalTF = 0;
long sumDF = 0;
FixedBitSet seenDocs = new FixedBitSet(maxDoc);
for(Map.Entry<BytesRef,Long> termEnt : terms.entrySet()) {
BytesRef term = termEnt.getKey();
SeedPostings postings = getSeedPostings(term.utf8ToString(), termEnt.getValue(), false, maxAllowed);
if (VERBOSE) {
System.out.println(" term=" + field + ":" + term.utf8ToString() + " docFreq=" + postings.docFreq + " seed=" + termEnt.getValue());
}
PostingsConsumer postingsConsumer = termsConsumer.startTerm(term);
long totalTF = 0;
int docID = 0;
while((docID = postings.nextDoc()) != DocsEnum.NO_MORE_DOCS) {
final int freq = postings.freq();
if (VERBOSE) {
System.out.println(" " + postings.upto + ": docID=" + docID + " freq=" + postings.freq);
}
postingsConsumer.startDoc(docID, doFreq ? postings.freq : -1);
seenDocs.set(docID);
if (doPos) {
totalTF += postings.freq;
for(int posUpto=0;posUpto<freq;posUpto++) {
int pos = postings.nextPosition();
BytesRef payload = postings.getPayload();
if (VERBOSE) {
if (doPayloads) {
System.out.println(" pos=" + pos + " payload=" + (payload == null ? "null" : payload.length + " bytes"));
} else {
System.out.println(" pos=" + pos);
}
}
postingsConsumer.addPosition(pos, doPayloads ? payload : null,
doOffsets ? postings.startOffset() : -1,
doOffsets ? postings.endOffset() : -1);
}
} else if (doFreq) {
totalTF += freq;
} else {
totalTF++;
}
postingsConsumer.finishDoc();
}
termsConsumer.finishTerm(term, new TermStats(postings.docFreq, doFreq ? totalTF : -1));
sumTotalTF += totalTF;
sumDF += postings.docFreq;
}
termsConsumer.finish(doFreq ? sumTotalTF : -1, sumDF, seenDocs.cardinality());
}
fieldsConsumer.close();
if (VERBOSE) {
System.out.println("TEST: after indexing: files=");
for(String file : dir.listAll()) {
System.out.println(" " + file + ": " + dir.fileLength(file) + " bytes");