/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.action.termvector;
import com.carrotsearch.hppc.ObjectLongOpenHashMap;
import com.carrotsearch.hppc.cursors.ObjectLongCursor;
import org.apache.lucene.index.*;
import org.apache.lucene.util.*;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.BytesStreamInput;
import java.io.IOException;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Iterator;
import static org.apache.lucene.util.ArrayUtil.grow;
/**
* This class represents the result of a {@link TermVectorRequest}. It works
* exactly like the {@link Fields} class except for one thing: It can return
* offsets and payloads even if positions are not present. You must call
* nextPosition() anyway to move the counter although this method only returns
* <tt>-1,</tt>, if no positions were returned by the {@link TermVectorRequest}.
* <p/>
* The data is stored in two byte arrays ({@code headerRef} and
* {@code termVectors}, both {@link ByteRef}) that have the following format:
* <p/>
* {@code headerRef}: Stores offsets per field in the {@code termVectors} array
* and some header information as {@link BytesRef}. Format is
* <ul>
* <li>String : "TV"</li>
* <li>vint: version (=-1)</li>
* <li>boolean: hasTermStatistics (are the term statistics stored?)</li>
* <li>boolean: hasFieldStatitsics (are the field statistics stored?)</li>
* <li>vint: number of fields</li>
* <ul>
* <li>String: field name 1</li>
* <li>vint: offset in {@code termVectors} for field 1</li>
* <li>...</li>
* <li>String: field name last field</li>
* <li>vint: offset in {@code termVectors} for last field</li>
* </ul>
* </ul>
* <p/>
* termVectors: Stores the actual term vectors as a {@link BytesRef}.
* <p/>
* Term vectors for each fields are stored in blocks, one for each field. The
* offsets in {@code headerRef} are used to find where the block for a field
* starts. Each block begins with a
* <ul>
* <li>vint: number of terms</li>
* <li>boolean: positions (has it positions stored?)</li>
* <li>boolean: offsets (has it offsets stored?)</li>
* <li>boolean: payloads (has it payloads stored?)</li>
* </ul>
* If the field statistics were requested ({@code hasFieldStatistics} is true,
* see {@code headerRef}), the following numbers are stored:
* <ul>
* <li>vlong: sum of total term freqencies of the field (sumTotalTermFreq)</li>
* <li>vlong: sum of document frequencies for each term (sumDocFreq)</li>
* <li>vint: number of documents in the shard that has an entry for this field
* (docCount)</li>
* </ul>
* <p/>
* After that, for each term it stores
* <ul>
* <ul>
* <li>vint: term lengths</li>
* <li>BytesRef: term name</li>
* </ul>
* <p/>
* If term statistics are requested ({@code hasTermStatistics} is true, see
* {@code headerRef}):
* <ul>
* <li>vint: document frequency, how often does this term appear in documents?</li>
* <li>vlong: total term frequency. Sum of terms in this field.</li>
* </ul>
* After that
* <ul>
* <li>vint: frequency (always returned)</li>
* <ul>
* <li>vint: position_1 (if positions == true)</li>
* <li>vint: startOffset_1 (if offset == true)</li>
* <li>vint: endOffset_1 (if offset == true)</li>
* <li>BytesRef: payload_1 (if payloads == true)</li>
* <li>...</li>
* <li>vint: endOffset_freqency (if offset == true)</li>
* <li>BytesRef: payload_freqency (if payloads == true)</li>
* <ul>
* </ul> </ul>
*/
public final class TermVectorFields extends Fields {
private final ObjectLongOpenHashMap<String> fieldMap;
private final BytesReference termVectors;
final boolean hasTermStatistic;
final boolean hasFieldStatistic;
/**
* @param headerRef Stores offsets per field in the {@code termVectors} and some
* header information as {@link BytesRef}.
* @param termVectors Stores the actual term vectors as a {@link BytesRef}.
*/
public TermVectorFields(BytesReference headerRef, BytesReference termVectors) throws IOException {
BytesStreamInput header = new BytesStreamInput(headerRef);
fieldMap = new ObjectLongOpenHashMap<>();
// here we read the header to fill the field offset map
String headerString = header.readString();
assert headerString.equals("TV");
int version = header.readInt();
assert version == -1;
hasTermStatistic = header.readBoolean();
hasFieldStatistic = header.readBoolean();
final int numFields = header.readVInt();
for (int i = 0; i < numFields; i++) {
fieldMap.put((header.readString()), header.readVLong());
}
header.close();
// reference to the term vector data
this.termVectors = termVectors;
}
@Override
public Iterator<String> iterator() {
final Iterator<ObjectLongCursor<String>> iterator = fieldMap.iterator();
return new Iterator<String>() {
@Override
public boolean hasNext() {
return iterator.hasNext();
}
@Override
public String next() {
return iterator.next().key;
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
};
}
@Override
public Terms terms(String field) throws IOException {
// first, find where in the termVectors bytes the actual term vector for
// this field is stored
if (!fieldMap.containsKey(field)) {
return null; // we don't have it.
}
long readOffset = fieldMap.lget();
return new TermVector(termVectors, readOffset);
}
@Override
public int size() {
return fieldMap.size();
}
private final class TermVector extends Terms {
private final BytesStreamInput perFieldTermVectorInput;
private final long readOffset;
private long numTerms;
private boolean hasPositions;
private boolean hasOffsets;
private boolean hasPayloads;
private long sumTotalTermFreq;
private long sumDocFreq;
private int docCount;
public TermVector(BytesReference termVectors, long readOffset) throws IOException {
this.perFieldTermVectorInput = new BytesStreamInput(termVectors);
this.readOffset = readOffset;
reset();
}
private void reset() throws IOException {
this.perFieldTermVectorInput.reset();
this.perFieldTermVectorInput.skip(readOffset);
// read how many terms....
this.numTerms = perFieldTermVectorInput.readVLong();
// ...if positions etc. were stored....
this.hasPositions = perFieldTermVectorInput.readBoolean();
this.hasOffsets = perFieldTermVectorInput.readBoolean();
this.hasPayloads = perFieldTermVectorInput.readBoolean();
// read the field statistics
this.sumTotalTermFreq = hasFieldStatistic ? readPotentiallyNegativeVLong(perFieldTermVectorInput) : -1;
this.sumDocFreq = hasFieldStatistic ? readPotentiallyNegativeVLong(perFieldTermVectorInput) : -1;
this.docCount = hasFieldStatistic ? readPotentiallyNegativeVInt(perFieldTermVectorInput) : -1;
}
@Override
public TermsEnum iterator(TermsEnum reuse) throws IOException {
// reset before asking for an iterator
reset();
// convert bytes ref for the terms to actual data
return new TermsEnum() {
int currentTerm = 0;
int freq = 0;
int docFreq = -1;
long totalTermFrequency = -1;
int[] positions = new int[1];
int[] startOffsets = new int[1];
int[] endOffsets = new int[1];
BytesRefBuilder[] payloads = new BytesRefBuilder[1];
final BytesRefBuilder spare = new BytesRefBuilder();
@Override
public BytesRef next() throws IOException {
if (currentTerm++ < numTerms) {
// term string. first the size...
int termVectorSize = perFieldTermVectorInput.readVInt();
spare.grow(termVectorSize);
// ...then the value.
perFieldTermVectorInput.readBytes(spare.bytes(), 0, termVectorSize);
spare.setLength(termVectorSize);
if (hasTermStatistic) {
docFreq = readPotentiallyNegativeVInt(perFieldTermVectorInput);
totalTermFrequency = readPotentiallyNegativeVLong(perFieldTermVectorInput);
}
freq = readPotentiallyNegativeVInt(perFieldTermVectorInput);
// grow the arrays to read the values. this is just
// for performance reasons. Re-use memory instead of
// realloc.
growBuffers();
// finally, read the values into the arrays
// currentPosition etc. so that we can just iterate
// later
writeInfos(perFieldTermVectorInput);
return spare.get();
} else {
return null;
}
}
private void writeInfos(final BytesStreamInput input) throws IOException {
for (int i = 0; i < freq; i++) {
if (hasPositions) {
positions[i] = input.readVInt();
}
if (hasOffsets) {
startOffsets[i] = input.readVInt();
endOffsets[i] = input.readVInt();
}
if (hasPayloads) {
int payloadLength = input.readVInt();
if (payloads[i] == null) {
payloads[i] = new BytesRefBuilder();
}
payloads[i].grow(payloadLength);
input.readBytes(payloads[i].bytes(), 0, payloadLength);
payloads[i].setLength(payloadLength);
}
}
}
private void growBuffers() {
if (hasPositions) {
positions = grow(positions, freq);
}
if (hasOffsets) {
startOffsets = grow(startOffsets, freq);
endOffsets = grow(endOffsets, freq);
}
if (hasPayloads) {
if (payloads.length < freq) {
payloads = Arrays.copyOf(payloads, ArrayUtil.oversize(freq, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
}
}
}
@Override
public SeekStatus seekCeil(BytesRef text) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public void seekExact(long ord) throws IOException {
throw new UnsupportedOperationException("Seek is not supported");
}
@Override
public BytesRef term() throws IOException {
return spare.get();
}
@Override
public long ord() throws IOException {
throw new UnsupportedOperationException("ordinals are not supported");
}
@Override
public int docFreq() throws IOException {
return docFreq;
}
@Override
public long totalTermFreq() throws IOException {
return totalTermFrequency;
}
@Override
public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
return docsAndPositions(liveDocs, reuse instanceof DocsAndPositionsEnum ? (DocsAndPositionsEnum) reuse : null, 0);
}
@Override
public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException {
final TermVectorDocsAndPosEnum retVal = (reuse instanceof TermVectorDocsAndPosEnum ? (TermVectorDocsAndPosEnum) reuse
: new TermVectorDocsAndPosEnum());
return retVal.reset(hasPositions ? positions : null, hasOffsets ? startOffsets : null, hasOffsets ? endOffsets
: null, hasPayloads ? payloads : null, freq);
}
};
}
@Override
public long size() throws IOException {
return numTerms;
}
@Override
public long getSumTotalTermFreq() throws IOException {
return sumTotalTermFreq;
}
@Override
public long getSumDocFreq() throws IOException {
return sumDocFreq;
}
@Override
public int getDocCount() throws IOException {
return docCount;
}
@Override
public boolean hasFreqs() {
return true;
}
@Override
public boolean hasOffsets() {
return hasOffsets;
}
@Override
public boolean hasPositions() {
return hasPositions;
}
@Override
public boolean hasPayloads() {
return hasPayloads;
}
}
private final class TermVectorDocsAndPosEnum extends DocsAndPositionsEnum {
private boolean hasPositions;
private boolean hasOffsets;
private boolean hasPayloads;
int curPos = -1;
int doc = -1;
private int freq;
private int[] startOffsets;
private int[] positions;
private BytesRefBuilder[] payloads;
private int[] endOffsets;
private DocsAndPositionsEnum reset(int[] positions, int[] startOffsets, int[] endOffsets, BytesRefBuilder[] payloads, int freq) {
curPos = -1;
doc = -1;
this.hasPositions = positions != null;
this.hasOffsets = startOffsets != null;
this.hasPayloads = payloads != null;
this.freq = freq;
this.startOffsets = startOffsets;
this.endOffsets = endOffsets;
this.payloads = payloads;
this.positions = positions;
return this;
}
@Override
public int nextDoc() throws IOException {
return doc = (doc == -1 ? 0 : NO_MORE_DOCS);
}
@Override
public int docID() {
return doc;
}
@Override
public int advance(int target) throws IOException {
while (nextDoc() < target && doc != NO_MORE_DOCS) {
}
return doc;
}
@Override
public int freq() throws IOException {
return freq;
}
// call nextPosition once before calling this one
// because else counter is not advanced
@Override
public int startOffset() throws IOException {
assert curPos < freq && curPos >= 0;
return hasOffsets ? startOffsets[curPos] : -1;
}
@Override
// can return -1 if posistions were not requested or
// stored but offsets were stored and requested
public int nextPosition() throws IOException {
assert curPos + 1 < freq;
++curPos;
// this is kind of cheating but if you don't need positions
// we safe lots fo space on the wire
return hasPositions ? positions[curPos] : -1;
}
@Override
public BytesRef getPayload() throws IOException {
assert curPos < freq && curPos >= 0;
if (hasPayloads) {
final BytesRefBuilder payload = payloads[curPos];
if (payload != null) {
return payload.get();
}
}
return null;
}
@Override
public int endOffset() throws IOException {
assert curPos < freq && curPos >= 0;
return hasOffsets ? endOffsets[curPos] : -1;
}
@Override
public long cost() {
return 1;
}
}
// read a vInt. this is used if the integer might be negative. In this case,
// the writer writes a 0 for -1 or value +1 and accordingly we have to
// substract 1 again
// adds one to mock not existing term freq
int readPotentiallyNegativeVInt(BytesStreamInput stream) throws IOException {
return stream.readVInt() - 1;
}
// read a vLong. this is used if the integer might be negative. In this
// case, the writer writes a 0 for -1 or value +1 and accordingly we have to
// substract 1 again
// adds one to mock not existing term freq
long readPotentiallyNegativeVLong(BytesStreamInput stream) throws IOException {
return stream.readVLong() - 1;
}
}