private SSTableReader streamIn(DataInput input, PendingFile localFile, PendingFile remoteFile) throws IOException
{
ColumnFamilyStore cfs = Table.open(localFile.desc.ksname).getColumnFamilyStore(localFile.desc.cfname);
DecoratedKey key;
SSTableWriter writer = new SSTableWriter(localFile.getFilename(), remoteFile.estimatedKeys);
CompactionController controller = new CompactionController(cfs, Collections.<SSTableReader>emptyList(), Integer.MIN_VALUE);
try
{
BytesReadTracker in = new BytesReadTracker(input);
long totalBytesRead = 0;
for (Pair<Long, Long> section : localFile.sections)
{
long length = section.right - section.left;
// skip to beginning of section inside chunk
if (remoteFile.compressionInfo != null)
((CompressedInputStream) underliningStream).position(section.left);
long bytesRead = 0;
while (bytesRead < length)
{
in.reset(0);
key = SSTableReader.decodeKey(StorageService.getPartitioner(), localFile.desc, ByteBufferUtil.readWithShortLength(in));
long dataSize = SSTableReader.readRowSize(in, localFile.desc);
if (cfs.containsCachedRow(key) && remoteFile.type == OperationType.AES && dataSize <= DatabaseDescriptor.getInMemoryCompactionLimit())
{
// need to update row cache
// Note: Because we won't just echo the columns, there is no need to use the PRESERVE_SIZE flag, contrarily to what appendFromStream does below
SSTableIdentityIterator iter = new SSTableIdentityIterator(cfs.metadata, in, localFile.getFilename(), key, 0, dataSize, IColumnSerializer.Flag.FROM_REMOTE);
PrecompactedRow row = new PrecompactedRow(controller, Collections.singletonList(iter));
// We don't expire anything so the row shouldn't be empty
assert !row.isEmpty();
writer.append(row);
// update cache
ColumnFamily cf = row.getFullColumnFamily();
cfs.maybeUpdateRowCache(key, cf);
}
else
{
writer.appendFromStream(key, cfs.metadata, dataSize, in);
cfs.invalidateCachedRow(key);
}
bytesRead += in.getBytesRead();
// when compressed, report total bytes of compressed chunks read since remoteFile.size is the sum of chunks transferred
if (remoteFile.compressionInfo != null)
remoteFile.progress = ((CompressedInputStream) underliningStream).getTotalCompressedBytesRead();
else
remoteFile.progress += in.getBytesRead();
totalBytesRead += in.getBytesRead();
}
}
StreamingMetrics.totalIncomingBytes.inc(totalBytesRead);
metrics.incomingBytes.inc(totalBytesRead);
return writer.closeAndOpenReader();
}
catch (Throwable e)
{
writer.abort();
if (e instanceof IOException)
throw (IOException) e;
else
throw Throwables.propagate(e);
}
finally
{
controller.close();
}
}