int tiny_block = Parameter.intValue("tiny-size", 15000);
String fileMatch = Parameter.value("file-match", "out-*");
// Stats to report
long blocks = 0;
long chunks = 0;
Histogram chunkSize = Metrics.newHistogram(ReadMuxStreamDirectory.class, "chunkSize");
Histogram blockSize = Metrics.newHistogram(ReadMuxStreamDirectory.class, "blockSize");
Histogram chunksPerBlock = Metrics.newHistogram(ReadMuxStreamDirectory.class, "chunksPerBlock");
// Get stats
int currentFile = 1;
Iterator<Path> dataFiles = Files.newDirectoryStream(streamDirectory, fileMatch).iterator();
Path lastPath = dataFiles.next();
long nextBlockPosition = 0;
RandomAccessFile input = new RandomAccessFile(lastPath.toFile(), "r");
while (true) {
if (nextBlockPosition != 0) {
input.seek(nextBlockPosition);
}
if (input.getFilePointer() >= input.length()) {
input.close();
if (!dataFiles.hasNext()) {
log.info("ran out of mux data files after : " + lastPath.toString());
break;
}
lastPath = dataFiles.next();
input = new RandomAccessFile(lastPath.toFile(), "r");
nextBlockPosition = 0;
}
// parse the next block
long currentBlock = blocks++;
int countIDs = input.readShort();
chunks += countIDs;
chunksPerBlock.update(countIDs);
ArrayList<Integer> streamList = new ArrayList<>(50);
for (int i = 0; i < countIDs; i++) {
int streamID = input.readInt();
streamList.add(streamID);
}
int bodySize = input.readInt();
long currentPosition = input.getFilePointer();
long currentBlockStart = nextBlockPosition;
nextBlockPosition = currentPosition + bodySize;
long currentBlockSize = nextBlockPosition - currentBlockStart;
blockSize.update(currentBlockSize);
if (currentBlockSize < tiny_block) {
log.info("Tiny block debug log");
log.info(Objects.toStringHelper("block")
.add("block", currentBlock)
.add("chunks", countIDs)
.add("size", currentBlockSize)
.add("os-file", lastPath.getFileName().toString())
.add("position", currentPosition)
.toString());
StringBuilder sb = new StringBuilder();
for (Integer i : streamList) {
sb.append(i);
sb.append('\n');
}
log.info("Stream ids in block : ");
log.info(sb.toString());
}
for (int i = 0; i < countIDs; i++) {
int chunkBodyOffset = input.readInt(); // throw away
int chunkLength = input.readInt();
chunkSize.update(chunkLength);
}
}
// Report stats
log.info("### Printing stats");
log.info("Total blocks : " + blocks);
log.info("Total chunks : " + chunks);
log.info("Median Chunks per Block : " + chunksPerBlock.getSnapshot().getMedian());
log.info("05th Percentile Chunks per Block : " + chunksPerBlock.getSnapshot().getValue(0.05));
log.info("95th Percentile Chunks per Block : " + chunksPerBlock.getSnapshot().get95thPercentile());
log.info("Median Block Size : " + blockSize.getSnapshot().getMedian());
log.info("Median Chunk Size : " + chunkSize.getSnapshot().getMedian());
log.info("05th Percentile Block Size : " + blockSize.getSnapshot().getValue(0.05));
log.info("05th Percentile Chunk Size : " + chunkSize.getSnapshot().getValue(0.05));
log.info("95th Percentile Block Size : " + blockSize.getSnapshot().get95thPercentile());