});
if (useStoredStates) {
LOG.info("Create file reader with consumer state: {}", consumerState);
// Has existing offsets, just resume from there.
MultiLiveStreamFileReader reader = new MultiLiveStreamFileReader(streamConfig, consumerState.getState());
reader.initialize();
return reader;
}
}
// TODO: Support starting from some time rather then from beginning.
// Otherwise, search for files with the smallest partition start time
// If no partition exists for the stream, start with one partition earlier than current time to make sure
// no event will be lost if events start flowing in about the same time.
long startTime = StreamUtils.getPartitionStartTime(currentTime - streamConfig.getPartitionDuration(),
streamConfig.getPartitionDuration());
long earliestNonExpiredTime = StreamUtils.getPartitionStartTime(currentTime - streamConfig.getTTL(),
streamConfig.getPartitionDuration());
for (Location partitionLocation : streamLocation.list()) {
if (!partitionLocation.isDirectory()) {
// Partition should be a directory
continue;
}
long partitionStartTime = StreamUtils.getPartitionStartTime(partitionLocation.getName());
boolean isPartitionExpired = partitionStartTime < earliestNonExpiredTime;
if (!isPartitionExpired && partitionStartTime < startTime) {
startTime = partitionStartTime;
}
}
// Create file offsets
// TODO: Be able to support dynamic name of stream writer instances.
// Maybe it's done through MultiLiveStreamHandler to alter list of file offsets dynamically
Location partitionLocation = StreamUtils.createPartitionLocation(streamLocation,
startTime, streamConfig.getPartitionDuration());
List<StreamFileOffset> fileOffsets = Lists.newArrayList();
getFileOffsets(partitionLocation, fileOffsets, generation);
LOG.info("Empty consumer state. Create file reader with file offsets: groupId={}, instanceId={} states={}",
consumerState.getGroupId(), consumerState.getInstanceId(), fileOffsets);
MultiLiveStreamFileReader reader = new MultiLiveStreamFileReader(streamConfig, fileOffsets);
reader.initialize();
return reader;
}