String otherId = oozieMeta.get(ZKUtils.ZKMetadataKeys.OOZIE_ID);
// If it's this server, we can just get them directly
if (otherId.equals(zk.getZKId())) {
BufferedReader reader = new XLogStreamer(filter, xLogService.getOozieLogPath(), xLogService.getOozieLogName(),
xLogService.getOozieLogRotation()).makeReader(startTime, endTime);
parsers.add(new TimestampedMessageParser(reader, filter));
}
// If it's another server, we'll have to use the REST API
else {
String otherUrl = oozieMeta.get(ZKUtils.ZKMetadataKeys.OOZIE_URL);
String jobId = filter.getFilterParams().get(DagXLogInfoService.JOB);
try {
// It's important that we specify ALL_SERVERS_PARAM=false in the GET request to prevent the other Oozie
// Server from trying aggregate logs from the other Oozie servers (and creating an infinite recursion)
final String url = otherUrl + "/v" + OozieClient.WS_PROTOCOL_VERSION + "/" + RestConstants.JOB
+ "/" + jobId + "?" + RestConstants.JOB_SHOW_PARAM + "=" + RestConstants.JOB_SHOW_LOG
+ "&" + RestConstants.ALL_SERVER_REQUEST + "=false" + AuthUrlClient.getQueryParamString(params);
BufferedReader reader = AuthUrlClient.callServer(url);
parsers.add(new SimpleTimestampedMessageParser(reader, filter));
}
catch(IOException ioe) {
log.warn("Failed to retrieve logs for job [" + jobId + "] from Oozie server with ID [" + otherId
+ "] at [" + otherUrl + "]; log information may be incomplete", ioe);
badOozies.add(otherId);
}
}
}
//If log param debug is set, we need to write start date and end date to outputstream.
if(filter.isDebugMode()){
writer.write(filter.getDebugMessage());
}
// Add a message about any servers we couldn't contact
if (!badOozies.isEmpty()) {
writer.write("Unable to contact the following Oozie Servers for logs (log information may be incomplete):\n");
for (String badOozie : badOozies) {
writer.write(" ");
writer.write(badOozie);
writer.write("\n");
}
writer.write("\n");
writer.flush();
}
// If it's just the one server (this server), then we don't need to do any more processing and can just copy it directly
if (parsers.size() == 1) {
TimestampedMessageParser parser = parsers.get(0);
parser.processRemaining(writer, bufferLen);
}
else {
// Now that we have a Reader for each server to get the logs from that server, we have to collate them. Within each
// server, the logs should already be in the correct order, so we can take advantage of that. We'll use the
// BufferedReaders to read the messages from the logs of each server and put them in order without having to bring
// every message into memory at the same time.
TreeMap<String, TimestampedMessageParser> timestampMap = new TreeMap<String, TimestampedMessageParser>();
// populate timestampMap with initial values
for (TimestampedMessageParser parser : parsers) {
if (parser.increment()) {
timestampMap.put(parser.getLastTimestamp(), parser);
}
}
int bytesWritten = 0;
while (timestampMap.size() > 1) {
// The first entry will be the earliest based on the timestamp (also removes it) from the map
TimestampedMessageParser earliestParser = timestampMap.pollFirstEntry().getValue();
// Write the message from that parser at that timestamp
writer.write(earliestParser.getLastMessage());
bytesWritten = earliestParser.getLastMessage().length();
if (bytesWritten > bufferLen) {
writer.flush();
bytesWritten = 0;
}
// Increment that parser to read the next message
if (earliestParser.increment()) {
// If it still has messages left, put it back in the map with the new last timestamp for it
timestampMap.put(earliestParser.getLastTimestamp(), earliestParser);
}
}
// If there's only one parser left in the map, then we can simply copy the rest of its lines directly to be faster
if (timestampMap.size() == 1) {
TimestampedMessageParser parser = timestampMap.values().iterator().next();
writer.write(parser.getLastMessage()); // don't forget the last message read by the parser
parser.processRemaining(writer, bufferLen, bytesWritten + parser.getLastMessage().length());
}
}
}
finally {
for (TimestampedMessageParser parser : parsers) {
parser.closeReader();
}
writer.flush();
}
}