// Run a MR job
// create a MR cluster
conf.setInt(TTConfig.TT_MAP_SLOTS, 1);
conf.setInt(TTConfig.TT_REDUCE_SLOTS, 1);
MiniMRCluster mrCluster = new MiniMRCluster(1, "file:///", 1, null, null,
new JobConf(conf));
// run a job
Path inDir = new Path(tempDir, "input");
Path outDir = new Path(tempDir, "output");
JobHistoryParser parser = null;
RewindableInputStream ris = null;
ArrayList<String> seenEvents = new ArrayList<String>(15);
try {
JobConf jConf = mrCluster.createJobConf();
// construct a job with 1 map and 1 reduce task.
Job job = MapReduceTestUtil.createJob(jConf, inDir, outDir, 1, 1);
// disable setup/cleanup
job.setJobSetupCleanupNeeded(false);
// set the output format to take care of the _temporary folder
job.setOutputFormatClass(MyOutputFormat.class);
// wait for the job to complete
job.waitForCompletion(false);
assertTrue("Job failed", job.isSuccessful());
JobID id = job.getJobID();
JobClient jc = new JobClient(jConf);
String user = jc.getAllJobs()[0].getUsername();
// get the jobhistory filepath
Path jhPath =
new Path(mrCluster.getJobTrackerRunner().getJobTracker()
.getJobHistoryDir());
Path inputPath = JobHistory.getJobHistoryFile(jhPath, id, user);
// wait for 10 secs for the jobhistory file to move into the done folder
for (int i = 0; i < 100; ++i) {
if (lfs.exists(inputPath)) {
break;
}
TimeUnit.MILLISECONDS.wait(100);
}
assertTrue("Missing job history file", lfs.exists(inputPath));
ris = getRewindableInputStream(inputPath, conf);
// Test if the JobHistoryParserFactory can detect the parser correctly
parser = JobHistoryParserFactory.getParser(ris);
// create a job builder
JobBuilder builder = new JobBuilder(id.toString());
// get events into seenEvents and also process them using builder
getHistoryEvents(parser, seenEvents, builder);
// Check against the gold standard
System.out.println("testCurrentJHParser validating using gold std ");
// The list of history events expected when parsing the above job's
// history log file
String[] goldLinesExpected = new String[] {
JSE, JPCE, JIE, JSCE, TSE, ASE, MFE, TFE, TSE, ASE, RFE, TFE, JFE
};
validateSeenHistoryEvents(seenEvents, goldLinesExpected);
// validate resource usage metrics
// get the job counters
Counters counters = job.getTaskReports(TaskType.MAP)[0].getTaskCounters();
// get the logged job
LoggedJob loggedJob = builder.build();
// get the logged attempts
LoggedTaskAttempt attempt =
loggedJob.getMapTasks().get(0).getAttempts().get(0);
// get the resource usage metrics
ResourceUsageMetrics metrics = attempt.getResourceUsageMetrics();
// check with the actual values
testResourceUsageMetricViaDeepCompare(metrics,
counters.findCounter(TaskCounter.CPU_MILLISECONDS).getValue(),
counters.findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES).getValue(),
counters.findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES).getValue(),
counters.findCounter(TaskCounter.COMMITTED_HEAP_BYTES).getValue(),
true);
} finally {
// stop the MR cluster
mrCluster.shutdown();
if (ris != null) {
ris.close();
}
if (parser != null) {