tezStats = new TezPigScriptStats(pc);
PigStats.start(tezStats);
conf.set(TezConfiguration.TEZ_USE_CLUSTER_HADOOP_LIBS, "true");
TezJobCompiler jc = new TezJobCompiler(pc, conf);
TezPlanContainer tezPlanContainer = compile(php, pc);
tezStats.initialize(tezPlanContainer);
tezScriptState.emitInitialPlanNotification(tezPlanContainer);
tezScriptState.emitLaunchStartedNotification(tezPlanContainer.size()); //number of DAGs to Launch
TezPlanContainerNode tezPlanContainerNode;
TezOperPlan tezPlan;
int processedDAGs = 0;
while ((tezPlanContainerNode = tezPlanContainer.getNextPlan(processedPlans)) != null) {
tezPlan = tezPlanContainerNode.getTezOperPlan();
processLoadAndParallelism(tezPlan, pc);
processedPlans.add(tezPlan);
ProgressReporter reporter = new ProgressReporter(tezPlanContainer.size(), processedDAGs);
if (tezPlan.size()==1 && tezPlan.getRoots().get(0) instanceof NativeTezOper) {
// Native Tez Plan
NativeTezOper nativeOper = (NativeTezOper)tezPlan.getRoots().get(0);
tezScriptState.emitJobsSubmittedNotification(1);
nativeOper.runJob(tezPlanContainerNode.getOperatorKey().toString());
} else {
TezPOPackageAnnotator pkgAnnotator = new TezPOPackageAnnotator(tezPlan);
pkgAnnotator.visit();
runningJob = jc.compile(tezPlanContainerNode, tezPlanContainer);
//TODO: Exclude vertex groups from numVerticesToLaunch ??
tezScriptState.dagLaunchNotification(runningJob.getName(), tezPlan, tezPlan.size());
runningJob.setPigStats(tezStats);
// Set the thread UDFContext so registered classes are available.
final UDFContext udfContext = UDFContext.getUDFContext();
Thread task = new Thread(runningJob) {
@Override
public void run() {
UDFContext.setUdfContext(udfContext.clone());
super.run();
}
};
JobControlThreadExceptionHandler jctExceptionHandler = new JobControlThreadExceptionHandler();
task.setUncaughtExceptionHandler(jctExceptionHandler);
task.setContextClassLoader(PigContext.getClassLoader());
// Mark the times that the jobs were submitted so it's reflected in job
// history props. TODO: Fix this. unused now
long scriptSubmittedTimestamp = System.currentTimeMillis();
// Job.getConfiguration returns the shared configuration object
Configuration jobConf = runningJob.getConfiguration();
jobConf.set("pig.script.submitted.timestamp",
Long.toString(scriptSubmittedTimestamp));
jobConf.set("pig.job.submitted.timestamp",
Long.toString(System.currentTimeMillis()));
Future<?> future = executor.submit(task);
tezScriptState.emitJobsSubmittedNotification(1);
boolean jobStarted = false;
while (!future.isDone()) {
if (!jobStarted && runningJob.getApplicationId() != null) {
jobStarted = true;
String appId = runningJob.getApplicationId().toString();
//For Oozie Pig action job id matching compatibility with MR mode
log.info("HadoopJobId: "+ appId.replace("application", "job"));
tezScriptState.emitJobStartedNotification(appId);
tezScriptState.dagStartedNotification(runningJob.getName(), appId);
}
reporter.notifyUpdate();
Thread.sleep(1000);
}
}
processedDAGs++;
if (tezPlanContainer.size() == processedDAGs) {
tezScriptState.emitProgressUpdatedNotification(100);
} else {
tezScriptState.emitProgressUpdatedNotification(
((tezPlanContainer.size() - processedDAGs)/tezPlanContainer.size()) * 100);
}
tezPlanContainer.updatePlan(tezPlan, reporter.notifyFinishedOrFailed());
}
tezStats.finish();
tezScriptState.emitLaunchCompletedNotification(tezStats.getNumberSuccessfulJobs());