@Override
public int execute(DriverContext driverContext) {
int rc = 1;
boolean cleanContext = false;
Context ctx = null;
DAGClient client = null;
TezSessionState session = null;
try {
// Get or create Context object. If we create it we have to clean it later as well.
ctx = driverContext.getCtx();
if (ctx == null) {
ctx = new Context(conf);
cleanContext = true;
}
// Need to remove this static hack. But this is the way currently to get a session.
SessionState ss = SessionState.get();
session = ss.getTezSession();
session = TezSessionPoolManager.getInstance().getSession(session, conf, false);
ss.setTezSession(session);
// jobConf will hold all the configuration for hadoop, tez, and hive
JobConf jobConf = utils.createConfiguration(conf);
// Get all user jars from work (e.g. input format stuff).
String[] inputOutputJars = work.configureJobConfAndExtractJars(jobConf);
// we will localize all the files (jars, plans, hashtables) to the
// scratch dir. let's create this and tmp first.
Path scratchDir = ctx.getMRScratchDir();
// create the tez tmp dir
scratchDir = utils.createTezDir(scratchDir, conf);
// If we have any jars from input format, we need to restart the session because
// AM will need them; so, AM has to be restarted. What a mess...
if (!session.hasResources(inputOutputJars) && session.isOpen()) {
LOG.info("Tez session being reopened to pass custom jars to AM");
session.close(false);
session = TezSessionPoolManager.getInstance().getSession(null, conf, false);
ss.setTezSession(session);
}
if (!session.isOpen()) {
// can happen if the user sets the tez flag after the session was
// established
LOG.info("Tez session hasn't been created yet. Opening session");
session.open(conf, inputOutputJars);
}
List<LocalResource> additionalLr = session.getLocalizedResources();
// unless already installed on all the cluster nodes, we'll have to
// localize hive-exec.jar as well.
LocalResource appJarLr = session.getAppJarLr();
// next we translate the TezWork to a Tez DAG
DAG dag = build(jobConf, work, scratchDir, appJarLr, additionalLr, ctx);
// submit will send the job to the cluster and start executing
client = submit(jobConf, dag, scratchDir, appJarLr, session);
// finally monitor will print progress until the job is done
TezJobMonitor monitor = new TezJobMonitor();
rc = monitor.monitorExecution(client, ctx.getHiveTxnManager(), conf);
// fetch the counters
Set<StatusGetOpts> statusGetOpts = EnumSet.of(StatusGetOpts.GET_COUNTERS);
counters = client.getDAGStatus(statusGetOpts).getDAGCounters();
TezSessionPoolManager.getInstance().returnSession(session);
if (LOG.isInfoEnabled()) {
for (CounterGroup group: counters) {
LOG.info(group.getDisplayName() +":");
for (TezCounter counter: group) {
LOG.info(" "+counter.getDisplayName()+": "+counter.getValue());
}
}
}
} catch (Exception e) {
LOG.error("Failed to execute tez graph.", e);
// rc will be 1 at this point indicating failure.
} finally {
Utilities.clearWork(conf);
if (cleanContext) {
try {
ctx.clear();
} catch (Exception e) {
/*best effort*/
LOG.warn("Failed to clean up after tez job");
}
}