UserPayload stage1Payload = TezUtils.createUserPayloadFromConf(stage1Conf);
UserPayload stage2Payload = TezUtils.createUserPayloadFromConf(stage2Conf);
UserPayload stage3Payload = TezUtils.createUserPayloadFromConf(stage3Conf);
DAG dag = DAG.create("testMRRSleepJobDagSubmit-" + random.nextInt(1000));
Class<? extends InputInitializer> inputInitializerClazz =
genSplitsInAM ?
(initializerClass == null ? MRInputAMSplitGenerator.class : initializerClass)
: null;
LOG.info("Using initializer class: " + initializerClass);
DataSourceDescriptor dsd;
if (!genSplitsInAM) {
dsd = MRInputHelpers
.configureMRInputWithLegacySplitGeneration(stage1Conf, remoteStagingDir, true);
} else {
if (initializerClass == null) {
dsd = MRInputLegacy.createConfigBuilder(stage1Conf, SleepInputFormat.class).build();
} else {
InputInitializerDescriptor iid =
InputInitializerDescriptor.create(inputInitializerClazz.getName());
dsd = MRInputLegacy.createConfigBuilder(stage1Conf, SleepInputFormat.class)
.setCustomInitializerDescriptor(iid).build();
}
}
Vertex stage1Vertex = Vertex.create("map", ProcessorDescriptor.create(
MapProcessor.class.getName()).setUserPayload(stage1Payload),
dsd.getNumberOfShards(), Resource.newInstance(256, 1));
stage1Vertex.addDataSource("MRInput", dsd);
Vertex stage2Vertex = Vertex.create("ireduce", ProcessorDescriptor.create(
ReduceProcessor.class.getName()).setUserPayload(stage2Payload),
1, Resource.newInstance(256, 1));
Vertex stage3Vertex = Vertex.create("reduce", ProcessorDescriptor.create(
ReduceProcessor.class.getName()).setUserPayload(stage3Payload),
1, Resource.newInstance(256, 1));
stage3Vertex.addDataSink("MROutput",
MROutputLegacy.createConfigBuilder(stage3Conf, NullOutputFormat.class).build());
// TODO env, resources
dag.addVertex(stage1Vertex);
dag.addVertex(stage2Vertex);
dag.addVertex(stage3Vertex);
Edge edge1 = Edge.create(stage1Vertex, stage2Vertex, EdgeProperty.create(
DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
SchedulingType.SEQUENTIAL, OutputDescriptor.create(
OrderedPartitionedKVOutput.class.getName()).setUserPayload(stage2Payload),
InputDescriptor.create(
OrderedGroupedInputLegacy.class.getName()).setUserPayload(stage2Payload)));
Edge edge2 = Edge.create(stage2Vertex, stage3Vertex, EdgeProperty.create(
DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
SchedulingType.SEQUENTIAL, OutputDescriptor.create(
OrderedPartitionedKVOutput.class.getName()).setUserPayload(stage3Payload),
InputDescriptor.create(
OrderedGroupedInputLegacy.class.getName()).setUserPayload(stage3Payload)));
dag.addEdge(edge1);
dag.addEdge(edge2);
TezConfiguration tezConf = new TezConfiguration(
mrrTezCluster.getConfig());
tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
remoteStagingDir.toString());
DAGClient dagClient = null;
boolean reuseSession = reUseTezSession != null;
TezClient tezSession = null;
if (!dagViaRPC) {
Preconditions.checkArgument(reuseSession == false);
}
if (!reuseSession) {
TezConfiguration tempTezconf = new TezConfiguration(tezConf);
if (!dagViaRPC) {
tempTezconf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
} else {
tempTezconf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
}
tezSession = TezClient.create("testsession", tempTezconf);
tezSession.start();
} else {
tezSession = reUseTezSession;
}
if(!dagViaRPC) {
// TODO Use utility method post TEZ-205 to figure out AM arguments etc.
dagClient = tezSession.submitDAG(dag);
}
if (dagViaRPC && closeSessionBeforeSubmit) {
YarnClient yarnClient = YarnClient.createYarnClient();
yarnClient.init(mrrTezCluster.getConfig());
yarnClient.start();
boolean sentKillSession = false;
while(true) {
Thread.sleep(500l);
ApplicationReport appReport =
yarnClient.getApplicationReport(tezSession.getAppMasterApplicationId());
if (appReport == null) {
continue;
}
YarnApplicationState appState = appReport.getYarnApplicationState();
if (!sentKillSession) {
if (appState == YarnApplicationState.RUNNING) {
tezSession.stop();
sentKillSession = true;
}
} else {
if (appState == YarnApplicationState.FINISHED
|| appState == YarnApplicationState.KILLED
|| appState == YarnApplicationState.FAILED) {
LOG.info("Application completed after sending session shutdown"
+ ", yarnApplicationState=" + appState
+ ", finalAppStatus=" + appReport.getFinalApplicationStatus());
Assert.assertEquals(YarnApplicationState.FINISHED,
appState);
Assert.assertEquals(FinalApplicationStatus.SUCCEEDED,
appReport.getFinalApplicationStatus());
break;
}
}
}
yarnClient.stop();
return null;
}
if(dagViaRPC) {
LOG.info("Submitting dag to tez session with appId=" + tezSession.getAppMasterApplicationId()
+ " and Dag Name=" + dag.getName());
if (additionalLocalResources != null) {
tezSession.addAppMasterLocalFiles(additionalLocalResources);
}
dagClient = tezSession.submitDAG(dag);
Assert.assertEquals(TezAppMasterStatus.RUNNING,