}
Utils.copyJarContents("resources/"+ConfigConstants.DEFAULT_JOB_MANAGER_WEB_PATH_NAME,
ApplicationMaster.class.getProtectionDomain().getCodeSource().getLocation().getPath());
JobManager jm;
{
String pathToNepheleConfig = currDir+"/stratosphere-conf-modified.yaml";
String[] args = {"-executionMode","cluster", "-configDir", pathToNepheleConfig};
// start the job manager
jm = JobManager.initialize( args );
// Start info server for jobmanager
jm.startInfoServer();
}
AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient();
rmClient.init(conf);
rmClient.start();
NMClient nmClient = NMClient.createNMClient();
nmClient.init(conf);
nmClient.start();
// Register with ResourceManager
LOG.info("registering ApplicationMaster");
rmClient.registerApplicationMaster(applicationMasterHost, 0, "http://"+applicationMasterHost+":"+GlobalConfiguration.getString(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, "undefined"));
// Priority for worker containers - priorities are intra-application
Priority priority = Records.newRecord(Priority.class);
priority.setPriority(0);
// Resource requirements for worker containers
Resource capability = Records.newRecord(Resource.class);
capability.setMemory(memoryPerTaskManager);
capability.setVirtualCores(coresPerTaskManager);
// Make container requests to ResourceManager
for (int i = 0; i < taskManagerCount; ++i) {
ContainerRequest containerAsk = new ContainerRequest(capability,
null, null, priority);
LOG.info("Requesting TaskManager container " + i);
rmClient.addContainerRequest(containerAsk);
}
LocalResource stratosphereJar = Records.newRecord(LocalResource.class);
LocalResource stratosphereConf = Records.newRecord(LocalResource.class);
// register Stratosphere Jar with remote HDFS
final Path remoteJarPath = new Path(remoteStratosphereJarPath);
Utils.registerLocalResource(fs, remoteJarPath, stratosphereJar);
// register conf with local fs.
Path remoteConfPath = Utils.setupLocalResource(conf, fs, appId, new Path("file://"+currDir+"/stratosphere-conf-modified.yaml"), stratosphereConf, new Path(clientHomeDir));
LOG.info("Prepared localresource for modified yaml: "+stratosphereConf);
boolean hasLog4j = new File(currDir+"/log4j.properties").exists();
// prepare the files to ship
LocalResource[] remoteShipRsc = null;
String[] remoteShipPaths = shipListString.split(",");
if(!shipListString.isEmpty()) {
remoteShipRsc = new LocalResource[remoteShipPaths.length];
{ // scope for i
int i = 0;
for(String remoteShipPathStr : remoteShipPaths) {
if(remoteShipPathStr == null || remoteShipPathStr.isEmpty()) {
continue;
}
remoteShipRsc[i] = Records.newRecord(LocalResource.class);
Path remoteShipPath = new Path(remoteShipPathStr);
Utils.registerLocalResource(fs, remoteShipPath, remoteShipRsc[i]);
i++;
}
}
}
// respect custom JVM options in the YAML file
final String javaOpts = GlobalConfiguration.getString(ConfigConstants.STRATOSPHERE_JVM_OPTIONS, "");
// Obtain allocated containers and launch
int allocatedContainers = 0;
int completedContainers = 0;
while (allocatedContainers < taskManagerCount) {
AllocateResponse response = rmClient.allocate(0);
for (Container container : response.getAllocatedContainers()) {
LOG.info("Got new Container for TM "+container.getId()+" on host "+container.getNodeId().getHost());
++allocatedContainers;
// Launch container by create ContainerLaunchContext
ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
String tmCommand = "$JAVA_HOME/bin/java -Xmx"+heapLimit+"m " + javaOpts ;
if(hasLog4j) {
tmCommand += " -Dlog.file=\""+ApplicationConstants.LOG_DIR_EXPANSION_VAR +"/taskmanager-log4j.log\" -Dlog4j.configuration=file:log4j.properties";
}
tmCommand += " eu.stratosphere.yarn.YarnTaskManagerRunner -configDir . "
+ " 1>"
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR
+ "/taskmanager-stdout.log"
+ " 2>"
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR
+ "/taskmanager-stderr.log";
ctx.setCommands(Collections.singletonList(tmCommand));
LOG.info("Starting TM with command="+tmCommand);
// copy resources to the TaskManagers.
Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(2);
localResources.put("stratosphere.jar", stratosphereJar);
localResources.put("stratosphere-conf.yaml", stratosphereConf);
// add ship resources
if(!shipListString.isEmpty()) {
Preconditions.checkNotNull(remoteShipRsc);
for( int i = 0; i < remoteShipPaths.length; i++) {
localResources.put(new Path(remoteShipPaths[i]).getName(), remoteShipRsc[i]);
}
}
ctx.setLocalResources(localResources);
// Setup CLASSPATH for Container (=TaskTracker)
Map<String, String> containerEnv = new HashMap<String, String>();
Utils.setupEnv(conf, containerEnv); //add stratosphere.jar to class path.
containerEnv.put(Client.ENV_CLIENT_USERNAME, yarnClientUsername);
ctx.setEnvironment(containerEnv);
UserGroupInformation user = UserGroupInformation.getCurrentUser();
try {
Credentials credentials = user.getCredentials();
DataOutputBuffer dob = new DataOutputBuffer();
credentials.writeTokenStorageToStream(dob);
ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(),
0, dob.getLength());
ctx.setTokens(securityTokens);
} catch (IOException e) {
LOG.warn("Getting current user info failed when trying to launch the container"
+ e.getMessage());
}
LOG.info("Launching container " + allocatedContainers);
nmClient.startContainer(container, ctx);
}
for (ContainerStatus status : response.getCompletedContainersStatuses()) {
++completedContainers;
LOG.info("Completed container (while allocating) "+status.getContainerId()+". Total Completed:" + completedContainers);
LOG.info("Diagnostics "+status.getDiagnostics());
}
Thread.sleep(100);
}
// Now wait for containers to complete
while (completedContainers < taskManagerCount) {
AllocateResponse response = rmClient.allocate(completedContainers
/ taskManagerCount);
for (ContainerStatus status : response.getCompletedContainersStatuses()) {
++completedContainers;
LOG.info("Completed container "+status.getContainerId()+". Total Completed:" + completedContainers);
LOG.info("Diagnostics "+status.getDiagnostics());
}
Thread.sleep(5000);
}
LOG.info("Shutting down JobManager");
jm.shutdown();
// Un-register with ResourceManager
rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", "");