/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.giraph.yarn;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
import org.apache.giraph.conf.GiraphConfiguration;
import org.apache.giraph.conf.GiraphConstants;
import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.protocolrecords
.RegisterApplicationMasterResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.log4j.Logger;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
/**
* The YARN Application Master for Giraph is launched when the GiraphYarnClient
* successfully requests an execution container from the Resource Manager. The
* Application Master is provided by Giraph to manage all requests for resources
* (worker nodes, memory, jar files, job configuration metadata, etc.) that
* Giraph will need to perform the job. When Giraph runs in a non-YARN context,
* the role of the Application Master is played by Hadoop when it launches our
* GraphMappers (worker/master task nodes) to run the job.
*/
public class GiraphApplicationMaster {
/** Logger */
private static final Logger LOG =
Logger.getLogger(GiraphApplicationMaster.class);
/** Exit code for YARN containers that were manually killed/aborted */
private static final int YARN_ABORT_EXIT_STATUS = -100;
/** Exit code for successfully run YARN containers */
private static final int YARN_SUCCESS_EXIT_STATUS = 0;
/** millis to sleep between heartbeats during long loops */
private static final int SLEEP_BETWEEN_HEARTBEATS_MSECS = 900;
/** A reusable map of resources already in HDFS for each task to copy-to-local
* env and use to launch each GiraphYarnTask. */
private static Map<String, LocalResource> LOCAL_RESOURCES;
/** Initialize the Configuration class with the resource file exported by
* the YarnClient. We will need to export this resource to the tasks also.
* Construct the HEARTBEAT to use to ping the RM about job progress/health.
*/
//TODO
/** For status update for clients - yet to be implemented\\
* Hostname of the container
*/
private String appMasterHostname = "";
/** Port on which the app master listens for status updates from clients*/
private int appMasterRpcPort = 0;
/** Tracking url to which app master publishes info for clients to monitor*/
private String appMasterTrackingUrl = "";
static {
// pick up new conf XML file and populate it with stuff exported from client
Configuration.addDefaultResource(GiraphConstants.GIRAPH_YARN_CONF_FILE);
}
/** GiraphApplicationMaster's application attempt id */
private final ApplicationAttemptId appAttemptId;
/** GiraphApplicationMaster container id. Leave me here, I'm very useful */
private final ContainerId containerId;
/** number of containers Giraph needs (conf.getMaxWorkers() + 1 master) */
private final int containersToLaunch;
/** MB of JVM heap per Giraph task container */
private final int heapPerContainer;
/** Giraph configuration for this job, transported here by YARN framework */
private final ImmutableClassesGiraphConfiguration giraphConf;
/** Yarn configuration for this job*/
private final YarnConfiguration yarnConf;
/** Completed Containers Counter */
private final AtomicInteger completedCount;
/** Failed Containers Counter */
private final AtomicInteger failedCount;
/** Number of containers requested (hopefully '-w' from our conf) */
private final AtomicInteger allocatedCount;
/** Number of successfully completed containers in this job run. */
private final AtomicInteger successfulCount;
/** the ACK #'s for AllocateRequests + heartbeats == last response # */
private AtomicInteger lastResponseId;
/** buffer tostore all tokens */
private ByteBuffer allTokens;
/** Executor to attempt asynchronous launches of Giraph containers */
private ExecutorService executor;
/** YARN progress is a <code>float</code> between 0.0f and 1.0f */
//Handle to communicate with the Resource Manager
@SuppressWarnings("rawtypes")
private AMRMClientAsync amRMClient;
/** Handle to communicate with the Node Manager */
private NMClientAsync nmClientAsync;
/** Listen to process the response from the Node Manager */
private NMCallbackHandler containerListener;
/** whether all containers finishe */
private volatile boolean done;
/**
* Construct the GiraphAppMaster, populate fields using env vars
* set up by YARN framework in this execution container.
* @param cId the ContainerId
* @param aId the ApplicationAttemptId
*/
protected GiraphApplicationMaster(ContainerId cId, ApplicationAttemptId aId)
throws IOException {
containerId = cId; // future good stuff will need me to operate.
appAttemptId = aId;
lastResponseId = new AtomicInteger(0);
giraphConf =
new ImmutableClassesGiraphConfiguration(new GiraphConfiguration());
yarnConf = new YarnConfiguration(giraphConf);
completedCount = new AtomicInteger(0);
failedCount = new AtomicInteger(0);
allocatedCount = new AtomicInteger(0);
successfulCount = new AtomicInteger(0);
containersToLaunch = giraphConf.getMaxWorkers() + 1;
executor = Executors.newFixedThreadPool(containersToLaunch);
heapPerContainer = giraphConf.getYarnTaskHeapMb();
LOG.info("GiraphAM for ContainerId " + cId + " ApplicationAttemptId " +
aId);
}
/**
* Coordinates all requests for Giraph's worker/master task containers, and
* manages application liveness heartbeat, completion status, teardown, etc.
* @return success or failure
*/
private boolean run() throws YarnException, IOException {
boolean success = false;
try {
getAllTokens();
registerRMCallBackHandler();
registerNMCallbackHandler();
registerAMToRM();
madeAllContainerRequestToRM();
LOG.info("Wait to finish ..");
while (!done) {
try {
Thread.sleep(200);
} catch (InterruptedException ex) {
LOG.error(ex);
//TODO:
}
}
LOG.info("Done " + done);
} finally {
// if we get here w/o problems, the executor is already long finished.
if (null != executor && !executor.isTerminated()) {
LOG.info("Forcefully terminating executors with done =:" + done);
executor.shutdownNow(); // force kill, especially if got here by throw
}
success = finish();
}
return success;
}
/**
* Call when the application is done
* @return if all containers succeed
*/
private boolean finish() {
// When the application completes, it should stop all running containers
LOG.info("Application completed. Stopping running containers");
nmClientAsync.stop();
// When the application completes, it should send a finish application
// signal to the RM
LOG.info("Application completed. Signalling finish to RM");
FinalApplicationStatus appStatus;
String appMessage = null;
boolean success = true;
if (failedCount.get() == 0 &&
completedCount.get() == containersToLaunch) {
appStatus = FinalApplicationStatus.SUCCEEDED;
} else {
appStatus = FinalApplicationStatus.FAILED;
appMessage = "Diagnostics." + ", total=" + containersToLaunch +
", completed=" + completedCount.get() + ", failed=" +
failedCount.get();
success = false;
}
try {
amRMClient.unregisterApplicationMaster(appStatus, appMessage, null);
} catch (YarnException ex) {
LOG.error("Failed to unregister application", ex);
} catch (IOException e) {
LOG.error("Failed to unregister application", e);
}
amRMClient.stop();
return success;
}
/**
* Add all containers' request
* @return
*/
private void madeAllContainerRequestToRM() {
// Setup ask for containers from RM
// Send request for containers to RM
// Until we get our fully allocated quota, we keep on polling RM for
// containers
// Keep looping until all the containers are launched and shell script
// executed on them ( regardless of success/failure).
for (int i = 0; i < containersToLaunch; ++i) {
ContainerRequest containerAsk = setupContainerAskForRM();
amRMClient.addContainerRequest(containerAsk);
}
}
/**
* Setup the request that will be sent to the RM for the container ask.
*
* @return the setup ResourceRequest to be sent to RM
*/
private ContainerRequest setupContainerAskForRM() {
// setup requirements for hosts
// using * as any host will do for the distributed shell app
// set the priority for the request
Priority pri = Records.newRecord(Priority.class);
// TODO - what is the range for priority? how to decide?
pri.setPriority(GiraphConstants.GIRAPH_YARN_PRIORITY);
// Set up resource type requirements
// For now, only memory is supported so we set memory requirements
Resource capability = Records.newRecord(Resource.class);
capability.setMemory(heapPerContainer);
ContainerRequest request = new ContainerRequest(capability, null, null,
pri);
LOG.info("Requested container ask: " + request.toString());
return request;
}
/**
* Populate allTokens with the tokens received
* @return
*/
private void getAllTokens() throws IOException {
Credentials credentials = UserGroupInformation.getCurrentUser()
.getCredentials();
DataOutputBuffer dob = new DataOutputBuffer();
credentials.writeTokenStorageToStream(dob);
// Now remove the AM->RM token so that containers cannot access it.
Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
while (iter.hasNext()) {
Token<?> token = iter.next();
if (LOG.isDebugEnabled()) {
LOG.debug("Token type :" + token.getKind());
}
if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
iter.remove();
}
}
allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
}
/**
* Register RM callback and start listening
* @return
*/
private void registerRMCallBackHandler() {
AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
amRMClient = AMRMClientAsync.createAMRMClientAsync(1000,
allocListener);
amRMClient.init(yarnConf);
amRMClient.start();
}
/**
* Register NM callback and start listening
* @return
*/
private void registerNMCallbackHandler() {
containerListener = new NMCallbackHandler();
nmClientAsync = new NMClientAsyncImpl(containerListener);
nmClientAsync.init(yarnConf);
nmClientAsync.start();
}
/**
* Register AM to RM
* @return AM register response
*/
private RegisterApplicationMasterResponse registerAMToRM()
throws YarnException {
// register Application Master with the YARN Resource Manager so we can
// begin requesting resources.
try {
if (UserGroupInformation.isSecurityEnabled()) {
LOG.info("SECURITY ENABLED ");
}
// TODO: provide actual call back details
RegisterApplicationMasterResponse response = amRMClient
.registerApplicationMaster(appMasterHostname
, appMasterRpcPort, appMasterTrackingUrl);
return response;
} catch (IOException ioe) {
throw new IllegalStateException(
"GiraphApplicationMaster failed to register with RM.", ioe);
}
}
/**
* For each container successfully allocated, attempt to set up and launch
* a Giraph worker/master task.
* @param allocatedContainers the containers we have currently allocated.
*/
private void startContainerLaunchingThreads(final List<Container>
allocatedContainers) {
for (Container allocatedContainer : allocatedContainers) {
LOG.info("Launching command on a new container." +
", containerId=" + allocatedContainer.getId() +
", containerNode=" + allocatedContainer.getNodeId().getHost() +
":" + allocatedContainer.getNodeId().getPort() +
", containerNodeURI=" + allocatedContainer.getNodeHttpAddress() +
", containerResourceMemory=" +
allocatedContainer.getResource().getMemory());
// Launch and start the container on a separate thread to keep the main
// thread unblocked as all containers may not be allocated at one go.
LaunchContainerRunnable runnableLaunchContainer =
new LaunchContainerRunnable(allocatedContainer, containerListener);
executor.execute(runnableLaunchContainer);
}
}
/**
* Lazily compose the map of jar and file names to LocalResource records for
* inclusion in GiraphYarnTask container requests. Can re-use the same map
* as Giraph tasks need identical HDFS-based resources (jars etc.) to run.
* @return the resource map for a ContainerLaunchContext
*/
private synchronized Map<String, LocalResource> getTaskResourceMap() {
// Set the local resources: just send the copies already in HDFS
if (null == LOCAL_RESOURCES) {
LOCAL_RESOURCES = Maps.newHashMap();
try {
// if you have to update the giraphConf for export to tasks, do it now
updateGiraphConfForExport();
YarnUtils.addFsResourcesToMap(LOCAL_RESOURCES, giraphConf,
appAttemptId.getApplicationId());
} catch (IOException ioe) {
// fail fast, this container will never launch.
throw new IllegalStateException("Could not configure the container" +
"launch context for GiraphYarnTasks.", ioe);
}
}
// else, return the prepopulated copy to reuse for each GiraphYarkTask
return LOCAL_RESOURCES;
}
/**
* If you're going to make ANY CHANGES to your local GiraphConfiguration
* while running the GiraphApplicationMaster, put them here.
* This method replaces the current XML file GiraphConfiguration
* stored in HDFS with the copy you have modified locally in-memory.
*/
private void updateGiraphConfForExport()
throws IOException {
// Giraph expects this MapReduce stuff
giraphConf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID,
appAttemptId.getAttemptId());
// now republish the giraph-conf.xml in HDFS
YarnUtils.exportGiraphConfiguration(giraphConf,
appAttemptId.getApplicationId());
}
/**
* Application entry point
* @param args command-line args (set by GiraphYarnClient, if any)
*/
public static void main(final String[] args) {
boolean result = false;
LOG.info("Starting GitaphAM ");
String containerIdString = System.getenv().get(
Environment.CONTAINER_ID.name());
if (containerIdString == null) {
// container id should always be set in the env by the framework
throw new IllegalArgumentException("ContainerId not found in env vars.");
}
ContainerId containerId = ConverterUtils.toContainerId(containerIdString);
ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId();
try {
GiraphApplicationMaster giraphAppMaster =
new GiraphApplicationMaster(containerId, appAttemptId);
result = giraphAppMaster.run();
// CHECKSTYLE: stop IllegalCatch
} catch (Throwable t) {
// CHECKSTYLE: resume IllegalCatch
LOG.error("GiraphApplicationMaster caught a " +
"top-level exception in main.", t);
System.exit(1);
}
if (result) {
LOG.info("Giraph Application Master completed successfully. exiting");
System.exit(0);
} else {
LOG.info("Giraph Application Master failed. exiting");
System.exit(2);
}
}
/**
* Thread to connect to the {@link ContainerManager} and launch the container
* that will house one of our Giraph worker (or master) tasks.
*/
private class LaunchContainerRunnable implements Runnable {
/** Allocated container */
private Container container;
/** NM listener */
private NMCallbackHandler containerListener;
/**
* Constructor.
* @param newGiraphTaskContainer Allocated container
* @param containerListener container listener.
*/
public LaunchContainerRunnable(final Container newGiraphTaskContainer,
NMCallbackHandler containerListener) {
this.container = newGiraphTaskContainer;
this.containerListener = containerListener;
}
/**
* Connects to CM, sets up container launch context
* for shell command and eventually dispatches the container
* start request to the CM.
*/
public void run() {
// Connect to ContainerManager
// configure the launcher for the Giraph task it will host
ContainerLaunchContext ctx = buildContainerLaunchContext();
// request CM to start this container as spec'd in ContainerLaunchContext
containerListener.addContainer(container.getId(), container);
nmClientAsync.startContainerAsync(container, ctx);
}
/**
* Boilerplate to set up the ContainerLaunchContext to tell the Container
* Manager how to launch our Giraph task in the execution container we have
* already allocated.
* @return a populated ContainerLaunchContext object.
*/
private ContainerLaunchContext buildContainerLaunchContext() {
LOG.info("Setting up container launch container for containerid=" +
container.getId());
ContainerLaunchContext launchContext = Records
.newRecord(ContainerLaunchContext.class);
// args inject the CLASSPATH, heap MB, and TaskAttemptID for launched task
final List<String> commands = generateShellExecCommand();
LOG.info("Conatain launch Commands :" + commands.get(0));
launchContext.setCommands(commands);
// Set up tokens for the container too. We are
// populating them mainly for NodeManagers to be able to download any
// files in the distributed file-system. The tokens are otherwise also
// useful in cases, for e.g., when one is running a
// "hadoop dfs" like command
launchContext.setTokens(allTokens.slice());
// add user information to the job
String jobUserName = "ERROR_UNKNOWN_USER";
UserGroupInformation ugi = null;
try {
ugi = UserGroupInformation.getCurrentUser();
jobUserName = ugi.getUserName();
} catch (IOException ioe) {
jobUserName =
System.getenv(ApplicationConstants.Environment.USER.name());
}
//launchContext.setUser(jobUserName);
LOG.info("Setting username in ContainerLaunchContext to: " + jobUserName);
// Set the environment variables to inject into remote task's container
buildEnvironment(launchContext);
// Set the local resources: just send the copies already in HDFS
launchContext.setLocalResources(getTaskResourceMap());
return launchContext;
}
/**
* Generates our command line string used to launch our Giraph tasks.
* @return the BASH shell commands to launch the job.
*/
private List<String> generateShellExecCommand() {
return ImmutableList.of("java " +
"-Xmx" + heapPerContainer + "M " +
"-Xms" + heapPerContainer + "M " +
"-cp .:${CLASSPATH} " +
"org.apache.giraph.yarn.GiraphYarnTask " +
appAttemptId.getApplicationId().getClusterTimestamp() + " " +
appAttemptId.getApplicationId().getId() + " " +
container.getId().getId() + " " +
appAttemptId.getAttemptId() + " " +
"1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR +
"/task-" + container.getId().getId() + "-stdout.log " +
"2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR +
"/task-" + container.getId().getId() + "-stderr.log "
);
}
/**
* Utility to populate the environment vars we wish to inject into the new
* containter's env when the Giraph BSP task is executed.
* @param launchContext the launch context which will set our environment
* vars in the app master's execution container.
*/
private void buildEnvironment(final ContainerLaunchContext launchContext) {
Map<String, String> classPathForEnv = Maps.<String, String>newHashMap();
// pick up the local classpath so when we instantiate a Configuration
// remotely, we also get the "mapred-site.xml" and "yarn-site.xml"
YarnUtils.addLocalClasspathToEnv(classPathForEnv, giraphConf);
// set this map of env vars into the launch context.
launchContext.setEnvironment(classPathForEnv);
}
}
/**
* CallbackHandler to process RM async calls
*/
private class RMCallbackHandler implements AMRMClientAsync.CallbackHandler {
@SuppressWarnings("unchecked")
@Override
public void onContainersCompleted(List<ContainerStatus>
completedContainers) {
LOG.info("Got response from RM for container ask, completedCnt=" +
completedContainers.size());
for (ContainerStatus containerStatus : completedContainers) {
LOG.info("Got container status for containerID=" +
containerStatus.getContainerId() + ", state=" +
containerStatus.getState() + ", exitStatus=" +
containerStatus.getExitStatus() + ", diagnostics=" +
containerStatus.getDiagnostics());
switch (containerStatus.getExitStatus()) {
case YARN_SUCCESS_EXIT_STATUS:
successfulCount.incrementAndGet();
break;
case YARN_ABORT_EXIT_STATUS:
break; // not success or fail
default:
failedCount.incrementAndGet();
break;
}
completedCount.incrementAndGet();
}
if (completedCount.get() == containersToLaunch) {
done = true;
LOG.info("All container compeleted. done = " + done);
} else {
LOG.info("After completion of one conatiner. current status is:" +
" completedCount :" + completedCount.get() +
" containersToLaunch :" + containersToLaunch +
" successfulCount :" + successfulCount.get() +
" failedCount :" + failedCount.get());
}
}
@Override
public void onContainersAllocated(List<Container> allocatedContainers) {
LOG.info("Got response from RM for container ask, allocatedCnt=" +
allocatedContainers.size());
allocatedCount.addAndGet(allocatedContainers.size());
LOG.info("Total allocated # of container so far : " +
allocatedCount.get() +
" allocated out of " + containersToLaunch + " required.");
startContainerLaunchingThreads(allocatedContainers);
}
@Override
public void onShutdownRequest() {
done = true;
}
@Override
public void onNodesUpdated(List<NodeReport> updatedNodes) {
}
@Override
public float getProgress() {
// set progress to deliver to RM on next heartbeat
float progress = (float) completedCount.get() /
containersToLaunch;
return progress;
}
@Override
public void onError(Throwable e) {
done = true;
amRMClient.stop();
}
}
/**
* CallbackHandler to process NM async calls
*/
private class NMCallbackHandler implements NMClientAsync.CallbackHandler {
/** List of containers */
private ConcurrentMap<ContainerId, Container> containers =
new ConcurrentHashMap<ContainerId, Container>();
/**
* Add a container
* @param containerId id of container
* @param container container object
* @return
*/
public void addContainer(ContainerId containerId, Container container) {
containers.putIfAbsent(containerId, container);
}
@Override
public void onContainerStopped(ContainerId containerId) {
if (LOG.isDebugEnabled()) {
LOG.debug("Succeeded to stop Container " + containerId);
}
containers.remove(containerId);
}
@Override
public void onContainerStatusReceived(ContainerId containerId,
ContainerStatus containerStatus) {
if (LOG.isDebugEnabled()) {
LOG.debug("Container Status: id=" + containerId + ", status=" +
containerStatus);
}
}
@Override
public void onContainerStarted(ContainerId containerId,
Map<String, ByteBuffer> allServiceResponse) {
if (LOG.isDebugEnabled()) {
LOG.debug("Succeeded to start Container " + containerId);
}
Container container = containers.get(containerId);
if (container != null) {
nmClientAsync.getContainerStatusAsync(containerId,
container.getNodeId());
}
}
@Override
public void onStartContainerError(ContainerId containerId, Throwable t) {
LOG.error("Failed to start Container " + containerId, t);
containers.remove(containerId);
}
@Override
public void onGetContainerStatusError(
ContainerId containerId, Throwable t) {
LOG.error("Failed to query the status of Container " + containerId, t);
}
@Override
public void onStopContainerError(ContainerId containerId, Throwable t) {
LOG.error("Failed to stop Container " + containerId);
containers.remove(containerId);
}
}
}