package org.apache.helix.provisioning.yarn;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Vector;
import java.util.concurrent.Executors;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.helix.HelixManager;
import org.apache.helix.api.Cluster;
import org.apache.helix.api.Participant;
import org.apache.helix.api.config.ContainerConfig;
import org.apache.helix.api.config.ResourceConfig;
import org.apache.helix.api.id.ParticipantId;
import org.apache.helix.api.id.ResourceId;
import org.apache.helix.controller.provisioner.ContainerId;
import org.apache.helix.controller.provisioner.ContainerProvider;
import org.apache.helix.controller.provisioner.ContainerSpec;
import org.apache.helix.controller.provisioner.ContainerState;
import org.apache.helix.controller.provisioner.Provisioner;
import org.apache.helix.controller.provisioner.TargetProvider;
import org.apache.helix.controller.provisioner.TargetProviderResponse;
import org.apache.helix.provisioning.ApplicationSpec;
import org.apache.helix.provisioning.ContainerAskResponse;
import org.apache.helix.provisioning.ContainerLaunchResponse;
import org.apache.helix.provisioning.ContainerReleaseResponse;
import org.apache.helix.provisioning.ContainerStopResponse;
import org.apache.helix.provisioning.ParticipantLauncher;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
public class YarnProvisioner implements Provisioner, TargetProvider, ContainerProvider {
private static final Log LOG = LogFactory.getLog(YarnProvisioner.class);
static GenericApplicationMaster applicationMaster;
static ListeningExecutorService service = MoreExecutors.listeningDecorator(Executors
.newCachedThreadPool());
public static AppMasterConfig applicationMasterConfig;
public static ApplicationSpec applicationSpec;
Map<ContainerId, Container> allocatedContainersMap = new HashMap<ContainerId, Container>();
// private HelixManager _helixManager;
private ResourceConfig _resourceConfig;
public YarnProvisioner() {
}
@Override
public ListenableFuture<ContainerId> allocateContainer(ContainerSpec spec) {
ContainerRequest containerAsk = setupContainerAskForRM(spec);
ListenableFuture<ContainerAskResponse> requestNewContainer =
applicationMaster.acquireContainer(containerAsk);
return Futures.transform(requestNewContainer,
new Function<ContainerAskResponse, ContainerId>() {
@Override
public ContainerId apply(ContainerAskResponse containerAskResponse) {
ContainerId helixContainerId =
ContainerId.from(containerAskResponse.getContainer().getId().toString());
allocatedContainersMap.put(helixContainerId, containerAskResponse.getContainer());
return helixContainerId;
}
});
}
@Override
public ListenableFuture<Boolean> deallocateContainer(final ContainerId containerId) {
ListenableFuture<ContainerReleaseResponse> releaseContainer =
applicationMaster.releaseContainer(allocatedContainersMap.get(containerId));
return Futures.transform(releaseContainer, new Function<ContainerReleaseResponse, Boolean>() {
@Override
public Boolean apply(ContainerReleaseResponse response) {
return response != null;
}
}, service);
}
@Override
public ListenableFuture<Boolean> startContainer(final ContainerId containerId,
Participant participant) {
Container container = allocatedContainersMap.get(containerId);
ContainerLaunchContext launchContext;
try {
launchContext = createLaunchContext(containerId, container, participant);
} catch (Exception e) {
LOG.error("Exception while creating context to launch container:" + containerId, e);
return null;
}
ListenableFuture<ContainerLaunchResponse> future =
applicationMaster.launchContainer(container, launchContext);
return Futures.transform(future, new Function<ContainerLaunchResponse, Boolean>() {
@Override
public Boolean apply(ContainerLaunchResponse response) {
return response != null;
}
}, service);
}
private ContainerLaunchContext createLaunchContext(ContainerId containerId, Container container,
Participant participant) throws Exception {
ContainerLaunchContext participantContainer = Records.newRecord(ContainerLaunchContext.class);
// Map<String, String> envs = System.getenv();
String appName = applicationMasterConfig.getAppName();
int appId = applicationMasterConfig.getAppId();
String serviceName = _resourceConfig.getId().stringify();
String serviceClasspath = applicationMasterConfig.getClassPath(serviceName);
String mainClass = applicationMasterConfig.getMainClass(serviceName);
String zkAddress = applicationMasterConfig.getZKAddress();
// set the localresources needed to launch container
Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
LocalResource servicePackageResource = Records.newRecord(LocalResource.class);
YarnConfiguration conf = new YarnConfiguration();
FileSystem fs;
fs = FileSystem.get(conf);
String pathSuffix = appName + "/" + appId + "/" + serviceName + ".tar";
Path dst = new Path(fs.getHomeDirectory(), pathSuffix);
FileStatus destStatus = fs.getFileStatus(dst);
// Set the type of resource - file or archive
// archives are untarred at destination
// we don't need the jar file to be untarred for now
servicePackageResource.setType(LocalResourceType.ARCHIVE);
// Set visibility of the resource
// Setting to most private option
servicePackageResource.setVisibility(LocalResourceVisibility.APPLICATION);
// Set the resource to be copied over
servicePackageResource.setResource(ConverterUtils.getYarnUrlFromPath(dst));
// Set timestamp and length of file so that the framework
// can do basic sanity checks for the local resource
// after it has been copied over to ensure it is the same
// resource the client intended to use with the application
servicePackageResource.setTimestamp(destStatus.getModificationTime());
servicePackageResource.setSize(destStatus.getLen());
LOG.info("Setting local resource:" + servicePackageResource + " for service" + serviceName);
localResources.put(serviceName, servicePackageResource);
// Set local resource info into app master container launch context
participantContainer.setLocalResources(localResources);
// Set the necessary security tokens as needed
// amContainer.setContainerTokens(containerToken);
// Set the env variables to be setup in the env where the application master will be run
LOG.info("Set the environment for the application master");
Map<String, String> env = new HashMap<String, String>();
env.put(serviceName, dst.getName());
// Add AppMaster.jar location to classpath
// At some point we should not be required to add
// the hadoop specific classpaths to the env.
// It should be provided out of the box.
// For now setting all required classpaths including
// the classpath to "." for the application jar
StringBuilder classPathEnv =
new StringBuilder(Environment.CLASSPATH.$()).append(File.pathSeparatorChar).append("./*");
classPathEnv.append(File.pathSeparatorChar);
classPathEnv.append(serviceClasspath);
for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
classPathEnv.append(File.pathSeparatorChar);
classPathEnv.append(c.trim());
}
classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties");
LOG.info("Setting classpath for service:\n" + classPathEnv.toString());
env.put("CLASSPATH", classPathEnv.toString());
participantContainer.setEnvironment(env);
if (applicationMaster.allTokens != null) {
LOG.info("Setting tokens: " + applicationMaster.allTokens);
participantContainer.setTokens(applicationMaster.allTokens);
}
// Set the necessary command to execute the application master
Vector<CharSequence> vargs = new Vector<CharSequence>(30);
// Set java executable command
LOG.info("Setting up app master command");
vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
// Set Xmx based on am memory size
vargs.add("-Xmx" + 4096 + "m");
// Set class name
vargs.add(ParticipantLauncher.class.getCanonicalName());
// Set params for container participant
vargs.add("--zkAddress " + zkAddress);
vargs.add("--cluster " + appName);
vargs.add("--participantId " + participant.getId().stringify());
vargs.add("--participantClass " + mainClass);
vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/ContainerParticipant.stdout");
vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/ContainerParticipant.stderr");
// Get final commmand
StringBuilder command = new StringBuilder();
for (CharSequence str : vargs) {
command.append(str).append(" ");
}
LOG.info("Completed setting up container launch command " + command.toString()
+ " with arguments \n" + vargs);
List<String> commands = new ArrayList<String>();
commands.add(command.toString());
participantContainer.setCommands(commands);
return participantContainer;
}
@Override
public ListenableFuture<Boolean> stopContainer(final ContainerId containerId) {
Container container = allocatedContainersMap.get(containerId);
ListenableFuture<ContainerStopResponse> future = applicationMaster.stopContainer(container);
return Futures.transform(future, new Function<ContainerStopResponse, Boolean>() {
@Override
public Boolean apply(ContainerStopResponse response) {
return response != null;
}
}, service);
}
@Override
public void init(HelixManager helixManager, ResourceConfig resourceConfig) {
// _helixManager = helixManager;
_resourceConfig = resourceConfig;
}
@Override
public TargetProviderResponse evaluateExistingContainers(Cluster cluster, ResourceId resourceId,
Collection<Participant> participants) {
TargetProviderResponse response = new TargetProviderResponse();
// ask for two containers at a time
List<ContainerSpec> containersToAcquire = Lists.newArrayList();
List<Participant> containersToStart = Lists.newArrayList();
List<Participant> containersToRelease = Lists.newArrayList();
List<Participant> containersToStop = Lists.newArrayList();
YarnProvisionerConfig provisionerConfig =
(YarnProvisionerConfig) cluster.getConfig().getResourceMap().get(resourceId)
.getProvisionerConfig();
int targetNumContainers = provisionerConfig.getNumContainers();
// Any container that is in a state should be put in this set
Set<ParticipantId> existingContainersIdSet = new HashSet<ParticipantId>();
// Cache halted containers to determine which to restart and which to release
Map<ParticipantId, Participant> excessHaltedContainers = Maps.newHashMap();
// Cache participants to ensure that excess participants are stopped
Map<ParticipantId, Participant> excessActiveContainers = Maps.newHashMap();
for (Participant participant : participants) {
ContainerConfig containerConfig = participant.getContainerConfig();
if (containerConfig != null && containerConfig.getState() != null) {
ContainerState state = containerConfig.getState();
switch (state) {
case ACQUIRING:
existingContainersIdSet.add(participant.getId());
break;
case ACQUIRED:
// acquired containers are ready to start
existingContainersIdSet.add(participant.getId());
containersToStart.add(participant);
break;
case CONNECTING:
existingContainersIdSet.add(participant.getId());
break;
case CONNECTED:
// active containers can be stopped or kept active
existingContainersIdSet.add(participant.getId());
excessActiveContainers.put(participant.getId(), participant);
break;
case DISCONNECTED:
// disconnected containers must be stopped
existingContainersIdSet.add(participant.getId());
containersToStop.add(participant);
case HALTING:
existingContainersIdSet.add(participant.getId());
break;
case HALTED:
// halted containers can be released or restarted
existingContainersIdSet.add(participant.getId());
excessHaltedContainers.put(participant.getId(), participant);
break;
case FINALIZING:
existingContainersIdSet.add(participant.getId());
break;
case FINALIZED:
break;
case FAILED:
// remove the failed instance
// _helixManager.getClusterManagmentTool().dropInstance(cluster.getId().toString(),
// new InstanceConfig(participant.getId()));
excessHaltedContainers.put(participant.getId(), participant);
break;
default:
break;
}
}
}
for (int i = 0; i < targetNumContainers; i++) {
ParticipantId participantId = ParticipantId.from(resourceId + "_container_" + (i));
excessActiveContainers.remove(participantId); // don't stop this container if active
if (excessHaltedContainers.containsKey(participantId)) {
// Halted containers can be restarted if necessary
// Participant participant = excessHaltedContainers.get(participantId);
// containersToStart.add(participant);
// excessHaltedContainers.remove(participantId); // don't release this container
} else if (!existingContainersIdSet.contains(participantId)) {
// Unallocated containers must be allocated
ContainerSpec containerSpec = new ContainerSpec(participantId);
int mem = 4096;
if (_resourceConfig.getUserConfig() != null) {
mem = _resourceConfig.getUserConfig().getIntField("memory", mem);
}
containerSpec.setMemory(mem);
containersToAcquire.add(containerSpec);
}
}
// Add all the containers that should be stopped because they fall outside the target range
containersToStop.addAll(excessActiveContainers.values());
// Add halted containers that should not be restarted
containersToRelease.addAll(excessHaltedContainers.values());
response.setContainersToAcquire(containersToAcquire);
response.setContainersToStart(containersToStart);
response.setContainersToRelease(containersToRelease);
response.setContainersToStop(containersToStop);
LOG.info("target provider response containers to acquire:" + response.getContainersToAcquire());
LOG.info("target provider response containers to start:" + response.getContainersToStart());
LOG.info("target provider response containers to stop:" + response.getContainersToStop());
LOG.info("target provider response containers to release:" + response.getContainersToRelease());
return response;
}
private ContainerRequest setupContainerAskForRM(ContainerSpec spec) {
// setup requirements for hosts
// using * as any host will do for the distributed shell app
// set the priority for the request
Priority pri = Records.newRecord(Priority.class);
int requestPriority = 0;
// TODO - what is the range for priority? how to decide?
pri.setPriority(requestPriority);
// Set up resource type requirements
// For now, only memory is supported so we set memory requirements
Resource capability = Records.newRecord(Resource.class);
int memory = spec.getMemory();
capability.setMemory(memory);
ContainerRequest request = new ContainerRequest(capability, null, null, pri);
LOG.info("Requested container ask: " + request.toString());
return request;
}
@Override
public ContainerProvider getContainerProvider() {
return this;
}
@Override
public TargetProvider getTargetProvider() {
return this;
}
}