/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.tez.mapreduce.hadoop;
import java.io.File;
import java.io.IOException;
import java.util.Map;
import java.util.Vector;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.tez.common.TezUtils;
import org.apache.tez.common.TezYARNUtils;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.mapreduce.combine.MRCombiner;
import org.apache.tez.mapreduce.partition.MRPartitioner;
import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
/**
* This class contains helper methods for frameworks which migrate from MapReduce to Tez, and need
* to continue to work with existing MapReduce configurations.
*/
@Public
@Evolving
public class MRHelpers {
private static final Log LOG = LogFactory.getLog(MRHelpers.class);
/**
* Translate MapReduce configuration keys to the equivalent Tez keys in the provided
* configuration. The translation is done in place. </p>
* This method is meant to be used by frameworks which rely upon existing MapReduce configuration
* instead of setting up their own.
*
* @param conf mr based configuration to be translated to tez
*/
public static void translateMRConfToTez(Configuration conf) {
convertVertexConfToTez(conf);
}
/**
* Update the provided configuration to use the new API (mapreduce) or the old API (mapred) based
* on the configured InputFormat, OutputFormat, Partitioner etc. Also ensures that keys not
* required by a particular mode are not present. </p>
*
* This method should be invoked after completely setting up the configuration. </p>
*
* Defaults to using the new API if relevant keys are not present.
*
*/
public static void configureMRApiUsage(Configuration conf) {
String oldMapperClass = "mapred.mapper.class";
conf.setBooleanIfUnset("mapred.mapper.new-api", conf.get(oldMapperClass) == null);
try {
if (conf.getBoolean("mapred.mapper.new-api", false)) {
String mode = "new map API";
ensureNotSet(conf, "mapred.input.format.class", mode);
ensureNotSet(conf, oldMapperClass, mode);
} else {
String mode = "map compatability";
ensureNotSet(conf, MRJobConfig.INPUT_FORMAT_CLASS_ATTR, mode);
ensureNotSet(conf, MRJobConfig.MAP_CLASS_ATTR, mode);
}
} catch (IOException e) {
throw new TezUncheckedException(e);
}
}
private static void convertVertexConfToTez(Configuration vertexConf) {
setStageKeysFromBaseConf(vertexConf, vertexConf, "unknown");
processDirectConversion(vertexConf);
setupMRComponents(vertexConf);
}
private static void setupMRComponents(Configuration conf) {
if (conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS) == null) {
conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS,
MRPartitioner.class.getName());
}
if (conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINER_CLASS) == null) {
boolean useNewApi = conf.getBoolean("mapred.mapper.new-api", false);
if (useNewApi) {
if (conf.get(MRJobConfig.COMBINE_CLASS_ATTR) != null) {
conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
}
} else {
if (conf.get("mapred.combiner.class") != null) {
conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
}
}
}
}
/**
* Pulls in specific keys from the base configuration, if they are not set at
* the stage level. An explicit list of keys is copied over (not all), which
* require translation to tez keys.
*/
private static void setStageKeysFromBaseConf(Configuration conf,
Configuration baseConf, String stage) {
// Don't clobber explicit tez config.
JobConf jobConf = null;
if (conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS) == null) {
// If this is set, but the comparator is not set, and their types differ -
// the job will break.
if (conf.get(MRJobConfig.MAP_OUTPUT_KEY_CLASS) == null) {
// Pull this in from the baseConf
// Create jobConf only if required.
jobConf = new JobConf(baseConf);
conf.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS, jobConf
.getMapOutputKeyClass().getName());
if (LOG.isDebugEnabled()) {
LOG.debug("Setting " + MRJobConfig.MAP_OUTPUT_KEY_CLASS
+ " for stage: " + stage
+ " based on job level configuration. Value: "
+ conf.get(MRJobConfig.MAP_OUTPUT_KEY_CLASS));
}
}
}
if (conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS) == null) {
if (conf.get(MRJobConfig.MAP_OUTPUT_VALUE_CLASS) == null) {
if (jobConf == null) {
// Create jobConf if not already created
jobConf = new JobConf(baseConf);
}
conf.set(MRJobConfig.MAP_OUTPUT_VALUE_CLASS, jobConf
.getMapOutputValueClass().getName());
if (LOG.isDebugEnabled()) {
LOG.debug("Setting " + MRJobConfig.MAP_OUTPUT_VALUE_CLASS
+ " for stage: " + stage
+ " based on job level configuration. Value: "
+ conf.get(MRJobConfig.MAP_OUTPUT_VALUE_CLASS));
}
}
}
}
private static void processDirectConversion(Configuration conf) {
for (Map.Entry<String, String> dep : DeprecatedKeys.getMRToTezRuntimeParamMap().entrySet()) {
if (conf.get(dep.getKey()) != null) {
// TODO Deprecation reason does not seem to reflect in the config ?
// The ordering is important in case of keys which are also deprecated.
// Unset will unset the deprecated keys and all it's variants.
final String mrValue = conf.get(dep.getKey());
final String tezValue = conf.get(dep.getValue());
conf.unset(dep.getKey());
if (tezValue == null) {
conf.set(dep.getValue(), mrValue, "TRANSLATED_TO_TEZ");
}
if (LOG.isDebugEnabled()) {
LOG.debug("Config: mr(unset):" + dep.getKey() + ", mr initial value="
+ mrValue + ", tez:" + dep.getValue() + "=" + conf.get(dep.getValue()));
}
}
}
}
private static String getChildLogLevel(Configuration conf, boolean isMap) {
if (isMap) {
return conf.get(
MRJobConfig.MAP_LOG_LEVEL,
JobConf.DEFAULT_LOG_LEVEL.toString()
);
} else {
return conf.get(
MRJobConfig.REDUCE_LOG_LEVEL,
JobConf.DEFAULT_LOG_LEVEL.toString()
);
}
}
private static void ensureNotSet(Configuration conf, String attr, String msg)
throws IOException {
if (conf.get(attr) != null) {
throw new IOException(attr + " is incompatible with " + msg + " mode.");
}
}
private static String getLog4jCmdLineProperties(Configuration conf,
boolean isMap) {
Vector<String> logProps = new Vector<String>(4);
TezUtils.addLog4jSystemProperties(getChildLogLevel(conf, isMap), logProps);
StringBuilder sb = new StringBuilder();
for (String str : logProps) {
sb.append(str).append(" ");
}
return sb.toString();
}
/**
* Generate JVM options based on MapReduce AM java options. </p>
* <p/>
* This is only meant to be used if frameworks are not setting up their own java options or
* relying on the defaults specified by Tez, and instead want to use the options which may already
* have been configured for an MR AppMaster.
*
* @param conf Configuration to be used to extract JVM opts specific info
* @return JAVA_OPTS string to be used in launching the JVM
*/
public static String getJavaOptsForMRAM(Configuration conf) {
// Admin opts
String mrAppMasterAdminOptions = conf.get(
MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS,
MRJobConfig.DEFAULT_MR_AM_ADMIN_COMMAND_OPTS);
// Add AM user command opts
String mrAppMasterUserOptions = conf.get(MRJobConfig.MR_AM_COMMAND_OPTS,
MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS);
return mrAppMasterAdminOptions.trim()
+ " " + mrAppMasterUserOptions.trim();
}
/**
* Generate JVM options based on MapReduce mapper java options. </p>
*
* This is only meant to be used if frameworks are not setting up their own java options,
* and would like to fallback to using java options which may already be configured for
* Hadoop MapReduce mappers.
*
* Uses mapreduce.admin.map.child.java.opts, mapreduce.map.java.opts and
* mapreduce.map.log.level from config to generate the opts.
*
* @param conf Configuration to be used to extract JVM opts specific info
* @return JAVA_OPTS string to be used in launching the JVM
*/
@SuppressWarnings("deprecation")
public static String getJavaOptsForMRMapper(Configuration conf) {
String adminOpts = conf.get(
MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS,
MRJobConfig.DEFAULT_MAPRED_ADMIN_JAVA_OPTS);
String userOpts = conf.get(
MRJobConfig.MAP_JAVA_OPTS,
conf.get(
JobConf.MAPRED_TASK_JAVA_OPTS,
JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS));
return adminOpts.trim() + " " + userOpts.trim() + " "
+ getLog4jCmdLineProperties(conf, true);
}
/**
* Generate JVM options based on MapReduce reducer java options. </p>
*
* This is only meant to be used if frameworks are not setting up their own java options,
* and would like to fallback to using java options which may already be configured for
* Hadoop MapReduce reducers.
*
* Uses mapreduce.admin.reduce.child.java.opts, mapreduce.reduce.java.opts
* and mapreduce.reduce.log.level from config to generate the opts.
*
* @param conf Configuration to be used to extract JVM opts specific info
* @return JAVA_OPTS string to be used in launching the JVM
*/
@SuppressWarnings("deprecation")
public static String getJavaOptsForMRReducer(Configuration conf) {
String adminOpts = conf.get(
MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS,
MRJobConfig.DEFAULT_MAPRED_ADMIN_JAVA_OPTS);
String userOpts = conf.get(
MRJobConfig.REDUCE_JAVA_OPTS,
conf.get(
JobConf.MAPRED_TASK_JAVA_OPTS,
JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS));
return adminOpts.trim() + " " + userOpts.trim() + " "
+ getLog4jCmdLineProperties(conf, false);
}
/**
* Extract the container resource requirements from the provided configuration, which would
* otherwise have been used when running a Hadoop MapReduce mapper. </p>
* <p/>
* This is only meant to be used if frameworks are not setting up their own {@link
* org.apache.hadoop.yarn.api.records.Resource} and would like to fallback to using resources
* which may already be configured for Hadoop MapReduce mappers.
*
* @param conf Configuration with MR specific settings used to extract
* information from
*
* @return Resource object used to define requirements for containers
* running Map tasks
*/
public static Resource getResourceForMRMapper(Configuration conf) {
return Resource.newInstance(conf.getInt(
MRJobConfig.MAP_MEMORY_MB, MRJobConfig.DEFAULT_MAP_MEMORY_MB),
conf.getInt(MRJobConfig.MAP_CPU_VCORES,
MRJobConfig.DEFAULT_MAP_CPU_VCORES));
}
/**
* Extract the container resource requirements from the provided configuration, which would
* otherwise have been used when running a Hadoop MapReduce reducer. </p>
* <p/>
* This is only meant to be used if frameworks are not setting up their own {@link
* org.apache.hadoop.yarn.api.records.Resource} and would like to fallback to using resources
* which may already be configured for Hadoop MapReduce reducers.
* <p/>
* Uses mapreduce.reduce.memory.mb and mapreduce.reduce.cpu.vcores from the
* provided configuration.
*
* @param conf Configuration with MR specific settings used to extract
* information from
* @return Resource object used to define requirements for containers
* running Reduce tasks
*/
public static Resource getResourceForMRReducer(Configuration conf) {
return Resource.newInstance(conf.getInt(
MRJobConfig.REDUCE_MEMORY_MB, MRJobConfig.DEFAULT_REDUCE_MEMORY_MB),
conf.getInt(MRJobConfig.REDUCE_CPU_VCORES,
MRJobConfig.DEFAULT_REDUCE_CPU_VCORES));
}
/**
* Setup classpath and other environment variables based on the configured values for MR Mappers
* or Reducers
*
* @param conf Configuration to retrieve settings from
* @param environment Environment to update
* @param isMap Whether task is a map or reduce task
*/
public static void updateEnvBasedOnMRTaskEnv(Configuration conf,
Map<String, String> environment, boolean isMap) {
// Shell
environment.put(Environment.SHELL.name(), conf.get(
MRJobConfig.MAPRED_ADMIN_USER_SHELL, MRJobConfig.DEFAULT_SHELL));
// Add pwd to LD_LIBRARY_PATH, add this before adding anything else
TezYARNUtils.addToEnvironment(environment, Environment.LD_LIBRARY_PATH.name(),
Environment.PWD.$(), File.pathSeparator);
// Add the env variables passed by the admin
TezYARNUtils.appendToEnvFromInputString(environment, conf.get(
MRJobConfig.MAPRED_ADMIN_USER_ENV,
MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV),
File.pathSeparator);
// Add the env variables passed by the user
String mapredChildEnv = (isMap ?
conf.get(MRJobConfig.MAP_ENV, "")
: conf.get(MRJobConfig.REDUCE_ENV, ""));
TezYARNUtils.appendToEnvFromInputString(environment, mapredChildEnv, File.pathSeparator);
// Set logging level in the environment.
environment.put(
"HADOOP_ROOT_LOGGER",
getChildLogLevel(conf, isMap) + ",CLA");
}
/**
* Setup environment variables based on the configured values for the MR AM
* @param conf Configuration from which to extract information
* @param environment Environment map to update
*/
public static void updateEnvBasedOnMRAMEnv(Configuration conf, Map<String, String> environment) {
TezYARNUtils.appendToEnvFromInputString(environment, conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV),
File.pathSeparator);
TezYARNUtils.appendToEnvFromInputString(environment, conf.get(MRJobConfig.MR_AM_ENV),
File.pathSeparator);
}
}