/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.giraph.conf;
import org.apache.giraph.aggregators.AggregatorWriter;
import org.apache.giraph.combiner.MessageCombiner;
import org.apache.giraph.edge.OutEdges;
import org.apache.giraph.edge.ReuseObjectsOutEdges;
import org.apache.giraph.factories.ComputationFactory;
import org.apache.giraph.graph.Vertex;
import org.apache.giraph.graph.VertexValueCombiner;
import org.apache.giraph.graph.VertexResolver;
import org.apache.giraph.factories.VertexValueFactory;
import org.apache.giraph.graph.Computation;
import org.apache.giraph.io.EdgeInputFormat;
import org.apache.giraph.io.EdgeOutputFormat;
import org.apache.giraph.io.VertexInputFormat;
import org.apache.giraph.io.VertexOutputFormat;
import org.apache.giraph.io.filters.EdgeInputFilter;
import org.apache.giraph.io.filters.VertexInputFilter;
import org.apache.giraph.job.GiraphJobObserver;
import org.apache.giraph.job.GiraphJobRetryChecker;
import org.apache.giraph.master.MasterCompute;
import org.apache.giraph.master.MasterObserver;
import org.apache.giraph.partition.GraphPartitionerFactory;
import org.apache.giraph.partition.Partition;
import org.apache.giraph.partition.ReusesObjectsPartition;
import org.apache.giraph.utils.ReflectionUtils;
import org.apache.giraph.worker.WorkerContext;
import org.apache.giraph.worker.WorkerObserver;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.net.DNS;
import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.PooledByteBufAllocator;
import io.netty.buffer.UnpooledByteBufAllocator;
import java.net.UnknownHostException;
/**
* Adds user methods specific to Giraph. This will be put into an
* ImmutableClassesGiraphConfiguration that provides the configuration plus
* the immutable classes.
*
* Keeps track of parameters which were set so it easily set them in another
* copy of configuration.
*/
public class GiraphConfiguration extends Configuration
implements GiraphConstants {
/** ByteBufAllocator to be used by netty */
private ByteBufAllocator nettyBufferAllocator = null;
/**
* Constructor that creates the configuration
*/
public GiraphConfiguration() {
configureHadoopSecurity();
}
/**
* Constructor.
*
* @param conf Configuration
*/
public GiraphConfiguration(Configuration conf) {
super(conf);
configureHadoopSecurity();
}
/**
* Get name of computation being run. We leave this up to the
* {@link ComputationFactory} to decide what to return.
*
* @return Name of computation being run
*/
public String getComputationName() {
ComputationFactory compFactory = ReflectionUtils.newInstance(
getComputationFactoryClass());
return compFactory.computationName(this);
}
/**
* Get the user's subclassed {@link ComputationFactory}
*
* @return User's computation factory class
*/
public Class<? extends ComputationFactory> getComputationFactoryClass() {
return COMPUTATION_FACTORY_CLASS.get(this);
}
/**
* Get the user's subclassed {@link Computation}
*
* @return User's computation class
*/
public Class<? extends Computation> getComputationClass() {
return COMPUTATION_CLASS.get(this);
}
/**
* Set the computation class (required)
*
* @param computationClass Runs vertex computation
*/
public void setComputationClass(
Class<? extends Computation> computationClass) {
COMPUTATION_CLASS.set(this, computationClass);
}
/**
* Set the vertex value factory class
*
* @param vertexValueFactoryClass Creates default vertex values
*/
public final void setVertexValueFactoryClass(
Class<? extends VertexValueFactory> vertexValueFactoryClass) {
VERTEX_VALUE_FACTORY_CLASS.set(this, vertexValueFactoryClass);
}
/**
* Set the edge input filter class
*
* @param edgeFilterClass class to use
*/
public void setEdgeInputFilterClass(
Class<? extends EdgeInputFilter> edgeFilterClass) {
EDGE_INPUT_FILTER_CLASS.set(this, edgeFilterClass);
}
/**
* Set the vertex input filter class
*
* @param vertexFilterClass class to use
*/
public void setVertexInputFilterClass(
Class<? extends VertexInputFilter> vertexFilterClass) {
VERTEX_INPUT_FILTER_CLASS.set(this, vertexFilterClass);
}
/**
* Get the vertex edges class
*
* @return vertex edges class
*/
public Class<? extends OutEdges> getOutEdgesClass() {
return VERTEX_EDGES_CLASS.get(this);
}
/**
* Set the vertex edges class
*
* @param outEdgesClass Determines the way edges are stored
*/
public final void setOutEdgesClass(
Class<? extends OutEdges> outEdgesClass) {
VERTEX_EDGES_CLASS.set(this, outEdgesClass);
}
/**
* Set the vertex implementation class
*
* @param vertexClass class of the vertex implementation
*/
public final void setVertexClass(Class<? extends Vertex> vertexClass) {
VERTEX_CLASS.set(this, vertexClass);
}
/**
* Set the vertex edges class used during edge-based input (if different
* from the one used during computation)
*
* @param inputOutEdgesClass Determines the way edges are stored
*/
public final void setInputOutEdgesClass(
Class<? extends OutEdges> inputOutEdgesClass) {
INPUT_VERTEX_EDGES_CLASS.set(this, inputOutEdgesClass);
}
/**
* True if the {@link org.apache.giraph.edge.OutEdges} implementation
* copies the passed edges to its own data structure,
* i.e. it doesn't keep references to Edge objects, target vertex ids or edge
* values passed to add() or initialize().
* This makes it possible to reuse edge objects passed to the data
* structure, to minimize object instantiation (see for example
* EdgeStore#addPartitionEdges()).
*
* @return True iff we can reuse the edge objects
*/
public boolean reuseEdgeObjects() {
return ReuseObjectsOutEdges.class.isAssignableFrom(
getOutEdgesClass());
}
/**
* True if the {@link Partition} implementation copies the passed vertices
* to its own data structure, i.e. it doesn't keep references to Vertex
* objects passed to it.
* This makes it possible to reuse vertex objects passed to the data
* structure, to minimize object instantiation.
*
* @return True iff we can reuse the vertex objects
*/
public boolean reuseVertexObjects() {
return ReusesObjectsPartition.class.isAssignableFrom(getPartitionClass());
}
/**
* Get Partition class used
* @return Partition class
*/
public Class<? extends Partition> getPartitionClass() {
return PARTITION_CLASS.get(this);
}
/**
* Does the job have a {@link VertexInputFormat}?
*
* @return True iff a {@link VertexInputFormat} has been specified.
*/
public boolean hasVertexInputFormat() {
return VERTEX_INPUT_FORMAT_CLASS.get(this) != null;
}
/**
* Set the vertex input format class (required)
*
* @param vertexInputFormatClass Determines how graph is input
*/
public void setVertexInputFormatClass(
Class<? extends VertexInputFormat> vertexInputFormatClass) {
VERTEX_INPUT_FORMAT_CLASS.set(this, vertexInputFormatClass);
}
/**
* Does the job have a {@link EdgeInputFormat}?
*
* @return True iff a {@link EdgeInputFormat} has been specified.
*/
public boolean hasEdgeInputFormat() {
return EDGE_INPUT_FORMAT_CLASS.get(this) != null;
}
/**
* Set the edge input format class (required)
*
* @param edgeInputFormatClass Determines how graph is input
*/
public void setEdgeInputFormatClass(
Class<? extends EdgeInputFormat> edgeInputFormatClass) {
EDGE_INPUT_FORMAT_CLASS.set(this, edgeInputFormatClass);
}
/**
* Set the master class (optional)
*
* @param masterComputeClass Runs master computation
*/
public final void setMasterComputeClass(
Class<? extends MasterCompute> masterComputeClass) {
MASTER_COMPUTE_CLASS.set(this, masterComputeClass);
}
/**
* Add a MasterObserver class (optional)
*
* @param masterObserverClass MasterObserver class to add.
*/
public final void addMasterObserverClass(
Class<? extends MasterObserver> masterObserverClass) {
MASTER_OBSERVER_CLASSES.add(this, masterObserverClass);
}
/**
* Add a WorkerObserver class (optional)
*
* @param workerObserverClass WorkerObserver class to add.
*/
public final void addWorkerObserverClass(
Class<? extends WorkerObserver> workerObserverClass) {
WORKER_OBSERVER_CLASSES.add(this, workerObserverClass);
}
/**
* Get job observer class
*
* @return GiraphJobObserver class set.
*/
public Class<? extends GiraphJobObserver> getJobObserverClass() {
return JOB_OBSERVER_CLASS.get(this);
}
/**
* Set job observer class
*
* @param klass GiraphJobObserver class to set.
*/
public void setJobObserverClass(Class<? extends GiraphJobObserver> klass) {
JOB_OBSERVER_CLASS.set(this, klass);
}
/**
* Get job retry checker class
*
* @return GiraphJobRetryChecker class set.
*/
public Class<? extends GiraphJobRetryChecker> getJobRetryCheckerClass() {
return JOB_RETRY_CHECKER_CLASS.get(this);
}
/**
* Set job retry checker class
*
* @param klass GiraphJobRetryChecker class to set.
*/
public void setJobRetryCheckerClass(
Class<? extends GiraphJobRetryChecker> klass) {
JOB_RETRY_CHECKER_CLASS.set(this, klass);
}
/**
* Check whether to enable jmap dumping thread.
*
* @return true if jmap dumper is enabled.
*/
public boolean isJMapHistogramDumpEnabled() {
return JMAP_ENABLE.get(this);
}
/**
* Check whether to enable heap memory supervisor thread
*
* @return true if jmap dumper is reactively enabled
*/
public boolean isReactiveJmapHistogramDumpEnabled() {
return REACTIVE_JMAP_ENABLE.get(this);
}
/**
* Set mapping from a key name to a list of classes.
*
* @param name String key name to use.
* @param xface interface of the classes being set.
* @param klasses Classes to set.
*/
public final void setClasses(String name, Class<?> xface,
Class<?> ... klasses) {
String[] klassNames = new String[klasses.length];
for (int i = 0; i < klasses.length; ++i) {
Class<?> klass = klasses[i];
if (!xface.isAssignableFrom(klass)) {
throw new RuntimeException(klass + " does not implement " +
xface.getName());
}
klassNames[i] = klasses[i].getName();
}
setStrings(name, klassNames);
}
/**
* Does the job have a {@link VertexOutputFormat}?
*
* @return True iff a {@link VertexOutputFormat} has been specified.
*/
public boolean hasVertexOutputFormat() {
return VERTEX_OUTPUT_FORMAT_CLASS.get(this) != null;
}
/**
* Set the vertex output format class (optional)
*
* @param vertexOutputFormatClass Determines how graph is output
*/
public final void setVertexOutputFormatClass(
Class<? extends VertexOutputFormat> vertexOutputFormatClass) {
VERTEX_OUTPUT_FORMAT_CLASS.set(this, vertexOutputFormatClass);
}
/**
* Does the job have a {@link EdgeOutputFormat} subdir?
*
* @return True iff a {@link EdgeOutputFormat} subdir has been specified.
*/
public boolean hasVertexOutputFormatSubdir() {
return !VERTEX_OUTPUT_FORMAT_SUBDIR.get(this).isEmpty();
}
/**
* Set the vertex output format path
*
* @param path path where the verteces will be written
*/
public final void setVertexOutputFormatSubdir(String path) {
VERTEX_OUTPUT_FORMAT_SUBDIR.set(this, path);
}
/**
* Check if output should be done during computation
*
* @return True iff output should be done during computation
*/
public final boolean doOutputDuringComputation() {
return DO_OUTPUT_DURING_COMPUTATION.get(this);
}
/**
* Set whether or not we should do output during computation
*
* @param doOutputDuringComputation True iff we want output to happen
* during computation
*/
public final void setDoOutputDuringComputation(
boolean doOutputDuringComputation) {
DO_OUTPUT_DURING_COMPUTATION.set(this, doOutputDuringComputation);
}
/**
* Check if VertexOutputFormat is thread-safe
*
* @return True iff VertexOutputFormat is thread-safe
*/
public final boolean vertexOutputFormatThreadSafe() {
return VERTEX_OUTPUT_FORMAT_THREAD_SAFE.get(this);
}
/**
* Set whether or not selected VertexOutputFormat is thread-safe
*
* @param vertexOutputFormatThreadSafe True iff selected VertexOutputFormat
* is thread-safe
*/
public final void setVertexOutputFormatThreadSafe(
boolean vertexOutputFormatThreadSafe) {
VERTEX_OUTPUT_FORMAT_THREAD_SAFE.set(this, vertexOutputFormatThreadSafe);
}
/**
* Does the job have a {@link EdgeOutputFormat}?
*
* @return True iff a {@link EdgeOutputFormat} has been specified.
*/
public boolean hasEdgeOutputFormat() {
return EDGE_OUTPUT_FORMAT_CLASS.get(this) != null;
}
/**
* Set the edge output format class (optional)
*
* @param edgeOutputFormatClass Determines how graph is output
*/
public final void setEdgeOutputFormatClass(
Class<? extends EdgeOutputFormat> edgeOutputFormatClass) {
EDGE_OUTPUT_FORMAT_CLASS.set(this, edgeOutputFormatClass);
}
/**
* Does the job have a {@link EdgeOutputFormat} subdir?
*
* @return True iff a {@link EdgeOutputFormat} subdir has been specified.
*/
public boolean hasEdgeOutputFormatSubdir() {
return !EDGE_OUTPUT_FORMAT_SUBDIR.get(this).isEmpty();
}
/**
* Set the edge output format path
*
* @param path path where the edges will be written
*/
public final void setEdgeOutputFormatSubdir(String path) {
EDGE_OUTPUT_FORMAT_SUBDIR.set(this, path);
}
/**
* Get the number of threads to use for writing output in the end of the
* application. If output format is not thread safe, returns 1.
*
* @return Number of output threads
*/
public final int getNumOutputThreads() {
if (!vertexOutputFormatThreadSafe()) {
return 1;
} else {
return NUM_OUTPUT_THREADS.get(this);
}
}
/**
* Set the number of threads to use for writing output in the end of the
* application. Will be used only if {#vertexOutputFormatThreadSafe} is true.
*
* @param numOutputThreads Number of output threads
*/
public void setNumOutputThreads(int numOutputThreads) {
NUM_OUTPUT_THREADS.set(this, numOutputThreads);
}
/**
* Get the message combiner class (optional)
*
* @return messageCombinerClass Determines how vertex messages are combined
*/
public Class<? extends MessageCombiner> getMessageCombinerClass() {
return MESSAGE_COMBINER_CLASS.get(this);
}
/**
* Set the message combiner class (optional)
*
* @param messageCombinerClass Determines how vertex messages are combined
*/
public void setMessageCombinerClass(
Class<? extends MessageCombiner> messageCombinerClass) {
MESSAGE_COMBINER_CLASS.set(this, messageCombinerClass);
}
/**
* Set the graph partitioner class (optional)
*
* @param graphPartitionerFactoryClass Determines how the graph is partitioned
*/
public final void setGraphPartitionerFactoryClass(
Class<? extends GraphPartitionerFactory> graphPartitionerFactoryClass) {
GRAPH_PARTITIONER_FACTORY_CLASS.set(this, graphPartitionerFactoryClass);
}
/**
* Set the vertex resolver class (optional)
*
* @param vertexResolverClass Determines how vertex mutations are resolved
*/
public final void setVertexResolverClass(
Class<? extends VertexResolver> vertexResolverClass) {
VERTEX_RESOLVER_CLASS.set(this, vertexResolverClass);
}
/**
* Whether to create a vertex that doesn't exist when it receives messages.
* This only affects DefaultVertexResolver.
*
* @return true if we should create non existent vertices that get messages.
*/
public final boolean getResolverCreateVertexOnMessages() {
return RESOLVER_CREATE_VERTEX_ON_MSGS.get(this);
}
/**
* Set whether to create non existent vertices when they receive messages.
*
* @param v true if we should create vertices when they get messages.
*/
public final void setResolverCreateVertexOnMessages(boolean v) {
RESOLVER_CREATE_VERTEX_ON_MSGS.set(this, v);
}
/**
* Set the vertex value combiner class (optional)
*
* @param vertexValueCombinerClass Determines how vertices are combined
*/
public final void setVertexValueCombinerClass(
Class<? extends VertexValueCombiner> vertexValueCombinerClass) {
VERTEX_VALUE_COMBINER_CLASS.set(this, vertexValueCombinerClass);
}
/**
* Set the worker context class (optional)
*
* @param workerContextClass Determines what code is executed on a each
* worker before and after each superstep and computation
*/
public final void setWorkerContextClass(
Class<? extends WorkerContext> workerContextClass) {
WORKER_CONTEXT_CLASS.set(this, workerContextClass);
}
/**
* Set the aggregator writer class (optional)
*
* @param aggregatorWriterClass Determines how the aggregators are
* written to file at the end of the job
*/
public final void setAggregatorWriterClass(
Class<? extends AggregatorWriter> aggregatorWriterClass) {
AGGREGATOR_WRITER_CLASS.set(this, aggregatorWriterClass);
}
/**
* Set the partition class (optional)
*
* @param partitionClass Determines the why partitions are stored
*/
public final void setPartitionClass(
Class<? extends Partition> partitionClass) {
PARTITION_CLASS.set(this, partitionClass);
}
/**
* Set worker configuration for determining what is required for
* a superstep.
*
* @param minWorkers Minimum workers to do a superstep
* @param maxWorkers Maximum workers to do a superstep
* (max map tasks in job)
* @param minPercentResponded 0 - 100 % of the workers required to
* have responded before continuing the superstep
*/
public final void setWorkerConfiguration(int minWorkers,
int maxWorkers,
float minPercentResponded) {
setInt(MIN_WORKERS, minWorkers);
setInt(MAX_WORKERS, maxWorkers);
MIN_PERCENT_RESPONDED.set(this, minPercentResponded);
}
public final int getMinWorkers() {
return getInt(MIN_WORKERS, -1);
}
public final int getMaxWorkers() {
return getInt(MAX_WORKERS, -1);
}
public final float getMinPercentResponded() {
return MIN_PERCENT_RESPONDED.get(this);
}
/**
* Utilize an existing ZooKeeper service. If this is not set, ZooKeeper
* will be dynamically started by Giraph for this job.
*
* @param serverList Comma separated list of servers and ports
* (i.e. zk1:2221,zk2:2221)
*/
public final void setZooKeeperConfiguration(String serverList) {
ZOOKEEPER_LIST.set(this, serverList);
}
/**
* Getter for SPLIT_MASTER_WORKER flag.
*
* @return boolean flag value.
*/
public final boolean getSplitMasterWorker() {
return SPLIT_MASTER_WORKER.get(this);
}
/**
* Get array of MasterObserver classes set in the configuration.
*
* @return array of MasterObserver classes.
*/
public Class<? extends MasterObserver>[] getMasterObserverClasses() {
return MASTER_OBSERVER_CLASSES.getArray(this);
}
/**
* Get array of WorkerObserver classes set in configuration.
*
* @return array of WorkerObserver classes.
*/
public Class<? extends WorkerObserver>[] getWorkerObserverClasses() {
return WORKER_OBSERVER_CLASSES.getArray(this);
}
/**
* Whether to track, print, and aggregate metrics.
*
* @return true if metrics are enabled, false otherwise (default)
*/
public boolean metricsEnabled() {
return METRICS_ENABLE.isTrue(this);
}
/**
* Get the task partition
*
* @return The task partition or -1 if not set
*/
public int getTaskPartition() {
return getInt("mapred.task.partition", -1);
}
/**
* Is this a "pure YARN" Giraph job, or is a MapReduce layer (v1 or v2)
* actually managing our cluster nodes, i.e. each task is a Mapper.
*
* @return TRUE if this is a pure YARN job.
*/
public boolean isPureYarnJob() {
return IS_PURE_YARN_JOB.get(this);
}
/**
* Jars required in "Pure YARN" jobs (names only, no paths) should
* be listed here in full, including Giraph framework jar(s).
*
* @return the comma-separated list of jar names for export to cluster.
*/
public String getYarnLibJars() {
return GIRAPH_YARN_LIBJARS.get(this);
}
/**
* Populate jar list for Pure YARN jobs.
*
* @param jarList a comma-separated list of jar names
*/
public void setYarnLibJars(String jarList) {
GIRAPH_YARN_LIBJARS.set(this, jarList);
}
/**
* Get heap size (in MB) for each task in our Giraph job run,
* assuming this job will run on the "pure YARN" profile.
*
* @return the heap size for all tasks, in MB
*/
public int getYarnTaskHeapMb() {
return GIRAPH_YARN_TASK_HEAP_MB.get(this);
}
/**
* Set heap size for Giraph tasks in our job run, assuming
* the job will run on the "pure YARN" profile.
*
* @param heapMb the heap size for all tasks
*/
public void setYarnTaskHeapMb(int heapMb) {
GIRAPH_YARN_TASK_HEAP_MB.set(this, heapMb);
}
/**
* Get the ZooKeeper list.
*
* @return ZooKeeper list of strings, comma separated or null if none set.
*/
public String getZookeeperList() {
return ZOOKEEPER_LIST.get(this);
}
/**
* Set the ZooKeeper list to the provided list. This method is used when the
* ZooKeeper is started internally and will set the zkIsExternal option to
* false as well.
*
* @param zkList list of strings, comma separated of zookeeper servers
*/
public void setZookeeperList(String zkList) {
ZOOKEEPER_LIST.set(this, zkList);
ZOOKEEPER_IS_EXTERNAL.set(this, false);
}
/**
* Was ZooKeeper provided externally?
*
* @return true iff was zookeeper is external
*/
public boolean isZookeeperExternal() {
return ZOOKEEPER_IS_EXTERNAL.get(this);
}
public String getLocalLevel() {
return LOG_LEVEL.get(this);
}
/**
* Use the log thread layout option?
*
* @return True if use the log thread layout option, false otherwise
*/
public boolean useLogThreadLayout() {
return LOG_THREAD_LAYOUT.get(this);
}
/**
* is this job run a local test?
*
* @return the test status as recorded in the Configuration
*/
public boolean getLocalTestMode() {
return LOCAL_TEST_MODE.get(this);
}
/**
* Flag this job as a local test run.
*
* @param flag the test status for this job
*/
public void setLocalTestMode(boolean flag) {
LOCAL_TEST_MODE.set(this, flag);
}
/**
* The number of server tasks in our ZK quorum for
* this job run.
*
* @return the number of ZK servers in the quorum
*/
public int getZooKeeperServerCount() {
return ZOOKEEPER_SERVER_COUNT.get(this);
}
public int getZooKeeperSessionTimeout() {
return ZOOKEEPER_SESSION_TIMEOUT.get(this);
}
public int getZookeeperOpsMaxAttempts() {
return ZOOKEEPER_OPS_MAX_ATTEMPTS.get(this);
}
public int getZookeeperOpsRetryWaitMsecs() {
return ZOOKEEPER_OPS_RETRY_WAIT_MSECS.get(this);
}
public boolean getNettyServerUseExecutionHandler() {
return NETTY_SERVER_USE_EXECUTION_HANDLER.get(this);
}
public int getNettyServerThreads() {
return NETTY_SERVER_THREADS.get(this);
}
public int getNettyServerExecutionThreads() {
return NETTY_SERVER_EXECUTION_THREADS.get(this);
}
/**
* Get the netty server execution concurrency. This depends on whether the
* netty server execution handler exists.
*
* @return Server concurrency
*/
public int getNettyServerExecutionConcurrency() {
if (getNettyServerUseExecutionHandler()) {
return getNettyServerExecutionThreads();
} else {
return getNettyServerThreads();
}
}
/**
* Used by netty client and server to create ByteBufAllocator
*
* @return ByteBufAllocator
*/
public ByteBufAllocator getNettyAllocator() {
if (nettyBufferAllocator == null) {
if (NETTY_USE_POOLED_ALLOCATOR.get(this)) { // Use pooled allocator
nettyBufferAllocator = new PooledByteBufAllocator(
NETTY_USE_DIRECT_MEMORY.get(this));
} else { // Use un-pooled allocator
// Note: Current default settings create un-pooled heap allocator
nettyBufferAllocator = new UnpooledByteBufAllocator(
NETTY_USE_DIRECT_MEMORY.get(this));
}
}
return nettyBufferAllocator;
}
public int getZookeeperConnectionAttempts() {
return ZOOKEEPER_CONNECTION_ATTEMPTS.get(this);
}
public int getZooKeeperMinSessionTimeout() {
return ZOOKEEPER_MIN_SESSION_TIMEOUT.get(this);
}
public int getZooKeeperMaxSessionTimeout() {
return ZOOKEEPER_MAX_SESSION_TIMEOUT.get(this);
}
public boolean getZooKeeperForceSync() {
return ZOOKEEPER_FORCE_SYNC.get(this);
}
public boolean getZooKeeperSkipAcl() {
return ZOOKEEPER_SKIP_ACL.get(this);
}
/**
* Get the number of map tasks in this job
*
* @return Number of map tasks in this job
*/
public int getMapTasks() {
int mapTasks = getInt("mapred.map.tasks", -1);
if (mapTasks == -1) {
throw new IllegalStateException("getMapTasks: Failed to get the map " +
"tasks!");
}
return mapTasks;
}
/**
* Use authentication? (if supported)
*
* @return True if should authenticate, false otherwise
*/
public boolean authenticate() {
return AUTHENTICATE.get(this);
}
/**
* Set the number of compute threads
*
* @param numComputeThreads Number of compute threads to use
*/
public void setNumComputeThreads(int numComputeThreads) {
NUM_COMPUTE_THREADS.set(this, numComputeThreads);
}
public int getNumComputeThreads() {
return NUM_COMPUTE_THREADS.get(this);
}
/**
* Set the number of input split threads
*
* @param numInputSplitsThreads Number of input split threads to use
*/
public void setNumInputSplitsThreads(int numInputSplitsThreads) {
NUM_INPUT_THREADS.set(this, numInputSplitsThreads);
}
public int getNumInputSplitsThreads() {
return NUM_INPUT_THREADS.get(this);
}
public long getInputSplitMaxVertices() {
return INPUT_SPLIT_MAX_VERTICES.get(this);
}
public long getInputSplitMaxEdges() {
return INPUT_SPLIT_MAX_EDGES.get(this);
}
/**
* Set whether to use unsafe serialization
*
* @param useUnsafeSerialization If true, use unsafe serialization
*/
public void useUnsafeSerialization(boolean useUnsafeSerialization) {
USE_UNSAFE_SERIALIZATION.set(this, useUnsafeSerialization);
}
/**
* Use message size encoding? This feature may help with complex message
* objects.
*
* @return Whether to use message size encoding
*/
public boolean useMessageSizeEncoding() {
return USE_MESSAGE_SIZE_ENCODING.get(this);
}
/**
* Set the checkpoint frequeuncy of how many supersteps to wait before
* checkpointing
*
* @param checkpointFrequency How often to checkpoint (0 means never)
*/
public void setCheckpointFrequency(int checkpointFrequency) {
CHECKPOINT_FREQUENCY.set(this, checkpointFrequency);
}
/**
* Get the checkpoint frequeuncy of how many supersteps to wait
* before checkpointing
*
* @return Checkpoint frequency (0 means never)
*/
public int getCheckpointFrequency() {
return CHECKPOINT_FREQUENCY.get(this);
}
/**
* Check if checkpointing is used
*
* @return True iff checkpointing is used
*/
public boolean useCheckpointing() {
return getCheckpointFrequency() != 0;
}
/**
* Set the max task attempts
*
* @param maxTaskAttempts Max task attempts to use
*/
public void setMaxTaskAttempts(int maxTaskAttempts) {
MAX_TASK_ATTEMPTS.set(this, maxTaskAttempts);
}
/**
* Get the max task attempts
*
* @return Max task attempts or -1, if not set
*/
public int getMaxTaskAttempts() {
return MAX_TASK_ATTEMPTS.get(this);
}
/**
* Get the number of milliseconds to wait for an event before continuing on
*
* @return Number of milliseconds to wait for an event before continuing on
*/
public int getEventWaitMsecs() {
return EVENT_WAIT_MSECS.get(this);
}
/**
* Set the number of milliseconds to wait for an event before continuing on
*
* @param eventWaitMsecs Number of milliseconds to wait for an event before
* continuing on
*/
public void setEventWaitMsecs(int eventWaitMsecs) {
EVENT_WAIT_MSECS.set(this, eventWaitMsecs);
}
/**
* Get the maximum milliseconds to wait before giving up trying to get the
* minimum number of workers before a superstep.
*
* @return Maximum milliseconds to wait before giving up trying to get the
* minimum number of workers before a superstep
*/
public int getMaxMasterSuperstepWaitMsecs() {
return MAX_MASTER_SUPERSTEP_WAIT_MSECS.get(this);
}
/**
* Set the maximum milliseconds to wait before giving up trying to get the
* minimum number of workers before a superstep.
*
* @param maxMasterSuperstepWaitMsecs Maximum milliseconds to wait before
* giving up trying to get the minimum
* number of workers before a superstep
*/
public void setMaxMasterSuperstepWaitMsecs(int maxMasterSuperstepWaitMsecs) {
MAX_MASTER_SUPERSTEP_WAIT_MSECS.set(this, maxMasterSuperstepWaitMsecs);
}
/**
* Check environment for Hadoop security token location in case we are
* executing the Giraph job on a MRv1 Hadoop cluster.
*/
public void configureHadoopSecurity() {
String hadoopTokenFilePath = System.getenv("HADOOP_TOKEN_FILE_LOCATION");
if (hadoopTokenFilePath != null) {
set("mapreduce.job.credentials.binary", hadoopTokenFilePath);
}
}
/**
* Check if we want to prioritize input splits which reside on the host.
*
* @return True iff we want to use input split locality
*/
public boolean useInputSplitLocality() {
return USE_INPUT_SPLIT_LOCALITY.get(this);
}
/**
* Get the local hostname on the given interface.
*
* @return The local hostname
* @throws UnknownHostException
*/
public String getLocalHostname() throws UnknownHostException {
return DNS.getDefaultHost(
GiraphConstants.DNS_INTERFACE.get(this),
GiraphConstants.DNS_NAMESERVER.get(this)).toLowerCase();
}
/**
* Set the maximum number of supersteps of this application. After this
* many supersteps are executed, the application will shutdown.
*
* @param maxNumberOfSupersteps Maximum number of supersteps
*/
public void setMaxNumberOfSupersteps(int maxNumberOfSupersteps) {
MAX_NUMBER_OF_SUPERSTEPS.set(this, maxNumberOfSupersteps);
}
/**
* Get the maximum number of supersteps of this application. After this
* many supersteps are executed, the application will shutdown.
*
* @return Maximum number of supersteps
*/
public int getMaxNumberOfSupersteps() {
return MAX_NUMBER_OF_SUPERSTEPS.get(this);
}
/**
* Whether the application with change or not the graph topology.
*
* @return true if the graph is static, false otherwise.
*/
public boolean isStaticGraph() {
return STATIC_GRAPH.isTrue(this);
}
/**
* Get the output directory to write YourKit snapshots to
*
* @param context Map context
* @return output directory
*/
public String getYourKitOutputDir(Mapper.Context context) {
final String cacheKey = "giraph.yourkit.outputDirCached";
String outputDir = get(cacheKey);
if (outputDir == null) {
outputDir = getStringVars(YOURKIT_OUTPUT_DIR, YOURKIT_OUTPUT_DIR_DEFAULT,
context);
set(cacheKey, outputDir);
}
return outputDir;
}
/**
* Get string, replacing variables in the output.
*
* %JOB_ID% => job id
* %TASK_ID% => task id
* %USER% => owning user name
*
* @param key name of key to lookup
* @param context mapper context
* @return value for key, with variables expanded
*/
public String getStringVars(String key, Mapper.Context context) {
return getStringVars(key, null, context);
}
/**
* Get string, replacing variables in the output.
*
* %JOB_ID% => job id
* %TASK_ID% => task id
* %USER% => owning user name
*
* @param key name of key to lookup
* @param defaultValue value to return if no mapping exists. This can also
* have variables, which will be substituted.
* @param context mapper context
* @return value for key, with variables expanded
*/
public String getStringVars(String key, String defaultValue,
Mapper.Context context) {
String value = get(key);
if (value == null) {
if (defaultValue == null) {
return null;
}
value = defaultValue;
}
value = value.replace("%JOB_ID%", context.getJobID().toString());
value = value.replace("%TASK_ID%", context.getTaskAttemptID().toString());
value = value.replace("%USER%", get("user.name", "unknown_user"));
return value;
}
/**
* Return if oneMessageToManyIds encoding can be enabled
*
* @return True if this option is true.
*/
public boolean useOneMessageToManyIdsEncoding() {
return MESSAGE_ENCODE_AND_STORE_TYPE.get(this)
.useOneMessageToManyIdsEncoding();
}
/**
* Get option whether to create a source vertex present only in edge input
*
* @return CREATE_EDGE_SOURCE_VERTICES option
*/
public boolean getCreateSourceVertex() {
return CREATE_EDGE_SOURCE_VERTICES.get(this);
}
/**
* set option whether to create a source vertex present only in edge input
* @param createVertex create source vertex option
*/
public void setCreateSourceVertex(boolean createVertex) {
CREATE_EDGE_SOURCE_VERTICES.set(this, createVertex);
}
/**
* Get the maximum timeout (in milliseconds) for waiting for all tasks
* to complete after the job is done.
*
* @return Wait task done timeout in milliseconds.
*/
public int getWaitTaskDoneTimeoutMs() {
return WAIT_TASK_DONE_TIMEOUT_MS.get(this);
}
/**
* Set the maximum timeout (in milliseconds) for waiting for all tasks
* to complete after the job is done.
*
* @param ms Milliseconds to set
*/
public void setWaitTaskDoneTimeoutMs(int ms) {
WAIT_TASK_DONE_TIMEOUT_MS.set(this, ms);
}
/**
* Check whether to track job progress on client or not
*
* @return True if job progress should be tracked on client
*/
public boolean trackJobProgressOnClient() {
return TRACK_JOB_PROGRESS_ON_CLIENT.get(this);
}
/**
* @return Number of retries when creating an HDFS file before failing.
*/
public int getHdfsFileCreationRetries() {
return HDFS_FILE_CREATION_RETRIES.get(this);
}
/**
* @return Milliseconds to wait before retrying an HDFS file creation
* operation.
*/
public int getHdfsFileCreationRetryWaitMs() {
return HDFS_FILE_CREATION_RETRY_WAIT_MS.get(this);
}
}