/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.giraph.comm.netty;
/*if_not[HADOOP_NON_SECURE]*/
import org.apache.giraph.comm.netty.handler.AuthorizeServerHandler;
/*end[HADOOP_NON_SECURE]*/
import org.apache.giraph.comm.netty.handler.RequestDecoder;
import org.apache.giraph.comm.netty.handler.RequestServerHandler;
import org.apache.giraph.comm.netty.handler.ResponseEncoder;
/*if_not[HADOOP_NON_SECURE]*/
import org.apache.giraph.comm.netty.handler.SaslServerHandler;
/*end[HADOOP_NON_SECURE]*/
import org.apache.giraph.comm.netty.handler.WorkerRequestReservedMap;
import org.apache.giraph.conf.GiraphConstants;
import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
import org.apache.giraph.graph.TaskInfo;
import org.apache.giraph.utils.PipelineUtils;
import org.apache.giraph.utils.ProgressableUtils;
import org.apache.giraph.utils.ThreadUtils;
import org.apache.hadoop.util.Progressable;
import org.apache.log4j.Logger;
import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.group.ChannelGroup;
import io.netty.channel.group.DefaultChannelGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.ChannelOption;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.channel.ChannelFuture;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
import io.netty.util.AttributeKey;
import io.netty.util.concurrent.DefaultEventExecutorGroup;
import io.netty.util.concurrent.EventExecutorGroup;
import io.netty.util.concurrent.ImmediateEventExecutor;
import io.netty.channel.AdaptiveRecvByteBufAllocator;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import static org.apache.giraph.conf.GiraphConstants.MAX_IPC_PORT_BIND_ATTEMPTS;
/**
* This server uses Netty and will implement all Giraph communication
*/
public class NettyServer {
/** Default maximum thread pool size */
public static final int MAXIMUM_THREAD_POOL_SIZE_DEFAULT = 32;
/*if_not[HADOOP_NON_SECURE]*/
/** Used to authenticate with netty clients */
public static final AttributeKey<SaslNettyServer>
CHANNEL_SASL_NETTY_SERVERS = AttributeKey.valueOf("channelSaslServers");
/*end[HADOOP_NON_SECURE]*/
/** Class logger */
private static final Logger LOG = Logger.getLogger(NettyServer.class);
/** Configuration */
private final ImmutableClassesGiraphConfiguration conf;
/** Progressable for reporting progress */
private final Progressable progressable;
/** Accepted channels */
private final ChannelGroup accepted = new DefaultChannelGroup(
ImmediateEventExecutor.INSTANCE);
/** Local hostname */
private final String localHostname;
/** Address of the server */
private InetSocketAddress myAddress;
/** Current task info */
private TaskInfo myTaskInfo;
/** Maximum number of threads */
private final int maxPoolSize;
/** TCP backlog */
private final int tcpBacklog;
/** Factory for {@link RequestServerHandler} */
private final RequestServerHandler.Factory requestServerHandlerFactory;
/*if_not[HADOOP_NON_SECURE]*/
/** Factory for {@link RequestServerHandler} */
private SaslServerHandler.Factory saslServerHandlerFactory;
/*end[HADOOP_NON_SECURE]*/
/** Server bootstrap */
private ServerBootstrap bootstrap;
/** Inbound byte counter for this client */
private final InboundByteCounter inByteCounter = new InboundByteCounter();
/** Outbound byte counter for this client */
private final OutboundByteCounter outByteCounter = new OutboundByteCounter();
/** Send buffer size */
private final int sendBufferSize;
/** Receive buffer size */
private final int receiveBufferSize;
/** Boss eventloop group */
private final EventLoopGroup bossGroup;
/** Worker eventloop group */
private final EventLoopGroup workerGroup;
/** Request completed map per worker */
private final WorkerRequestReservedMap workerRequestReservedMap;
/** Use execution group? */
private final boolean useExecutionGroup;
/** Execution handler (if used) */
private final EventExecutorGroup executionGroup;
/** Name of the handler before the execution handler (if used) */
private final String handlerToUseExecutionGroup;
/** Handles all uncaught exceptions in netty threads */
private final Thread.UncaughtExceptionHandler exceptionHandler;
/**
* Constructor for creating the server
*
* @param conf Configuration to use
* @param requestServerHandlerFactory Factory for request handlers
* @param myTaskInfo Current task info
* @param progressable Progressable for reporting progress
* @param exceptionHandler handle uncaught exceptions
*/
public NettyServer(ImmutableClassesGiraphConfiguration conf,
RequestServerHandler.Factory requestServerHandlerFactory,
TaskInfo myTaskInfo, Progressable progressable,
Thread.UncaughtExceptionHandler exceptionHandler) {
this.conf = conf;
this.progressable = progressable;
this.requestServerHandlerFactory = requestServerHandlerFactory;
/*if_not[HADOOP_NON_SECURE]*/
this.saslServerHandlerFactory = new SaslServerHandler.Factory();
/*end[HADOOP_NON_SECURE]*/
this.myTaskInfo = myTaskInfo;
this.exceptionHandler = exceptionHandler;
sendBufferSize = GiraphConstants.SERVER_SEND_BUFFER_SIZE.get(conf);
receiveBufferSize = GiraphConstants.SERVER_RECEIVE_BUFFER_SIZE.get(conf);
workerRequestReservedMap = new WorkerRequestReservedMap(conf);
maxPoolSize = GiraphConstants.NETTY_SERVER_THREADS.get(conf);
bossGroup = new NioEventLoopGroup(4,
ThreadUtils.createThreadFactory(
"netty-server-boss-%d", exceptionHandler));
workerGroup = new NioEventLoopGroup(maxPoolSize,
ThreadUtils.createThreadFactory(
"netty-server-worker-%d", exceptionHandler));
try {
this.localHostname = conf.getLocalHostname();
} catch (UnknownHostException e) {
throw new IllegalStateException("NettyServer: unable to get hostname");
}
tcpBacklog = conf.getInt(GiraphConstants.TCP_BACKLOG.getKey(),
conf.getInt(GiraphConstants.MAX_WORKERS,
GiraphConstants.TCP_BACKLOG.getDefaultValue()));
handlerToUseExecutionGroup =
GiraphConstants.NETTY_SERVER_EXECUTION_AFTER_HANDLER.get(conf);
useExecutionGroup =
GiraphConstants.NETTY_SERVER_USE_EXECUTION_HANDLER.get(conf);
if (useExecutionGroup) {
int executionThreads = conf.getNettyServerExecutionThreads();
executionGroup = new DefaultEventExecutorGroup(executionThreads,
ThreadUtils.createThreadFactory(
"netty-server-exec-%d", exceptionHandler));
if (LOG.isInfoEnabled()) {
LOG.info("NettyServer: Using execution group with " +
executionThreads + " threads for " +
handlerToUseExecutionGroup + ".");
}
} else {
executionGroup = null;
}
}
/*if_not[HADOOP_NON_SECURE]*/
/**
* Constructor for creating the server
*
* @param conf Configuration to use
* @param requestServerHandlerFactory Factory for request handlers
* @param myTaskInfo Current task info
* @param progressable Progressable for reporting progress
* @param saslServerHandlerFactory Factory for SASL handlers
* @param exceptionHandler handle uncaught exceptions
*/
public NettyServer(ImmutableClassesGiraphConfiguration conf,
RequestServerHandler.Factory requestServerHandlerFactory,
TaskInfo myTaskInfo,
Progressable progressable,
SaslServerHandler.Factory saslServerHandlerFactory,
Thread.UncaughtExceptionHandler exceptionHandler) {
this(conf, requestServerHandlerFactory, myTaskInfo,
progressable, exceptionHandler);
this.saslServerHandlerFactory = saslServerHandlerFactory;
}
/*end[HADOOP_NON_SECURE]*/
/**
* Start the server with the appropriate port
*/
public void start() {
bootstrap = new ServerBootstrap();
bootstrap.group(bossGroup, workerGroup)
.channel(NioServerSocketChannel.class)
.option(ChannelOption.SO_BACKLOG, tcpBacklog)
.option(ChannelOption.ALLOCATOR, conf.getNettyAllocator())
.childOption(ChannelOption.SO_KEEPALIVE, true)
.childOption(ChannelOption.TCP_NODELAY, true)
.childOption(ChannelOption.SO_SNDBUF, sendBufferSize)
.childOption(ChannelOption.SO_RCVBUF, receiveBufferSize)
.childOption(ChannelOption.ALLOCATOR, conf.getNettyAllocator())
.childOption(ChannelOption.RCVBUF_ALLOCATOR,
new AdaptiveRecvByteBufAllocator(receiveBufferSize / 4,
receiveBufferSize, receiveBufferSize));
/**
* Pipeline setup: depends on whether configured to use authentication
* or not.
*/
bootstrap.childHandler(new ChannelInitializer<SocketChannel>() {
@Override
protected void initChannel(SocketChannel ch) throws Exception {
/*if_not[HADOOP_NON_SECURE]*/
if (conf.authenticate()) {
LOG.info("start: Will use Netty pipeline with " +
"authentication and authorization of clients.");
// After a client authenticates, the two authentication-specific
// pipeline components SaslServerHandler and ResponseEncoder are
// removed, leaving the pipeline the same as in the non-authenticated
// configuration except for the presence of the Authorize component.
PipelineUtils.addLastWithExecutorCheck("serverInboundByteCounter",
inByteCounter, handlerToUseExecutionGroup, executionGroup, ch);
if (conf.doCompression()) {
PipelineUtils.addLastWithExecutorCheck("compressionDecoder",
conf.getNettyCompressionDecoder(),
handlerToUseExecutionGroup, executionGroup, ch);
}
PipelineUtils.addLastWithExecutorCheck("serverOutboundByteCounter",
outByteCounter, handlerToUseExecutionGroup, executionGroup, ch);
if (conf.doCompression()) {
PipelineUtils.addLastWithExecutorCheck("compressionEncoder",
conf.getNettyCompressionEncoder(),
handlerToUseExecutionGroup, executionGroup, ch);
}
PipelineUtils.addLastWithExecutorCheck("requestFrameDecoder",
new LengthFieldBasedFrameDecoder(1024 * 1024 * 1024, 0, 4, 0, 4),
handlerToUseExecutionGroup, executionGroup, ch);
PipelineUtils.addLastWithExecutorCheck("requestDecoder",
new RequestDecoder(conf, inByteCounter),
handlerToUseExecutionGroup, executionGroup, ch);
// Removed after authentication completes:
PipelineUtils.addLastWithExecutorCheck("saslServerHandler",
saslServerHandlerFactory.newHandler(conf),
handlerToUseExecutionGroup, executionGroup, ch);
PipelineUtils.addLastWithExecutorCheck("authorizeServerHandler",
new AuthorizeServerHandler(), handlerToUseExecutionGroup,
executionGroup, ch);
PipelineUtils.addLastWithExecutorCheck("requestServerHandler",
requestServerHandlerFactory.newHandler(workerRequestReservedMap,
conf, myTaskInfo, exceptionHandler),
handlerToUseExecutionGroup, executionGroup, ch);
// Removed after authentication completes:
PipelineUtils.addLastWithExecutorCheck("responseEncoder",
new ResponseEncoder(), handlerToUseExecutionGroup,
executionGroup, ch);
} else {
LOG.info("start: Using Netty without authentication.");
/*end[HADOOP_NON_SECURE]*/
// Store all connected channels in order to ensure that we can close
// them on stop(), or else stop() may hang waiting for the
// connections to close on their own
ch.pipeline().addLast("connectedChannels",
new ChannelInboundHandlerAdapter() {
@Override
public void channelActive(ChannelHandlerContext ctx)
throws Exception {
accepted.add(ctx.channel());
ctx.fireChannelActive();
}
});
PipelineUtils.addLastWithExecutorCheck("serverInboundByteCounter",
inByteCounter, handlerToUseExecutionGroup, executionGroup, ch);
if (conf.doCompression()) {
PipelineUtils.addLastWithExecutorCheck("compressionDecoder",
conf.getNettyCompressionDecoder(),
handlerToUseExecutionGroup, executionGroup, ch);
}
PipelineUtils.addLastWithExecutorCheck("serverOutboundByteCounter",
outByteCounter, handlerToUseExecutionGroup, executionGroup, ch);
if (conf.doCompression()) {
PipelineUtils.addLastWithExecutorCheck("compressionEncoder",
conf.getNettyCompressionEncoder(),
handlerToUseExecutionGroup, executionGroup, ch);
}
PipelineUtils.addLastWithExecutorCheck("requestFrameDecoder",
new LengthFieldBasedFrameDecoder(1024 * 1024 * 1024, 0, 4, 0, 4),
handlerToUseExecutionGroup, executionGroup, ch);
PipelineUtils.addLastWithExecutorCheck("requestDecoder",
new RequestDecoder(conf, inByteCounter),
handlerToUseExecutionGroup, executionGroup, ch);
PipelineUtils.addLastWithExecutorCheck("requestServerHandler",
requestServerHandlerFactory.newHandler(
workerRequestReservedMap, conf, myTaskInfo, exceptionHandler),
handlerToUseExecutionGroup, executionGroup, ch);
/*if_not[HADOOP_NON_SECURE]*/
}
/*end[HADOOP_NON_SECURE]*/
}
});
int taskId = conf.getTaskPartition();
int numTasks = conf.getInt("mapred.map.tasks", 1);
// Number of workers + 1 for master
int numServers = conf.getInt(GiraphConstants.MAX_WORKERS, numTasks) + 1;
int portIncrementConstant =
(int) Math.pow(10, Math.ceil(Math.log10(numServers)));
int bindPort = GiraphConstants.IPC_INITIAL_PORT.get(conf) + taskId;
int bindAttempts = 0;
final int maxIpcPortBindAttempts = MAX_IPC_PORT_BIND_ATTEMPTS.get(conf);
final boolean failFirstPortBindingAttempt =
GiraphConstants.FAIL_FIRST_IPC_PORT_BIND_ATTEMPT.get(conf);
// Simple handling of port collisions on the same machine while
// preserving debugability from the port number alone.
// Round up the max number of workers to the next power of 10 and use
// it as a constant to increase the port number with.
while (bindAttempts < maxIpcPortBindAttempts) {
this.myAddress = new InetSocketAddress(localHostname, bindPort);
if (failFirstPortBindingAttempt && bindAttempts == 0) {
if (LOG.isInfoEnabled()) {
LOG.info("start: Intentionally fail first " +
"binding attempt as giraph.failFirstIpcPortBindAttempt " +
"is true, port " + bindPort);
}
++bindAttempts;
bindPort += portIncrementConstant;
continue;
}
try {
ChannelFuture f = bootstrap.bind(myAddress).sync();
accepted.add(f.channel());
break;
} catch (InterruptedException e) {
throw new IllegalStateException(e);
// CHECKSTYLE: stop IllegalCatchCheck
} catch (Exception e) {
// CHECKSTYLE: resume IllegalCatchCheck
LOG.warn("start: Likely failed to bind on attempt " +
bindAttempts + " to port " + bindPort, e.getCause());
++bindAttempts;
bindPort += portIncrementConstant;
}
}
if (bindAttempts == maxIpcPortBindAttempts || myAddress == null) {
throw new IllegalStateException(
"start: Failed to start NettyServer with " +
bindAttempts + " attempts");
}
if (LOG.isInfoEnabled()) {
LOG.info("start: Started server " +
"communication server: " + myAddress + " with up to " +
maxPoolSize + " threads on bind attempt " + bindAttempts +
" with sendBufferSize = " + sendBufferSize +
" receiveBufferSize = " + receiveBufferSize);
}
}
/**
* Stop the server.
*/
public void stop() {
if (LOG.isInfoEnabled()) {
LOG.info("stop: Halting netty server");
}
ProgressableUtils.awaitChannelGroupFuture(accepted.close(), progressable);
if (LOG.isInfoEnabled()) {
LOG.info("stop: Start releasing resources");
}
bossGroup.shutdownGracefully();
workerGroup.shutdownGracefully();
ProgressableUtils.awaitTerminationFuture(bossGroup, progressable);
ProgressableUtils.awaitTerminationFuture(workerGroup, progressable);
if (useExecutionGroup) {
executionGroup.shutdownGracefully();
ProgressableUtils.awaitTerminationFuture(executionGroup, progressable);
}
if (LOG.isInfoEnabled()) {
LOG.info("stop: Netty server halted");
}
}
public InetSocketAddress getMyAddress() {
return myAddress;
}
}