*

* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* parameter, and return a single Protobuf message as their value.  A service runs on

* a port and is defined by a parameter class and a value class.

*

*

* <p>Copied local so can fix HBASE-900.

*

* @see HBaseClient

*/

  public static final Log LOG = LogFactory.getLog("org.apache.hadoop.ipc.HBaseServer");

  private final boolean authorize;

  protected boolean isSecurityEnabled;

  public static final byte CURRENT_VERSION = 0;

  /**

   * How many calls/handler are allowed in the queue.

   */

  private static final int DEFAULT_MAX_CALLQUEUE_LENGTH_PER_HANDLER = 10;

  /**

   * The maximum size that we can hold in the IPC queue

   */

  private static final int DEFAULT_MAX_CALLQUEUE_SIZE =

    1024 * 1024 * 1024;

  static final int BUFFER_INITIAL_SIZE = 1024;

  private static final String WARN_DELAYED_CALLS =

      "hbase.ipc.warn.delayedrpc.number";

  private static final int DEFAULT_WARN_DELAYED_CALLS = 1000;

  private final int warnDelayedCalls;

  private AtomicInteger delayedCalls;

  private final IPCUtil ipcUtil;

  private static final String AUTH_FAILED_FOR = "Auth failed for ";

  private static final String AUTH_SUCCESSFUL_FOR = "Auth successful for ";

  private static final Log AUDITLOG =

      LogFactory.getLog("SecurityLogger."+Server.class.getName());

  protected SecretManager<TokenIdentifier> secretManager;

  protected ServiceAuthorizationManager authManager;

  protected static final ThreadLocal<RpcServer> SERVER =

    new ThreadLocal<RpcServer>();

  private volatile boolean started = false;

  private static final ReflectionCache methodCache = new ReflectionCache();

  private static final Map<String, Class<? extends IpcProtocol>> PROTOCOL_CACHE =

      new ConcurrentHashMap<String, Class<? extends IpcProtocol>>();

  @SuppressWarnings("unchecked")

  static Class<? extends IpcProtocol> getProtocolClass(

      String protocolName, Configuration conf)

  throws ClassNotFoundException {

    Class<? extends IpcProtocol> protocol =

        PROTOCOL_CACHE.get(protocolName);

    if (protocol == null) {

      protocol = (Class<? extends IpcProtocol>)

          conf.getClassByName(protocolName);

      PROTOCOL_CACHE.put(protocolName, protocol);

    }

    return protocol;

  }

  /** Returns the server instance called under or null.  May be called under

   * {@code #call(Class, RpcRequestBody, long, MonitoredRPCHandler)} implementations,

   * and under protobuf methods of parameters and return values.

   * Permits applications to access the server context.

   * @return HBaseServer

   */

  public static RpcServer get() {

    return SERVER.get();

  }

  /** This is set to Call object before Handler invokes an RPC and reset

   * after the call returns.

   */

  protected static final ThreadLocal<Call> CurCall = new ThreadLocal<Call>();

  /** Returns the remote side ip address when invoked inside an RPC

   *  Returns null incase of an error.

   *  @return InetAddress

   */

  public static InetAddress getRemoteIp() {

    Call call = CurCall.get();

    if (call != null) {

      return call.connection.socket.getInetAddress();

    }

    return null;

  }

  /** Returns remote address as a string when invoked inside an RPC.

   *  Returns null in case of an error.

   *  @return String

   */

  public static String getRemoteAddress() {

    Call call = CurCall.get();

    if (call != null) {

      return call.connection.getHostAddress();

    }

    return null;

  }

  protected String bindAddress;

  protected int port;                             // port we listen on

  private int handlerCount;                       // number of handler threads

  private int priorityHandlerCount;

  private int readThreads;                        // number of read threads

  protected int maxIdleTime;                      // the maximum idle time after

                                                  // which a client may be

                                                  // disconnected

  protected int thresholdIdleConnections;         // the number of idle

                                                  // connections after which we

                                                  // will start cleaning up idle

                                                  // connections

  int maxConnectionsToNuke;                       // the max number of

                                                  // connections to nuke

                                                  // during a cleanup

  protected MetricsHBaseServer metrics;

  protected Configuration conf;

  private int maxQueueLength;

  private int maxQueueSize;

  protected int socketSendBufferSize;

  protected final boolean tcpNoDelay;   // if T then disable Nagle's Algorithm

  protected final boolean tcpKeepAlive; // if T then use keepalives

  protected final long purgeTimeout;    // in milliseconds

  volatile protected boolean running = true;         // true while server runs

  protected BlockingQueue<Call> callQueue; // queued calls

  protected final Counter callQueueSize = new Counter();

  protected BlockingQueue<Call> priorityCallQueue;

  protected int highPriorityLevel;  // what level a high priority call is at

  protected final List<Connection> connectionList =

    Collections.synchronizedList(new LinkedList<Connection>());

  //maintain a list

  //of client connections

  private Listener listener = null;

  protected Responder responder = null;

  protected int numConnections = 0;

  private Handler[] handlers = null;

  private Handler[] priorityHandlers = null;

  /** replication related queue; */

  protected BlockingQueue<Call> replicationQueue;

  private int numOfReplicationHandlers = 0;

  private Handler[] replicationHandlers = null;

  protected HBaseRPCErrorHandler errorHandler = null;

  /**

   * A convenience method to bind to a given address and report

   * better exceptions if the address is not a valid host.

   * @param socket the socket to bind

   * @param address the address to bind to

   * @param backlog the number of connections allowed in the queue

   * @throws BindException if the address can't be bound

   * @throws UnknownHostException if the address isn't a valid host name

   * @throws IOException other random errors from bind

   */

  public static void bind(ServerSocket socket, InetSocketAddress address,

                          int backlog) throws IOException {

    try {

      socket.bind(address, backlog);

    } catch (BindException e) {

      BindException bindException =

        new BindException("Problem binding to " + address + " : " +

            e.getMessage());

      bindException.initCause(e);

      throw bindException;

    } catch (SocketException e) {

      // If they try to bind to a different host's address, give a better

      // error message.

      if ("Unresolved address".equals(e.getMessage())) {

        throw new UnknownHostException("Invalid hostname for server: " +

                                       address.getHostName());

      }

      throw e;

    }

  }

  /** A call queued for handling. */

  protected class Call implements RpcCallContext {

    protected int id;                             // the client's call id

    protected Method method;

    protected Message param;                      // the parameter passed

    // Optional cell data passed outside of protobufs.

    protected CellScanner cellScanner;

    protected Connection connection;              // connection to client

    protected long timestamp;      // the time received when response is null

                                   // the time served when response is not null

    protected ByteBuffer response;                // the response for this call

    protected boolean delayResponse;

    protected Responder responder;

    protected boolean delayReturnValue;           // if the return value should be

                                                  // set at call completion

    protected long size;                          // size of current call

    protected boolean isError;

    protected TraceInfo tinfo;

    public Call(int id, Method method, Message param, CellScanner cellScanner,

        Connection connection, Responder responder, long size, TraceInfo tinfo) {

      this.id = id;

      this.method = method;

      this.param = param;

      this.cellScanner = cellScanner;

      this.connection = connection;

      this.timestamp = System.currentTimeMillis();

      this.response = null;

      this.delayResponse = false;

      this.responder = responder;

      this.isError = false;

      this.size = size;

      this.tinfo = tinfo;

    }

    @Override

    public String toString() {

      return "callId: " + this.id + " methodName: " +

        ((this.method != null)? this.method.getName(): null) + " param: " +

        (this.param != null? TextFormat.shortDebugString(this.param): "") +

        " from " + connection.toString();

    }

    protected synchronized void setSaslTokenResponse(ByteBuffer response) {

      this.response = response;

    }

    protected synchronized void setResponse(Object m, final CellScanner cells,

        Throwable t, String errorMsg) {

      if (this.isError) return;

      if (t != null) this.isError = true;

      ByteBufferOutputStream bbos = null;

      try {

        ResponseHeader.Builder headerBuilder = ResponseHeader.newBuilder();

        // Presume it a pb Message.  Could be null.

        Message result = (Message)m;

        // Call id.

        headerBuilder.setCallId(this.id);

        if (t != null) {

          ExceptionResponse.Builder exceptionBuilder = ExceptionResponse.newBuilder();

          exceptionBuilder.setExceptionClassName(t.getClass().getName());

          exceptionBuilder.setStackTrace(errorMsg);

          exceptionBuilder.setDoNotRetry(t instanceof DoNotRetryIOException);

          if (t instanceof RegionMovedException) {

            // Special casing for this exception.  This is only one carrying a payload.

            // Do this instead of build a generic system for allowing exceptions carry

            // any kind of payload.

            RegionMovedException rme = (RegionMovedException)t;

            exceptionBuilder.setHostname(rme.getHostname());

            exceptionBuilder.setPort(rme.getPort());

          }

          // Set the exception as the result of the method invocation.

          headerBuilder.setException(exceptionBuilder.build());

        }

        ByteBuffer cellBlock =

          ipcUtil.buildCellBlock(this.connection.codec, this.connection.compressionCodec, cells);

        if (cellBlock != null) {

          CellBlockMeta.Builder cellBlockBuilder = CellBlockMeta.newBuilder();

          // Presumes the cellBlock bytebuffer has been flipped so limit has total size in it.

          cellBlockBuilder.setLength(cellBlock.limit());

          headerBuilder.setCellBlockMeta(cellBlockBuilder.build());

        }

        Message header = headerBuilder.build();

        bbos = IPCUtil.write(header, result, cellBlock);

        if (connection.useWrap) {

          wrapWithSasl(bbos);

        }

        if (LOG.isDebugEnabled()) {

          LOG.debug("Header " + TextFormat.shortDebugString(header) +

            ", result " + (result != null? TextFormat.shortDebugString(result): "null"));

        }

      } catch (IOException e) {

        LOG.warn("Exception while creating response " + e);

      }

      ByteBuffer bb = null;

      if (bbos != null) {

        // TODO: If SASL, maybe buffer already been flipped and written?

        bb = bbos.getByteBuffer();

        bb.position(0);

      }

      this.response = bb;

    }

    private void wrapWithSasl(ByteBufferOutputStream response)

    throws IOException {

      if (connection.useSasl) {

        // getByteBuffer calls flip()

        ByteBuffer buf = response.getByteBuffer();

        byte[] token;

        // synchronization may be needed since there can be multiple Handler

        // threads using saslServer to wrap responses.

        synchronized (connection.saslServer) {

          token = connection.saslServer.wrap(buf.array(),

              buf.arrayOffset(), buf.remaining());

        }

        if (LOG.isDebugEnabled())

          LOG.debug("Adding saslServer wrapped token of size " + token.length

              + " as call response.");

        buf.clear();

        DataOutputStream saslOut = new DataOutputStream(response);

        saslOut.writeInt(token.length);

        saslOut.write(token, 0, token.length);

      }

    }

    @Override

    public synchronized void endDelay(Object result) throws IOException {

      assert this.delayResponse;

      assert this.delayReturnValue || result == null;

      this.delayResponse = false;

      delayedCalls.decrementAndGet();

      if (this.delayReturnValue) {

        this.setResponse(result, null, null, null);

      }

      this.responder.doRespond(this);

    }

    @Override

    public synchronized void endDelay() throws IOException {

      this.endDelay(null);

    }

    @Override

    public synchronized void startDelay(boolean delayReturnValue) {

      assert !this.delayResponse;

      this.delayResponse = true;

      this.delayReturnValue = delayReturnValue;

      int numDelayed = delayedCalls.incrementAndGet();

      if (numDelayed > warnDelayedCalls) {

        LOG.warn("Too many delayed calls: limit " + warnDelayedCalls +

            " current " + numDelayed);

      }

    }

    @Override

    public synchronized void endDelayThrowing(Throwable t) throws IOException {

      this.setResponse(null, null, t, StringUtils.stringifyException(t));

      this.delayResponse = false;

      this.sendResponseIfReady();

    }

    @Override

    public synchronized boolean isDelayed() {

      return this.delayResponse;

    }

    @Override

    public synchronized boolean isReturnValueDelayed() {

      return this.delayReturnValue;

    }

    @Override

    public void throwExceptionIfCallerDisconnected() throws CallerDisconnectedException {

      if (!connection.channel.isOpen()) {

        long afterTime = System.currentTimeMillis() - timestamp;

        throw new CallerDisconnectedException(

            "Aborting call " + this + " after " + afterTime + " ms, since " +

            "caller disconnected");

      }

    }

    public long getSize() {

      return this.size;

    }

    /**

     * If we have a response, and delay is not set, then respond

     * immediately.  Otherwise, do not respond to client.  This is

     * called the by the RPC code in the context of the Handler thread.

     */

    public synchronized void sendResponseIfReady() throws IOException {

      if (!this.delayResponse) {

        this.responder.doRespond(this);

      }

    }

  }

  /** Listens on the socket. Creates jobs for the handler threads*/

  private class Listener extends Thread {

    private ServerSocketChannel acceptChannel = null; //the accept channel

    private Selector selector = null; //the selector that we use for the server

    private Reader[] readers = null;

    private int currentReader = 0;

    private InetSocketAddress address; //the address we bind at

    private Random rand = new Random();

    private long lastCleanupRunTime = 0; //the last time when a cleanup connec-

                                         //-tion (for idle connections) ran

    private long cleanupInterval = 10000; //the minimum interval between

                                          //two cleanup runs

    private int backlogLength = conf.getInt("ipc.server.listen.queue.size", 128);

    private ExecutorService readPool;

    public Listener() throws IOException {

      address = new InetSocketAddress(bindAddress, port);

      // Create a new server socket and set to non blocking mode

      acceptChannel = ServerSocketChannel.open();

      acceptChannel.configureBlocking(false);

      // Bind the server socket to the local host and port

      bind(acceptChannel.socket(), address, backlogLength);

      port = acceptChannel.socket().getLocalPort(); //Could be an ephemeral port

      // create a selector;

      selector= Selector.open();

      readers = new Reader[readThreads];

      readPool = Executors.newFixedThreadPool(readThreads,

        new ThreadFactoryBuilder().setNameFormat(

          "IPC Reader %d on port " + port).setDaemon(true).build());

      for (int i = 0; i < readThreads; ++i) {

        Reader reader = new Reader();

        readers[i] = reader;

        readPool.execute(reader);

      }

      LOG.info(getName() + ": started " + readThreads + " reader(s) in Listener.");

      // Register accepts on the server socket with the selector.

      acceptChannel.register(selector, SelectionKey.OP_ACCEPT);

      this.setName("IPC Server listener on " + port);

      this.setDaemon(true);

    }

    private class Reader implements Runnable {

      private volatile boolean adding = false;

      private final Selector readSelector;

      Reader() throws IOException {

        this.readSelector = Selector.open();

      }

      public void run() {

        try {

          doRunLoop();

        } finally {

          try {

            readSelector.close();

          } catch (IOException ioe) {

            LOG.error(getName() + ": error closing read selector in " + getName(), ioe);

          }

        }

      }

      private synchronized void doRunLoop() {

        while (running) {

          SelectionKey key = null;

          try {

            readSelector.select();

            while (adding) {

              this.wait(1000);

            }

            Iterator<SelectionKey> iter = readSelector.selectedKeys().iterator();

            while (iter.hasNext()) {

              key = iter.next();

              iter.remove();

              if (key.isValid()) {

                if (key.isReadable()) {

                  doRead(key);

                }

              }

              key = null;

            }

          } catch (InterruptedException e) {

            if (running) {                      // unexpected -- log it

              LOG.info(getName() + ": unexpectedly interrupted: " +

                StringUtils.stringifyException(e));

            }

          } catch (IOException ex) {

            LOG.error(getName() + ": error in Reader", ex);

          }

        }

      }

      /**

       * This gets reader into the state that waits for the new channel

       * to be registered with readSelector. If it was waiting in select()

       * the thread will be woken up, otherwise whenever select() is called

       * it will return even if there is nothing to read and wait

       * in while(adding) for finishAdd call

       */

      public void startAdd() {

        adding = true;

        readSelector.wakeup();

      }

      public synchronized SelectionKey registerChannel(SocketChannel channel)

        throws IOException {

        return channel.register(readSelector, SelectionKey.OP_READ);

      }

      public synchronized void finishAdd() {

        adding = false;

        this.notify();

      }

    }

    /** cleanup connections from connectionList. Choose a random range

     * to scan and also have a limit on the number of the connections

     * that will be cleanedup per run. The criteria for cleanup is the time

     * for which the connection was idle. If 'force' is true then all

     * connections will be looked at for the cleanup.

     * @param force all connections will be looked at for cleanup

     */

    private void cleanupConnections(boolean force) {

      if (force || numConnections > thresholdIdleConnections) {

        long currentTime = System.currentTimeMillis();

        if (!force && (currentTime - lastCleanupRunTime) < cleanupInterval) {

          return;

        }

        int start = 0;

        int end = numConnections - 1;

        if (!force) {

          start = rand.nextInt() % numConnections;

          end = rand.nextInt() % numConnections;

          int temp;

          if (end < start) {

            temp = start;

            start = end;

            end = temp;

          }

        }

        int i = start;

        int numNuked = 0;

        while (i <= end) {

          Connection c;

          synchronized (connectionList) {

            try {

              c = connectionList.get(i);

            } catch (Exception e) {return;}

          }

          if (c.timedOut(currentTime)) {

            if (LOG.isDebugEnabled())

              LOG.debug(getName() + ": disconnecting client " + c.getHostAddress());

            closeConnection(c);

            numNuked++;

            end--;

            //noinspection UnusedAssignment

            c = null;

            if (!force && numNuked == maxConnectionsToNuke) break;

          }

          else i++;

        }

        lastCleanupRunTime = System.currentTimeMillis();

      }

    }

    @Override

    public void run() {

      LOG.info(getName() + ": starting");

      SERVER.set(HBaseServer.this);

      while (running) {

        SelectionKey key = null;

        try {

          selector.select(); // FindBugs IS2_INCONSISTENT_SYNC

          Iterator<SelectionKey> iter = selector.selectedKeys().iterator();

          while (iter.hasNext()) {

            key = iter.next();

            iter.remove();

            try {

              if (key.isValid()) {

                if (key.isAcceptable())

                  doAccept(key);

              }

            } catch (IOException ignored) {

            }

            key = null;

          }

        } catch (OutOfMemoryError e) {

          if (errorHandler != null) {

            if (errorHandler.checkOOME(e)) {

              LOG.info(getName() + ": exiting on OutOfMemoryError");

              closeCurrentConnection(key, e);

              cleanupConnections(true);

              return;

            }

          } else {

            // we can run out of memory if we have too many threads

            // log the event and sleep for a minute and give

            // some thread(s) a chance to finish

            LOG.warn(getName() + ": OutOfMemoryError in server select", e);

            closeCurrentConnection(key, e);

            cleanupConnections(true);

            try { Thread.sleep(60000); } catch (Exception ignored) {}

          }

        } catch (Exception e) {

          closeCurrentConnection(key, e);

        }

        cleanupConnections(false);

      }

      LOG.info(getName() + ": stopping");

      synchronized (this) {

        try {

          acceptChannel.close();

          selector.close();

        } catch (IOException ignored) { }

        selector= null;

        acceptChannel= null;

        // clean up all connections

        while (!connectionList.isEmpty()) {

          closeConnection(connectionList.remove(0));

        }

      }

    }

    private void closeCurrentConnection(SelectionKey key, Throwable e) {

      if (key != null) {

        Connection c = (Connection)key.attachment();

        if (c != null) {

          if (LOG.isDebugEnabled()) {

            LOG.debug(getName() + ": disconnecting client " + c.getHostAddress() +

                (e != null ? " on error " + e.getMessage() : ""));

          }

          closeConnection(c);

          key.attach(null);

        }

      }

    }

    InetSocketAddress getAddress() {

      return (InetSocketAddress)acceptChannel.socket().getLocalSocketAddress();

    }

    void doAccept(SelectionKey key) throws IOException, OutOfMemoryError {

      Connection c;

      ServerSocketChannel server = (ServerSocketChannel) key.channel();

      SocketChannel channel;

      while ((channel = server.accept()) != null) {

        channel.configureBlocking(false);

        channel.socket().setTcpNoDelay(tcpNoDelay);

        channel.socket().setKeepAlive(tcpKeepAlive);

        Reader reader = getReader();

        try {

          reader.startAdd();

          SelectionKey readKey = reader.registerChannel(channel);

          c = getConnection(channel, System.currentTimeMillis());

          readKey.attach(c);

          synchronized (connectionList) {

            connectionList.add(numConnections, c);

            numConnections++;

          }

          if (LOG.isDebugEnabled())

            LOG.debug(getName() + ": connection from " + c.toString() +

                "; # active connections: " + numConnections +

                "; # queued calls: " + callQueue.size());

        } finally {

          reader.finishAdd();

        }

      }

    }

    void doRead(SelectionKey key) throws InterruptedException {

      int count = 0;

      Connection c = (Connection)key.attachment();

      if (c == null) {

        return;

      }

      c.setLastContact(System.currentTimeMillis());

      try {

        count = c.readAndProcess();

      } catch (InterruptedException ieo) {

        throw ieo;

      } catch (Exception e) {

        LOG.warn(getName() + ": count of bytes read: " + count, e);

        count = -1; //so that the (count < 0) block is executed

      }

      if (count < 0) {

        if (LOG.isDebugEnabled()) {

          LOG.debug(getName() + ": disconnecting client " + c.getHostAddress() +

            ", because count=" + count +

            ". Number of active connections: " + numConnections);

        }

        closeConnection(c);

        // c = null;

      } else {

        c.setLastContact(System.currentTimeMillis());

      }

    }

    synchronized void doStop() {

      if (selector != null) {

        selector.wakeup();

        Thread.yield();

      }

      if (acceptChannel != null) {

        try {

          acceptChannel.socket().close();

        } catch (IOException e) {

          LOG.info(getName() + ": exception in closing listener socket. " + e);

        }

      }

      readPool.shutdownNow();

    }

    // The method that will return the next reader to work with

    // Simplistic implementation of round robin for now

    Reader getReader() {

      currentReader = (currentReader + 1) % readers.length;

      return readers[currentReader];

    }

  }

  // Sends responses of RPC back to clients.

    private final Selector writeSelector;

    private int pending;         // connections waiting to register

    Responder() throws IOException {

      this.setName("IPC Server Responder");

      this.setDaemon(true);

      writeSelector = Selector.open(); // create a selector

      pending = 0;

    }

    @Override

    public void run() {

      LOG.info(getName() + ": starting");

      SERVER.set(HBaseServer.this);

      try {

        doRunLoop();

      } finally {

        LOG.info(getName() + ": stopping");

        try {

          writeSelector.close();

        } catch (IOException ioe) {

          LOG.error(getName() + ": couldn't close write selector", ioe);

        }

      }

    }

    private void doRunLoop() {

      long lastPurgeTime = 0;   // last check for old calls.

      while (running) {

        try {

          waitPending();     // If a channel is being registered, wait.

          writeSelector.select(purgeTimeout);

          Iterator<SelectionKey> iter = writeSelector.selectedKeys().iterator();

          while (iter.hasNext()) {

            SelectionKey key = iter.next();

            iter.remove();

            try {

              if (key.isValid() && key.isWritable()) {

                  doAsyncWrite(key);

              }

            } catch (IOException e) {

              LOG.info(getName() + ": asyncWrite", e);

            }

          }

          long now = System.currentTimeMillis();

          if (now < lastPurgeTime + purgeTimeout) {

            continue;

          }

          lastPurgeTime = now;

          //

          // If there were some calls that have not been sent out for a

          // long time, discard them.

          //

          if (LOG.isDebugEnabled()) LOG.debug(getName() + ": checking for old call responses.");

          ArrayList<Call> calls;

          // get the list of channels from list of keys.

          synchronized (writeSelector.keys()) {

            calls = new ArrayList<Call>(writeSelector.keys().size());

            iter = writeSelector.keys().iterator();

            while (iter.hasNext()) {

              SelectionKey key = iter.next();

              Call call = (Call)key.attachment();

              if (call != null && key.channel() == call.connection.channel) {