*

* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* HMaster is the "master server" for HBase. An HBase cluster has one active

* master.  If many masters are started, all compete.  Whichever wins goes on to

* run the cluster.  All others park themselves in their constructor until

* master or cluster shutdown or until the active master loses its lease in

* zookeeper.  Thereafter, all running master jostle to take over master role.

*

* <p>The Master can be asked shutdown the cluster. See {@link #shutdown()}.  In

* this case it will tell all regionservers to go down and then wait on them

* all reporting in that they are down.  This master will then shut itself down.

*

* <p>You can also shutdown just this master.  Call {@link #stopMaster()}.

*

* @see Watcher

*/

@SuppressWarnings("deprecation")

RegionServerStatusProtos.RegionServerStatusService.BlockingInterface,

MasterServices, Server {

  private static final Log LOG = LogFactory.getLog(HMaster.class.getName());

  // MASTER is name of the webapp and the attribute name used stuffing this

  //instance into web context.

  public static final String MASTER = "master";

  // The configuration for the Master

  private final Configuration conf;

  // server for the web ui

  private InfoServer infoServer;

  // Our zk client.

  private ZooKeeperWatcher zooKeeper;

  // Manager and zk listener for master election

  private ActiveMasterManager activeMasterManager;

  // Region server tracker

  RegionServerTracker regionServerTracker;

  // Draining region server tracker

  private DrainingServerTracker drainingServerTracker;

  // Tracker for load balancer state

  private LoadBalancerTracker loadBalancerTracker;

  // master address manager and watcher

  private MasterAddressTracker masterAddressManager;

  // RPC server for the HMaster

  private final RpcServerInterface rpcServer;

  private JvmPauseMonitor pauseMonitor;

  // Set after we've called HBaseServer#openServer and ready to receive RPCs.

  // Set back to false after we stop rpcServer.  Used by tests.

  private volatile boolean rpcServerOpen = false;

  /** Namespace stuff */

  private TableNamespaceManager tableNamespaceManager;

  private NamespaceJanitor namespaceJanitorChore;

  /**

   * This servers address.

   */

  private final InetSocketAddress isa;

  // Metrics for the HMaster

  private final MetricsMaster metricsMaster;

  // file system manager for the master FS operations

  private MasterFileSystem fileSystemManager;

  // server manager to deal with region server info

  ServerManager serverManager;

  // manager of assignment nodes in zookeeper

  AssignmentManager assignmentManager;

  // manager of catalog regions

  private CatalogTracker catalogTracker;

  // Cluster status zk tracker and local setter

  private ClusterStatusTracker clusterStatusTracker;

  // buffer for "fatal error" notices from region servers

  // in the cluster. This is only used for assisting

  // operations/debugging.

  private MemoryBoundedLogMessageBuffer rsFatals;

  // This flag is for stopping this Master instance.  Its set when we are

  // stopping or aborting

  private volatile boolean stopped = false;

  // Set on abort -- usually failure of our zk session.

  private volatile boolean abort = false;

  // flag set after we become the active master (used for testing)

  private volatile boolean isActiveMaster = false;

  // flag set after we complete initialization once active,

  // it is not private since it's used in unit tests

  volatile boolean initialized = false;

  // flag set after we complete assignMeta.

  private volatile boolean serverShutdownHandlerEnabled = false;

  // Instance of the hbase executor service.

  ExecutorService executorService;

  private LoadBalancer balancer;

  private Thread balancerChore;

  private Thread clusterStatusChore;

  private ClusterStatusPublisher clusterStatusPublisherChore = null;

  private CatalogJanitor catalogJanitorChore;

  private LogCleaner logCleaner;

  private HFileCleaner hfileCleaner;

  private MasterCoprocessorHost cpHost;

  private final ServerName serverName;

  private TableDescriptors tableDescriptors;

  // Table level lock manager for schema changes

  private TableLockManager tableLockManager;

  // Time stamps for when a hmaster was started and when it became active

  private long masterStartTime;

  private long masterActiveTime;

  /** time interval for emitting metrics values */

  private final int msgInterval;

  /**

   * MX Bean for MasterInfo

   */

  private ObjectName mxBean = null;

  //should we check the compression codec type at master side, default true, HBASE-6370

  private final boolean masterCheckCompression;

  private SpanReceiverHost spanReceiverHost;

  private Map<String, Service> coprocessorServiceHandlers = Maps.newHashMap();

  // monitor for snapshot of hbase tables

  private SnapshotManager snapshotManager;

  /** The health check chore. */

  private HealthCheckChore healthCheckChore;

  /**

   * is in distributedLogReplay mode. When true, SplitLogWorker directly replays WAL edits to newly

   * assigned region servers instead of creating recovered.edits files.

   */

  private final boolean distributedLogReplay;

  /** flag used in test cases in order to simulate RS failures during master initialization */

  private volatile boolean initializationBeforeMetaAssignment = false;

  /** The following is used in master recovery scenario to re-register listeners */

  private List<ZooKeeperListener> registeredZKListenersBeforeRecovery;

  /**

   * Initializes the HMaster. The steps are as follows:

   * <p>

   * <ol>

   * <li>Initialize HMaster RPC and address

   * <li>Connect to ZooKeeper.

   * </ol>

   * <p>

   * Remaining steps of initialization occur in {@link #run()} so that they

   * run in their own thread rather than within the context of the constructor.

   * @throws InterruptedException

   */

  public HMaster(final Configuration conf)

  throws IOException, KeeperException, InterruptedException {

    this.conf = new Configuration(conf);

    // Disable the block cache on the master

    this.conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);

    FSUtils.setupShortCircuitRead(conf);

    // Server to handle client requests.

    String hostname = Strings.domainNamePointerToHostName(DNS.getDefaultHost(

      conf.get("hbase.master.dns.interface", "default"),

      conf.get("hbase.master.dns.nameserver", "default")));

    int port = conf.getInt(HConstants.MASTER_PORT, HConstants.DEFAULT_MASTER_PORT);

    // Test that the hostname is reachable

    InetSocketAddress initialIsa = new InetSocketAddress(hostname, port);

    if (initialIsa.getAddress() == null) {

      throw new IllegalArgumentException("Failed resolve of hostname " + initialIsa);

    }

    // Verify that the bind address is reachable if set

    String bindAddress = conf.get("hbase.master.ipc.address");

    if (bindAddress != null) {

      initialIsa = new InetSocketAddress(bindAddress, port);

      if (initialIsa.getAddress() == null) {

        throw new IllegalArgumentException("Failed resolve of bind address " + initialIsa);

      }

    }

    String name = "master/" + initialIsa.toString();

    // Set how many times to retry talking to another server over HConnection.

    HConnectionManager.setServerSideHConnectionRetries(this.conf, name, LOG);

    int numHandlers = conf.getInt(HConstants.MASTER_HANDLER_COUNT,

      conf.getInt(HConstants.REGION_SERVER_HANDLER_COUNT, HConstants.DEFAULT_MASTER_HANLDER_COUNT));

    this.rpcServer = new RpcServer(this, name, getServices(),

      initialIsa, // BindAddress is IP we got for this server.

      conf,

      new FifoRpcScheduler(conf, numHandlers));

    // Set our address.

    this.isa = this.rpcServer.getListenerAddress();

    // We don't want to pass isa's hostname here since it could be 0.0.0.0

    this.serverName = ServerName.valueOf(hostname, this.isa.getPort(), System.currentTimeMillis());

    this.rsFatals = new MemoryBoundedLogMessageBuffer(

      conf.getLong("hbase.master.buffer.for.rs.fatals", 1*1024*1024));

    // login the zookeeper client principal (if using security)

    ZKUtil.loginClient(this.conf, "hbase.zookeeper.client.keytab.file",

      "hbase.zookeeper.client.kerberos.principal", this.isa.getHostName());

    // initialize server principal (if using secure Hadoop)

    UserProvider provider = UserProvider.instantiate(conf);

    provider.login("hbase.master.keytab.file",

      "hbase.master.kerberos.principal", this.isa.getHostName());

    LOG.info("hbase.rootdir=" + FSUtils.getRootDir(this.conf) +

        ", hbase.cluster.distributed=" + this.conf.getBoolean("hbase.cluster.distributed", false));

    // set the thread name now we have an address

    setName(MASTER + ":" + this.serverName.toShortString());

    Replication.decorateMasterConfiguration(this.conf);

    // Hack! Maps DFSClient => Master for logs.  HDFS made this

    // config param for task trackers, but we can piggyback off of it.

    if (this.conf.get("mapred.task.id") == null) {

      this.conf.set("mapred.task.id", "hb_m_" + this.serverName.toString());

    }

    this.zooKeeper = new ZooKeeperWatcher(conf, MASTER + ":" + isa.getPort(), this, true);

    this.rpcServer.startThreads();

    this.pauseMonitor = new JvmPauseMonitor(conf);

    this.pauseMonitor.start();

    // metrics interval: using the same property as region server.

    this.msgInterval = conf.getInt("hbase.regionserver.msginterval", 3 * 1000);

    //should we check the compression codec type at master side, default true, HBASE-6370

    this.masterCheckCompression = conf.getBoolean("hbase.master.check.compression", true);

    this.metricsMaster = new MetricsMaster( new MetricsMasterWrapperImpl(this));

    // Health checker thread.

    int sleepTime = this.conf.getInt(HConstants.HEALTH_CHORE_WAKE_FREQ,

      HConstants.DEFAULT_THREAD_WAKE_FREQUENCY);

    if (isHealthCheckerConfigured()) {

      healthCheckChore = new HealthCheckChore(sleepTime, this, getConfiguration());

    }

    // Do we publish the status?

    boolean shouldPublish = conf.getBoolean(HConstants.STATUS_PUBLISHED,

        HConstants.STATUS_PUBLISHED_DEFAULT);

    Class<? extends ClusterStatusPublisher.Publisher> publisherClass =

        conf.getClass(ClusterStatusPublisher.STATUS_PUBLISHER_CLASS,

            ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS,

            ClusterStatusPublisher.Publisher.class);

    if (shouldPublish) {

      if (publisherClass == null) {

        LOG.warn(HConstants.STATUS_PUBLISHED + " is true, but " +

            ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS +

            " is not set - not publishing status");

      } else {

        clusterStatusPublisherChore = new ClusterStatusPublisher(this, conf, publisherClass);

        Threads.setDaemonThreadRunning(clusterStatusPublisherChore.getThread());

      }

    }

    distributedLogReplay = this.conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY,

      HConstants.DEFAULT_DISTRIBUTED_LOG_REPLAY_CONFIG);

  }

  /**

   * @return list of blocking services and their security info classes that this server supports

   */

  private List<BlockingServiceAndInterface> getServices() {

    List<BlockingServiceAndInterface> bssi = new ArrayList<BlockingServiceAndInterface>(3);

    bssi.add(new BlockingServiceAndInterface(

        MasterProtos.MasterService.newReflectiveBlockingService(this),

        MasterProtos.MasterService.BlockingInterface.class));

    bssi.add(new BlockingServiceAndInterface(

        RegionServerStatusProtos.RegionServerStatusService.newReflectiveBlockingService(this),

        RegionServerStatusProtos.RegionServerStatusService.BlockingInterface.class));

    return bssi;

  }

  /**

   * Stall startup if we are designated a backup master; i.e. we want someone

   * else to become the master before proceeding.

   * @param c configuration

   * @param amm

   * @throws InterruptedException

   */

  private static void stallIfBackupMaster(final Configuration c,

      final ActiveMasterManager amm)

  throws InterruptedException {