* Copyright 2011 The Apache Software Foundation

*

* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* HMaster is the "master server" for HBase. An HBase cluster has one active

* master.  If many masters are started, all compete.  Whichever wins goes on to

* run the cluster.  All others park themselves in their constructor until

* master or cluster shutdown or until the active master loses its lease in

* zookeeper.  Thereafter, all running master jostle to take over master role.

*

* <p>The Master can be asked shutdown the cluster. See {@link #shutdown()}.  In

* this case it will tell all regionservers to go down and then wait on them

* all reporting in that they are down.  This master will then shut itself down.

*

* <p>You can also shutdown just this master.  Call {@link #stopMaster()}.

*

* @see HMasterInterface

* @see HMasterRegionInterface

* @see Watcher

*/

Server {

  private static final Log LOG = LogFactory.getLog(HMaster.class.getName());

  // MASTER is name of the webapp and the attribute name used stuffing this

  //instance into web context.

  public static final String MASTER = "master";

  // The configuration for the Master

  private final Configuration conf;

  // server for the web ui

  private InfoServer infoServer;

  // Our zk client.

  private ZooKeeperWatcher zooKeeper;

  // Manager and zk listener for master election

  private ActiveMasterManager activeMasterManager;

  // Region server tracker

  private RegionServerTracker regionServerTracker;

  // Draining region server tracker

  private DrainingServerTracker drainingServerTracker;

  // RPC server for the HMaster

  private final RpcServer rpcServer;

  /**

   * This servers address.

   */

  private final InetSocketAddress isa;

  // Metrics for the HMaster

  private final MasterMetrics metrics;

  // file system manager for the master FS operations

  private MasterFileSystem fileSystemManager;

  // server manager to deal with region server info

  private ServerManager serverManager;

  // manager of assignment nodes in zookeeper

  AssignmentManager assignmentManager;

  // manager of catalog regions

  private CatalogTracker catalogTracker;

  // Cluster status zk tracker and local setter

  private ClusterStatusTracker clusterStatusTracker;

 

  // buffer for "fatal error" notices from region servers

  // in the cluster. This is only used for assisting

  // operations/debugging.

  private MemoryBoundedLogMessageBuffer rsFatals;

  // This flag is for stopping this Master instance.  Its set when we are

  // stopping or aborting

  private volatile boolean stopped = false;

  // Set on abort -- usually failure of our zk session.

  private volatile boolean abort = false;

  // flag set after we become the active master (used for testing)

  private volatile boolean isActiveMaster = false;

  // flag set after we complete initialization once active (used for testing)

  private volatile boolean initialized = false;

  // flag set after we complete assignRootAndMeta.

  private volatile boolean serverShutdownHandlerEnabled = false;

  // Instance of the hbase executor service.

  ExecutorService executorService;

  private LoadBalancer balancer;

  private Thread balancerChore;

  // If 'true', the balancer is 'on'.  If 'false', the balancer will not run.

  private volatile boolean balanceSwitch = true;

  private CatalogJanitor catalogJanitorChore;

  private LogCleaner logCleaner;

  private MasterCoprocessorHost cpHost;

  private final ServerName serverName;

  private TableDescriptors tableDescriptors;

  // Time stamps for when a hmaster was started and when it became active

  private long masterStartTime;

  private long masterActiveTime;

  /**

   * MX Bean for MasterInfo

   */

  private ObjectName mxBean = null;

  /**

   * Initializes the HMaster. The steps are as follows:

   * <p>

   * <ol>

   * <li>Initialize HMaster RPC and address

   * <li>Connect to ZooKeeper.

   * </ol>

   * <p>

   * Remaining steps of initialization occur in {@link #run()} so that they

   * run in their own thread rather than within the context of the constructor.

   * @throws InterruptedException

   */

  public HMaster(final Configuration conf)

  throws IOException, KeeperException, InterruptedException {

    this.conf = new Configuration(conf);

    // Disable the block cache on the master

    this.conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);

    // Set how many times to retry talking to another server over HConnection.

    HConnectionManager.setServerSideHConnectionRetries(this.conf, LOG);

    // Server to handle client requests.

    String hostname = Strings.domainNamePointerToHostName(DNS.getDefaultHost(

      conf.get("hbase.master.dns.interface", "default"),

      conf.get("hbase.master.dns.nameserver", "default")));

    int port = conf.getInt(HConstants.MASTER_PORT, HConstants.DEFAULT_MASTER_PORT);

    // Creation of a HSA will force a resolve.

    InetSocketAddress initialIsa = new InetSocketAddress(hostname, port);

    if (initialIsa.getAddress() == null) {

      throw new IllegalArgumentException("Failed resolve of " + this.isa);

    }

    int numHandlers = conf.getInt("hbase.master.handler.count",

      conf.getInt("hbase.regionserver.handler.count", 25));

    this.rpcServer = HBaseRPC.getServer(this,

      new Class<?>[]{HMasterInterface.class, HMasterRegionInterface.class},

        initialIsa.getHostName(), // BindAddress is IP we got for this server.

        initialIsa.getPort(),

        numHandlers,

        0, // we dont use high priority handlers in master

        conf.getBoolean("hbase.rpc.verbose", false), conf,

        0); // this is a DNC w/o high priority handlers

    // Set our address.

    this.isa = this.rpcServer.getListenerAddress();

    this.serverName = new ServerName(this.isa.getHostName(),

      this.isa.getPort(), System.currentTimeMillis());

    this.rsFatals = new MemoryBoundedLogMessageBuffer(

        conf.getLong("hbase.master.buffer.for.rs.fatals", 1*1024*1024));

    // initialize server principal (if using secure Hadoop)

    User.login(conf, "hbase.master.keytab.file",

      "hbase.master.kerberos.principal", this.isa.getHostName());

    // set the thread name now we have an address

    setName(MASTER + "-" + this.serverName.toString());

    Replication.decorateMasterConfiguration(this.conf);

    // Hack! Maps DFSClient => Master for logs.  HDFS made this

    // config param for task trackers, but we can piggyback off of it.

    if (this.conf.get("mapred.task.id") == null) {

      this.conf.set("mapred.task.id", "hb_m_" + this.serverName.toString());

    }

    this.zooKeeper = new ZooKeeperWatcher(conf, MASTER + ":" + isa.getPort(), this, true);

    this.rpcServer.startThreads();

    this.metrics = new MasterMetrics(getServerName().toString());

  }

  /**

   * Stall startup if we are designated a backup master; i.e. we want someone

   * else to become the master before proceeding.

   * @param c

   * @param amm

   * @throws InterruptedException

   */

  private static void stallIfBackupMaster(final Configuration c,

      final ActiveMasterManager amm)

  throws InterruptedException {

    // If we're a backup master, stall until a primary to writes his address

    if (!c.getBoolean(HConstants.MASTER_TYPE_BACKUP,

      HConstants.DEFAULT_MASTER_TYPE_BACKUP)) {

      return;

    }

    LOG.debug("HMaster started in backup mode.  " +

      "Stalling until master znode is written.");

    // This will only be a minute or so while the cluster starts up,

    // so don't worry about setting watches on the parent znode

    while (!amm.isActiveMaster()) {

      LOG.debug("Waiting for master address ZNode to be written " +

        "(Also watching cluster state node)");

      Thread.sleep(c.getInt("zookeeper.session.timeout", 180 * 1000));

    }

   

  }

  /**

   * Main processing loop for the HMaster.

   * <ol>

   * <li>Block until becoming active master

   * <li>Finish initialization via finishInitialization(MonitoredTask)

   * <li>Enter loop until we are stopped

   * <li>Stop services and perform cleanup once stopped

   * </ol>

   */

  @Override

  public void run() {

    MonitoredTask startupStatus =

      TaskMonitor.get().createStatus("Master startup");

    startupStatus.setDescription("Master startup");

    masterStartTime = System.currentTimeMillis();

    try {

      /*

       * Block on becoming the active master.

       *

       * We race with other masters to write our address into ZooKeeper.  If we

       * succeed, we are the primary/active master and finish initialization.

       *

       * If we do not succeed, there is another active master and we should

       * now wait until it dies to try and become the next active master.  If we

       * do not succeed on our first attempt, this is no longer a cluster startup.

       */

      becomeActiveMaster(startupStatus);

      // We are either the active master or we were asked to shutdown

      if (!this.stopped) {

        finishInitialization(startupStatus, false);

        loop();

      }

    } catch (Throwable t) {

      // HBASE-5680: Likely hadoop23 vs hadoop 20.x/1.x incompatibility

      if (t instanceof NoClassDefFoundError &&

          t.getMessage().contains("org/apache/hadoop/hdfs/protocol/FSConstants$SafeModeAction")) {

          // improved error message for this special case

          abort("HBase is having a problem with its Hadoop jars.  You may need to "

              + "recompile HBase against Hadoop version "

              +  org.apache.hadoop.util.VersionInfo.getVersion()

              + " or change your hadoop jars to start properly", t);

      } else {

        abort("Unhandled exception. Starting shutdown.", t);

      }

    } finally {

      startupStatus.cleanup();

     

      stopChores();

      // Wait for all the remaining region servers to report in IFF we were

      // running a cluster shutdown AND we were NOT aborting.

      if (!this.abort && this.serverManager != null &&

          this.serverManager.isClusterShutdown()) {

        this.serverManager.letRegionServersShutdown();

      }

      stopServiceThreads();

      // Stop services started for both backup and active masters

      if (this.activeMasterManager != null) this.activeMasterManager.stop();

      if (this.catalogTracker != null) this.catalogTracker.stop();

      if (this.serverManager != null) this.serverManager.stop();

      if (this.assignmentManager != null) this.assignmentManager.stop();

      if (this.fileSystemManager != null) this.fileSystemManager.stop();

      this.zooKeeper.close();

    }

    LOG.info("HMaster main thread exiting");

  }

  /**

   * Try becoming active master.

   * @param startupStatus

   * @return True if we could successfully become the active master.

   * @throws InterruptedException

   */

  private boolean becomeActiveMaster(MonitoredTask startupStatus)

  throws InterruptedException {

    // TODO: This is wrong!!!! Should have new servername if we restart ourselves,

    // if we come back to life.

    this.activeMasterManager = new ActiveMasterManager(zooKeeper, this.serverName,

        this);

    this.zooKeeper.registerListener(activeMasterManager);

    stallIfBackupMaster(this.conf, this.activeMasterManager);

    // The ClusterStatusTracker is setup before the other

    // ZKBasedSystemTrackers because it's needed by the activeMasterManager

    // to check if the cluster should be shutdown.

    this.clusterStatusTracker = new ClusterStatusTracker(getZooKeeper(), this);

    this.clusterStatusTracker.start();

    return this.activeMasterManager.blockUntilBecomingActiveMaster(startupStatus,

        this.clusterStatusTracker);

  }

  /**

   * Initialize all ZK based system trackers.

   * @throws IOException

   * @throws InterruptedException

   */

  private void initializeZKBasedSystemTrackers() throws IOException,

      InterruptedException, KeeperException {

    this.catalogTracker = new CatalogTracker(this.zooKeeper, this.conf,

        this, conf.getInt("hbase.master.catalog.timeout", Integer.MAX_VALUE));

    this.catalogTracker.start();

    this.balancer = LoadBalancerFactory.getLoadBalancer(conf);

    this.assignmentManager = new AssignmentManager(this, serverManager,

        this.catalogTracker, this.balancer, this.executorService);

    zooKeeper.registerListenerFirst(assignmentManager);

    this.regionServerTracker = new RegionServerTracker(zooKeeper, this,

        this.serverManager);

    this.regionServerTracker.start();

    this.drainingServerTracker = new DrainingServerTracker(zooKeeper, this,

      this.serverManager);

    this.drainingServerTracker.start();

    // Set the cluster as up.  If new RSs, they'll be waiting on this before

    // going ahead with their startup.

    boolean wasUp = this.clusterStatusTracker.isClusterUp();

    if (!wasUp) this.clusterStatusTracker.setClusterUp();

    LOG.info("Server active/primary master; " + this.serverName +

        ", sessionid=0x" +

        Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()) +

        ", cluster-up flag was=" + wasUp);

  }

  // Check if we should stop every second.

  private Sleeper stopSleeper = new Sleeper(1000, this);

  private void loop() {

    while (!this.stopped) {

      stopSleeper.sleep();

    }

  }

  /**

   * Finish initialization of HMaster after becoming the primary master.

   *

   * <ol>

   * <li>Initialize master components - file system manager, server manager,

   *     assignment manager, region server tracker, catalog tracker, etc</li>

   * <li>Start necessary service threads - rpc server, info server,

   *     executor services, etc</li>

   * <li>Set cluster as UP in ZooKeeper</li>

   * <li>Wait for RegionServers to check-in</li>

   * <li>Split logs and perform data recovery, if necessary</li>

   * <li>Ensure assignment of root and meta regions<li>

   * <li>Handle either fresh cluster start or master failover</li>

   * </ol>

   * @param masterRecovery

   *

   * @throws IOException

   * @throws InterruptedException

   * @throws KeeperException

   */

  private void finishInitialization(MonitoredTask status, boolean masterRecovery)

  throws IOException, InterruptedException, KeeperException {

    isActiveMaster = true;

    /*

     * We are active master now... go initialize components we need to run.

     * Note, there may be dross in zk from previous runs; it'll get addressed

     * below after we determine if cluster startup or failover.

     */

    status.setStatus("Initializing Master file system");

    this.masterActiveTime = System.currentTimeMillis();

    // TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring.

    this.fileSystemManager = new MasterFileSystem(this, this, metrics, masterRecovery);

    this.tableDescriptors =

      new FSTableDescriptors(this.fileSystemManager.getFileSystem(),

      this.fileSystemManager.getRootDir());

    // publish cluster ID

    status.setStatus("Publishing Cluster ID in ZooKeeper");

    ClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId());

    if (!masterRecovery) {

      this.executorService = new ExecutorService(getServerName().toString());

      this.serverManager = new ServerManager(this, this);

    }

    status.setStatus("Initializing ZK system trackers");

    initializeZKBasedSystemTrackers();

   

    if (!masterRecovery) {

      // initialize master side coprocessors before we start handling requests

      status.setStatus("Initializing master coprocessors");

      this.cpHost = new MasterCoprocessorHost(this, this.conf);

      // start up all service threads.

      status.setStatus("Initializing master service threads");

      startServiceThreads();

    }

    // Wait for region servers to report in.

    this.serverManager.waitForRegionServers(status);

    // Check zk for regionservers that are up but didn't register

    for (ServerName sn: this.regionServerTracker.getOnlineServers()) {

      if (!this.serverManager.isServerOnline(sn)) {

        // Not registered; add it.

        LOG.info("Registering server found up in zk but who has not yet " +

          "reported in: " + sn);

        this.serverManager.recordNewServer(sn, HServerLoad.EMPTY_HSERVERLOAD);

      }

    }

    if (!masterRecovery) {

      this.assignmentManager.startTimeOutMonitor();

    }

    // TODO: Should do this in background rather than block master startup

    status.setStatus("Splitting logs after master startup");

    splitLogAfterStartup(this.fileSystemManager);

    // Make sure root and meta assigned before proceeding.

    assignRootAndMeta(status);

    enableServerShutdownHandler();

    // Update meta with new HRI if required. i.e migrate all HRI with HTD to

    // HRI with out HTD in meta and update the status in ROOT. This must happen

    // before we assign all user regions or else the assignment will fail.

    // TODO: Remove this when we do 0.94.

    org.apache.hadoop.hbase.catalog.MetaMigrationRemovingHTD.

      updateMetaWithNewHRI(this);

    // Fixup assignment manager status

    status.setStatus("Starting assignment manager");

    this.assignmentManager.joinCluster();

    this.balancer.setClusterStatus(getClusterStatus());

    this.balancer.setMasterServices(this);

    // Fixing up missing daughters if any

    status.setStatus("Fixing up missing daughters");

    fixupDaughters(status);

    if (!masterRecovery) {

      // Start balancer and meta catalog janitor after meta and regions have

      // been assigned.

      status.setStatus("Starting balancer and catalog janitor");

      this.balancerChore = getAndStartBalancerChore(this);

      this.catalogJanitorChore = new CatalogJanitor(this, this);

      startCatalogJanitorChore();

      registerMBean();

    }

    status.markComplete("Initialization successful");

    LOG.info("Master has completed initialization");

    initialized = true;

    // clear the dead servers with same host name and port of online server because we are not

    // removing dead server with same hostname and port of rs which is trying to check in before

    // master initialization. See HBASE-5916.

    this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();

   

    if (!masterRecovery) {

      if (this.cpHost != null) {

        // don't let cp initialization errors kill the master

        try {

          this.cpHost.postStartMaster();

        } catch (IOException ioe) {

          LOG.error("Coprocessor postStartMaster() hook failed", ioe);

        }

      }

    }

  }

 

  /**

   * If ServerShutdownHandler is disabled, we enable it and expire those dead

   * but not expired servers.

   *

   * @throws IOException

   */

  private void enableServerShutdownHandler() throws IOException {

    if (!serverShutdownHandlerEnabled) {

      serverShutdownHandlerEnabled = true;

      this.serverManager.expireDeadNotExpiredServers();

    }

  }

 

  /**

   * Useful for testing purpose also where we have

   * master restart scenarios.

   */

  protected void startCatalogJanitorChore() {

    Threads.setDaemonThreadRunning(catalogJanitorChore.getThread());

  }

  /**

   * Override to change master's splitLogAfterStartup. Used testing

   * @param mfs

   */

  protected void splitLogAfterStartup(final MasterFileSystem mfs) {

    mfs.splitLogAfterStartup();

  }

  /**

   * Check <code>-ROOT-</code> and <code>.META.</code> are assigned.  If not,

   * assign them.

   * @throws InterruptedException

   * @throws IOException

   * @throws KeeperException

   * @return Count of regions we assigned.

   */

  int assignRootAndMeta(MonitoredTask status)

  throws InterruptedException, IOException, KeeperException {

    int assigned = 0;

    long timeout = this.conf.getLong("hbase.catalog.verification.timeout", 1000);

    // Work on ROOT region.  Is it in zk in transition?

    status.setStatus("Assigning ROOT region");

    boolean rit = this.assignmentManager.

      processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.ROOT_REGIONINFO);

    ServerName currentRootServer = null;

    boolean rootRegionLocation = catalogTracker.verifyRootRegionLocation(timeout);

    if (!rit && !rootRegionLocation) {

      currentRootServer = this.catalogTracker.getRootLocation();

      splitLogAndExpireIfOnline(currentRootServer);

      this.assignmentManager.assignRoot();

      waitForRootAssignment();

      assigned++;

    } else if (rit && !rootRegionLocation) {

      waitForRootAssignment();

      assigned++;

    } else {

      // Region already assigned. We didn't assign it. Add to in-memory state.

      this.assignmentManager.regionOnline(HRegionInfo.ROOT_REGIONINFO,

          this.catalogTracker.getRootLocation());

    }

    // Enable the ROOT table if on process fail over the RS containing ROOT

    // was active.

    enableCatalogTables(Bytes.toString(HConstants.ROOT_TABLE_NAME));

    LOG.info("-ROOT- assigned=" + assigned + ", rit=" + rit +

      ", location=" + catalogTracker.getRootLocation());

    // Work on meta region

    status.setStatus("Assigning META region");

    rit = this.assignmentManager.

      processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.FIRST_META_REGIONINFO);

    boolean metaRegionLocation = this.catalogTracker.verifyMetaRegionLocation(timeout);

    if (!rit && !metaRegionLocation) {

      ServerName currentMetaServer =

        this.catalogTracker.getMetaLocationOrReadLocationFromRoot();

      if (currentMetaServer != null

          && !currentMetaServer.equals(currentRootServer)) {

        splitLogAndExpireIfOnline(currentMetaServer);

      }

      assignmentManager.assignMeta();

      enableSSHandWaitForMeta();

      assigned++;

    } else if (rit && !metaRegionLocation) {

      enableSSHandWaitForMeta();

      assigned++;

    } else {

      // Region already assigned.  We didnt' assign it.  Add to in-memory state.

      this.assignmentManager.regionOnline(HRegionInfo.FIRST_META_REGIONINFO,

        this.catalogTracker.getMetaLocation());

    }

    enableCatalogTables(Bytes.toString(HConstants.META_TABLE_NAME));

    LOG.info(".META. assigned=" + assigned + ", rit=" + rit +

      ", location=" + catalogTracker.getMetaLocation());

    status.setStatus("META and ROOT assigned.");

    return assigned;

  }

  private void enableSSHandWaitForMeta() throws IOException,

      InterruptedException {

    enableServerShutdownHandler();

    this.catalogTracker.waitForMeta();

    // Above check waits for general meta availability but this does not

    // guarantee that the transition has completed

    this.assignmentManager

        .waitForAssignment(HRegionInfo.FIRST_META_REGIONINFO);

  }

  private void waitForRootAssignment() throws InterruptedException {

    this.catalogTracker.waitForRoot();

    // This guarantees that the transition has completed

    this.assignmentManager.waitForAssignment(HRegionInfo.ROOT_REGIONINFO);

  }

  private void enableCatalogTables(String catalogTableName) {

    if (!this.assignmentManager.getZKTable().isEnabledTable(catalogTableName)) {

      this.assignmentManager.setEnabledTable(catalogTableName);

    }

  }

  void fixupDaughters(final MonitoredTask status) throws IOException {

    final Map<HRegionInfo, Result> offlineSplitParents =

      new HashMap<HRegionInfo, Result>();

    // This visitor collects offline split parents in the .META. table

    MetaReader.Visitor visitor = new MetaReader.Visitor() {

      @Override

      public boolean visit(Result r) throws IOException {

        if (r == null || r.isEmpty()) return true;

        HRegionInfo info =

          MetaReader.parseHRegionInfoFromCatalogResult(

            r, HConstants.REGIONINFO_QUALIFIER);

        if (info == null) return true; // Keep scanning

        if (info.isOffline() && info.isSplit()) {

          offlineSplitParents.put(info, r);

        }

        // Returning true means "keep scanning"

        return true;

      }

    };

    // Run full scan of .META. catalog table passing in our custom visitor

    MetaReader.fullScan(this.catalogTracker, visitor);

    // Now work on our list of found parents. See if any we can clean up.

    int fixups = 0;

    for (Map.Entry<HRegionInfo, Result> e : offlineSplitParents.entrySet()) {

      fixups += ServerShutdownHandler.fixupDaughters(

          e.getValue(), assignmentManager, catalogTracker);

    }

    if (fixups != 0) {

      LOG.info("Scanned the catalog and fixed up " + fixups +

        " missing daughter region(s)");

    }

  }

  /**

   * Split a server's log and expire it if we find it is one of the online

   * servers.

   * @param sn ServerName to check.

   * @throws IOException

   */

  private void splitLogAndExpireIfOnline(final ServerName sn)

      throws IOException {

    if (sn == null || !serverManager.isServerOnline(sn)) {

      return;

    }

    LOG.info("Forcing splitLog and expire of " + sn);

    fileSystemManager.splitLog(sn);

    serverManager.expireServer(sn);

  }

  @Override

  public ProtocolSignature getProtocolSignature(

      String protocol, long version, int clientMethodsHashCode)

  throws IOException {

    if (HMasterInterface.class.getName().equals(protocol)) {

      return new ProtocolSignature(HMasterInterface.VERSION, null);

    } else if (HMasterRegionInterface.class.getName().equals(protocol)) {

      return new ProtocolSignature(HMasterRegionInterface.VERSION, null);

    }

    throw new IOException("Unknown protocol: " + protocol);

  }

  public long getProtocolVersion(String protocol, long clientVersion) {

    if (HMasterInterface.class.getName().equals(protocol)) {

      return HMasterInterface.VERSION;

    } else if (HMasterRegionInterface.class.getName().equals(protocol)) {

      return HMasterRegionInterface.VERSION;

    }

    // unknown protocol

    LOG.warn("Version requested for unimplemented protocol: "+protocol);

    return -1;

  }

  @Override

  public TableDescriptors getTableDescriptors() {

    return this.tableDescriptors;

  }

  /** @return InfoServer object. Maybe null.*/

  public InfoServer getInfoServer() {

    return this.infoServer;

  }

  @Override

  public Configuration getConfiguration() {

    return this.conf;

  }

  @Override

  public ServerManager getServerManager() {

    return this.serverManager;

  }

  @Override

  public ExecutorService getExecutorService() {

    return this.executorService;

  }

  @Override

  public MasterFileSystem getMasterFileSystem() {

    return this.fileSystemManager;

  }

  /**

   * Get the ZK wrapper object - needed by master_jsp.java

   * @return the zookeeper wrapper

   */

  public ZooKeeperWatcher getZooKeeperWatcher() {

    return this.zooKeeper;

  }

  /*

   * Start up all services. If any of these threads gets an unhandled exception

   * then they just die with a logged message.  This should be fine because

   * in general, we do not expect the master to get such unhandled exceptions

   *  as OOMEs; it should be lightly loaded. See what HRegionServer does if

   *  need to install an unexpected exception handler.

   */

  private void startServiceThreads() throws IOException{

   // Start the executor service pools

   this.executorService.startExecutorService(ExecutorType.MASTER_OPEN_REGION,

      conf.getInt("hbase.master.executor.openregion.threads", 5));

   this.executorService.startExecutorService(ExecutorType.MASTER_CLOSE_REGION,

      conf.getInt("hbase.master.executor.closeregion.threads", 5));

   this.executorService.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS,

      conf.getInt("hbase.master.executor.serverops.threads", 3));

   this.executorService.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS,

      conf.getInt("hbase.master.executor.serverops.threads", 5));

  

   // We depend on there being only one instance of this executor running

   // at a time.  To do concurrency, would need fencing of enable/disable of

   // tables.

   this.executorService.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);

   // Start log cleaner thread

   String n = Thread.currentThread().getName();

   this.logCleaner =

      new LogCleaner(conf.getInt("hbase.master.cleaner.interval", 60 * 1000),

         this, conf, getMasterFileSystem().getFileSystem(),

         getMasterFileSystem().getOldLogDir());

         Threads.setDaemonThreadRunning(logCleaner.getThread(), n + ".oldLogCleaner");

   // Put up info server.

   int port = this.conf.getInt("hbase.master.info.port", 60010);

   if (port >= 0) {

     String a = this.conf.get("hbase.master.info.bindAddress", "0.0.0.0");

     this.infoServer = new InfoServer(MASTER, a, port, false, this.conf);

     this.infoServer.addServlet("status", "/master-status", MasterStatusServlet.class);

     this.infoServer.addServlet("dump", "/dump", MasterDumpServlet.class);

     this.infoServer.setAttribute(MASTER, this);

     this.infoServer.start();

    }

  

    // Start allowing requests to happen.

    this.rpcServer.openServer();

    if (LOG.isDebugEnabled()) {

      LOG.debug("Started service threads");

    }

  }

  private void stopServiceThreads() {

    if (LOG.isDebugEnabled()) {

      LOG.debug("Stopping service threads");

    }

    if (this.rpcServer != null) this.rpcServer.stop();

    // Clean up and close up shop

    if (this.logCleaner!= null) this.logCleaner.interrupt();

    if (this.infoServer != null) {

      LOG.info("Stopping infoServer");

      try {

        this.infoServer.stop();

      } catch (Exception ex) {

        ex.printStackTrace();

      }

    }

    if (this.executorService != null) this.executorService.shutdown();

  }

  private static Thread getAndStartBalancerChore(final HMaster master) {

    String name = master.getServerName() + "-BalancerChore";

    int balancerPeriod =

      master.getConfiguration().getInt("hbase.balancer.period", 300000);

    // Start up the load balancer chore

    Chore chore = new Chore(name, balancerPeriod, master) {

      @Override

      protected void chore() {

        master.balance();

      }

    };

    return Threads.setDaemonThreadRunning(chore.getThread());

  }

  private void stopChores() {

    if (this.balancerChore != null) {

      this.balancerChore.interrupt();

    }

    if (this.catalogJanitorChore != null) {

      this.catalogJanitorChore.interrupt();

    }

  }

  @Override

  public MapWritable regionServerStartup(final int port,

    final long serverStartCode, final long serverCurrentTime)

  throws IOException {

    // Register with server manager

    InetAddress ia = HBaseServer.getRemoteIp();

    ServerName rs = this.serverManager.regionServerStartup(ia, port,

      serverStartCode, serverCurrentTime);

    // Send back some config info

    MapWritable mw = createConfigurationSubset();

    mw.put(new Text(HConstants.KEY_FOR_HOSTNAME_SEEN_BY_MASTER),

      new Text(rs.getHostname()));

    return mw;

  }

  /**

   * @return Subset of configuration to pass initializing regionservers: e.g.

   * the filesystem to use and root directory to use.

   */

  protected MapWritable createConfigurationSubset() {

    MapWritable mw = addConfig(new MapWritable(), HConstants.HBASE_DIR);

    return addConfig(mw, "fs.default.name");

  }

  private MapWritable addConfig(final MapWritable mw, final String key) {

    mw.put(new Text(key), new Text(this.conf.get(key)));

    return mw;

  }

  @Override

  public void regionServerReport(final byte [] sn, final HServerLoad hsl)

  throws IOException {

    this.serverManager.regionServerReport(ServerName.parseVersionedServerName(sn), hsl);

    if (hsl != null && this.metrics != null) {

      // Up our metrics.

      this.metrics.incrementRequests(hsl.getTotalNumberOfRequests());

    }

  }

  @Override

  public void reportRSFatalError(byte [] sn, String errorText) {

    String msg = "Region server " + Bytes.toString(sn) +

      " reported a fatal error:\n" + errorText;

    LOG.error(msg);

    rsFatals.add(msg);

  }

  public boolean isMasterRunning() {

    return !isStopped();

  }

  /**

   * @return Maximum time we should run balancer for

   */

  private int getBalancerCutoffTime() {

    int balancerCutoffTime =

      getConfiguration().getInt("hbase.balancer.max.balancing", -1);

    if (balancerCutoffTime == -1) {

      // No time period set so create one -- do half of balancer period.

      int balancerPeriod =

        getConfiguration().getInt("hbase.balancer.period", 300000);

      balancerCutoffTime = balancerPeriod / 2;

      // If nonsense period, set it to balancerPeriod

      if (balancerCutoffTime <= 0) balancerCutoffTime = balancerPeriod;

    }

    return balancerCutoffTime;

  }

  @Override

  public boolean balance() {

    // If balance not true, don't run balancer.

    if (!this.balanceSwitch) return false;

    // Do this call outside of synchronized block.

    int maximumBalanceTime = getBalancerCutoffTime();

    long cutoffTime = System.currentTimeMillis() + maximumBalanceTime;

    boolean balancerRan;

    synchronized (this.balancer) {

      // Only allow one balance run at at time.

      if (this.assignmentManager.isRegionsInTransition()) {

        LOG.debug("Not running balancer because " +

          this.assignmentManager.getRegionsInTransition().size() +

          " region(s) in transition: " +

          org.apache.commons.lang.StringUtils.

            abbreviate(this.assignmentManager.getRegionsInTransition().toString(), 256));

        return false;

      }

      if (this.serverManager.areDeadServersInProgress()) {

        LOG.debug("Not running balancer because processing dead regionserver(s): " +

          this.serverManager.getDeadServers());

        return false;

      }

      if (this.cpHost != null) {

        try {

          if (this.cpHost.preBalance()) {

            LOG.debug("Coprocessor bypassing balancer request");

            return false;

          }

        } catch (IOException ioe) {

          LOG.error("Error invoking master coprocessor preBalance()", ioe);

          return false;

        }

      }

      Map<String, Map<ServerName, List<HRegionInfo>>> assignmentsByTable =

        this.assignmentManager.getAssignmentsByTable();

      List<RegionPlan> plans = new ArrayList<RegionPlan>();

      for (Map<ServerName, List<HRegionInfo>> assignments : assignmentsByTable.values()) {

        List<RegionPlan> partialPlans = this.balancer.balanceCluster(assignments);

        if (partialPlans != null) plans.addAll(partialPlans);

      }

      int rpCount = 0;  // number of RegionPlans balanced so far

      long totalRegPlanExecTime = 0;

      balancerRan = plans != null;

      if (plans != null && !plans.isEmpty()) {

        for (RegionPlan plan: plans) {

          LOG.info("balance " + plan);

          long balStartTime = System.currentTimeMillis();

          this.assignmentManager.balance(plan);

          totalRegPlanExecTime += System.currentTimeMillis()-balStartTime;

          rpCount++;

          if (rpCount < plans.size() &&

              // if performing next balance exceeds cutoff time, exit the loop

              (System.currentTimeMillis() + (totalRegPlanExecTime / rpCount)) > cutoffTime) {

            LOG.debug("No more balancing till next balance run; maximumBalanceTime=" +

              maximumBalanceTime);

            break;

          }

        }

      }

      if (this.cpHost != null) {

        try {

          this.cpHost.postBalance();

        } catch (IOException ioe) {

          // balancing already succeeded so don't change the result

          LOG.error("Error invoking master coprocessor postBalance()", ioe);

        }

      }

    }

    return balancerRan;

  }

  enum BalanceSwitchMode {

    SYNC,

    ASYNC

  }

  /**

   * Assigns balancer switch according to BalanceSwitchMode

   * @param b new balancer switch

   * @param mode BalanceSwitchMode

   * @return old balancer switch

   */

  public boolean switchBalancer(final boolean b, BalanceSwitchMode mode) {

    boolean oldValue = this.balanceSwitch;

    boolean newValue = b;

    try {

      if (this.cpHost != null) {

        newValue = this.cpHost.preBalanceSwitch(newValue);

      }

      if (mode == BalanceSwitchMode.SYNC) {

        synchronized (this.balancer) {       

          this.balanceSwitch = newValue;

        }

      } else {

        this.balanceSwitch = newValue;       

      }

      LOG.info("BalanceSwitch=" + newValue);

      if (this.cpHost != null) {

        this.cpHost.postBalanceSwitch(oldValue, newValue);

      }

    } catch (IOException ioe) {

      LOG.warn("Error flipping balance switch", ioe);

    }

    return oldValue;   

  }

 

  @Override

  public boolean synchronousBalanceSwitch(final boolean b) {

    return switchBalancer(b, BalanceSwitchMode.SYNC);

  }

 

  @Override

  public boolean balanceSwitch(final boolean b) {

    return switchBalancer(b, BalanceSwitchMode.ASYNC);

  }

  /**

   * Switch for the background CatalogJanitor thread.

   * Used for testing.  The thread will continue to run.  It will just be a noop

   * if disabled.

   * @param b If false, the catalog janitor won't do anything.

   */

  public void setCatalogJanitorEnabled(final boolean b) {

    ((CatalogJanitor)this.catalogJanitorChore).setEnabled(b);

  }

  @Override

  public void move(final byte[] encodedRegionName, final byte[] destServerName)

  throws UnknownRegionException {

    Pair<HRegionInfo, ServerName> p =

      this.assignmentManager.getAssignment(encodedRegionName);

    if (p == null)

      throw new UnknownRegionException(Bytes.toStringBinary(encodedRegionName));

    ServerName dest = null;

    if (destServerName == null || destServerName.length == 0) {

      LOG.info("Passed destination servername is null or empty so choosing a server at random");

      List<ServerName> destServers = this.serverManager.getOnlineServersList();

      destServers.remove(p.getSecond());

      // If i have only one RS then destination can be null.

      dest = balancer.randomAssignment(destServers);

    } else {

      dest = new ServerName(Bytes.toString(destServerName));

    }

   

    // Now we can do the move

    RegionPlan rp = new RegionPlan(p.getFirst(), p.getSecond(), dest);

   

    try {

      if (this.cpHost != null) {

        if (this.cpHost.preMove(p.getFirst(), p.getSecond(), dest)) {

          return;

        }

      }

      LOG.info("Added move plan " + rp + ", running balancer");

      this.assignmentManager.balance(rp);

      if (this.cpHost != null) {

        this.cpHost.postMove(p.getFirst(), p.getSecond(), dest);

      }

    } catch (IOException ioe) {

      UnknownRegionException ure = new UnknownRegionException(

          Bytes.toStringBinary(encodedRegionName));

      ure.initCause(ioe);

      throw ure;

    }

  }

  public void createTable(HTableDescriptor hTableDescriptor,

    byte [][] splitKeys)

  throws IOException {

    if (!isMasterRunning()) {

      throw new MasterNotRunningException();

    }

    HRegionInfo [] newRegions = getHRegionInfos(hTableDescriptor, splitKeys);