*

* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* A non-instantiable class that manages {@link HConnection}s.

* This class has a static Map of {@link HConnection} instances keyed by

* {@link Configuration}; all invocations of {@link #getConnection(Configuration)}

* that pass the same {@link Configuration} instance will be returned the same

* {@link  HConnection} instance (Adding properties to a Configuration

* instance does not change its object identity).  Sharing {@link HConnection}

* instances is usually what you want; all clients of the {@link HConnection}

* instances share the HConnections' cache of Region locations rather than each

* having to discover for itself the location of meta, etc.  It makes

* sense for the likes of the pool of HTables class {@link HTablePool}, for

* instance (If concerned that a single {@link HConnection} is insufficient

* for sharing amongst clients in say an heavily-multithreaded environment,

* in practise its not proven to be an issue.  Besides, {@link HConnection} is

* implemented atop Hadoop RPC and as of this writing, Hadoop RPC does a

* connection per cluster-member, exclusively).

*

* <p>But sharing connections

* makes clean up of {@link HConnection} instances a little awkward.  Currently,

* clients cleanup by calling

* {@link #deleteConnection(Configuration)}.  This will shutdown the

* zookeeper connection the HConnection was using and clean up all

* HConnection resources as well as stopping proxies to servers out on the

* cluster. Not running the cleanup will not end the world; it'll

* just stall the closeup some and spew some zookeeper connection failed

* messages into the log.  Running the cleanup on a {@link HConnection} that is

* subsequently used by another will cause breakage so be careful running

* cleanup.

* <p>To create a {@link HConnection} that is not shared by others, you can

* create a new {@link Configuration} instance, pass this new instance to

* {@link #getConnection(Configuration)}, and then when done, close it up by

* doing something like the following:

* <pre>

* {@code

* Configuration newConfig = new Configuration(originalConf);

* HConnection connection = HConnectionManager.getConnection(newConfig);

* // Use the connection to your hearts' delight and then when done...

* HConnectionManager.deleteConnection(newConfig, true);

* }

* </pre>

* <p>Cleanup used to be done inside in a shutdown hook.  On startup we'd

* register a shutdown hook that called {@link #deleteAllConnections()}

* on its way out but the order in which shutdown hooks run is not defined so

* were problematic for clients of HConnection that wanted to register their

* own shutdown hooks so we removed ours though this shifts the onus for

* cleanup to the client.

*/

@InterfaceStability.Evolving

  // An LRU Map of HConnectionKey -> HConnection (TableServer).  All

  // access must be synchronized.  This map is not private because tests

  // need to be able to tinker with it.

  static final Map<HConnectionKey, HConnectionImplementation> HBASE_INSTANCES;

  public static final int MAX_CACHED_HBASE_INSTANCES;

  /** Parameter name for what client protocol to use. */

  public static final String CLIENT_PROTOCOL_CLASS = "hbase.clientprotocol.class";

  /** Default client protocol class name. */

  public static final String DEFAULT_CLIENT_PROTOCOL_CLASS = ClientProtocol.class.getName();

  /** Parameter name for what admin protocol to use. */

  public static final String REGION_PROTOCOL_CLASS = "hbase.adminprotocol.class";

  /** Default admin protocol class name. */

  public static final String DEFAULT_ADMIN_PROTOCOL_CLASS = AdminProtocol.class.getName();

  private static final Log LOG = LogFactory.getLog(HConnectionManager.class);

  static {

    // We set instances to one more than the value specified for {@link

    // HConstants#ZOOKEEPER_MAX_CLIENT_CNXNS}. By default, the zk default max

    // connections to the ensemble from the one client is 30, so in that case we

    // should run into zk issues before the LRU hit this value of 31.

    MAX_CACHED_HBASE_INSTANCES = HBaseConfiguration.create().getInt(

        HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS,

        HConstants.DEFAULT_ZOOKEPER_MAX_CLIENT_CNXNS) + 1;

    HBASE_INSTANCES = new LinkedHashMap<HConnectionKey, HConnectionImplementation>(

        (int) (MAX_CACHED_HBASE_INSTANCES / 0.75F) + 1, 0.75F, true) {

       @Override

      protected boolean removeEldestEntry(

          Map.Entry<HConnectionKey, HConnectionImplementation> eldest) {

         return size() > MAX_CACHED_HBASE_INSTANCES;

       }

    };

  }

  /*

   * Non-instantiable.

   */

  protected HConnectionManager() {

    super();

  }

  /**

   * Get the connection that goes with the passed <code>conf</code>

   * configuration instance.

   * If no current connection exists, method creates a new connection for the

   * passed <code>conf</code> instance.

   * @param conf configuration

   * @return HConnection object for <code>conf</code>

   * @throws ZooKeeperConnectionException

   */

  public static HConnection getConnection(Configuration conf)

  throws IOException {

    HConnectionKey connectionKey = new HConnectionKey(conf);

    synchronized (HBASE_INSTANCES) {

      HConnectionImplementation connection = HBASE_INSTANCES.get(connectionKey);

      if (connection == null) {

        connection = new HConnectionImplementation(conf, true);

        HBASE_INSTANCES.put(connectionKey, connection);

      } else if (connection.isClosed()) {

        HConnectionManager.deleteConnection(connectionKey, true);

        connection = new HConnectionImplementation(conf, true);

        HBASE_INSTANCES.put(connectionKey, connection);

      }

      connection.incCount();

      return connection;

    }

  }

  /**

   * Create a new HConnection instance using the passed <code>conf</code>

   * instance.

   * Note: This bypasses the usual HConnection life cycle management!

   * Use this with caution, the caller is responsible for closing the

   * created connection.

   * @param conf configuration

   * @return HConnection object for <code>conf</code>

   * @throws ZooKeeperConnectionException

   */

  public static HConnection createConnection(Configuration conf)

  throws IOException {

    return new HConnectionImplementation(conf, false);

  }

  /**

   * Delete connection information for the instance specified by configuration.

   * If there are no more references to it, this will then close connection to

   * the zookeeper ensemble and let go of all resources.

   *

   * @param conf

   *          configuration whose identity is used to find {@link HConnection}

   *          instance.

   */

  public static void deleteConnection(Configuration conf) {

    deleteConnection(new HConnectionKey(conf), false);

  }

  /**

   * Delete stale connection information for the instance specified by configuration.

   * This will then close connection to

   * the zookeeper ensemble and let go of all resources.

   *

   * @param connection

   */

  public static void deleteStaleConnection(HConnection connection) {

    deleteConnection(connection, true);

  }

  /**

   * Delete information for all connections.

   */

  public static void deleteAllConnections() {

    synchronized (HBASE_INSTANCES) {

      Set<HConnectionKey> connectionKeys = new HashSet<HConnectionKey>();

      connectionKeys.addAll(HBASE_INSTANCES.keySet());

      for (HConnectionKey connectionKey : connectionKeys) {

        deleteConnection(connectionKey, false);

      }

      HBASE_INSTANCES.clear();

    }

  }

  private static void deleteConnection(HConnection connection, boolean staleConnection) {

    synchronized (HBASE_INSTANCES) {

      for (Entry<HConnectionKey, HConnectionImplementation> connectionEntry : HBASE_INSTANCES

          .entrySet()) {

        if (connectionEntry.getValue() == connection) {

          deleteConnection(connectionEntry.getKey(), staleConnection);

          break;

        }

      }

    }

  }

  private static void deleteConnection(HConnectionKey connectionKey, boolean staleConnection) {

    synchronized (HBASE_INSTANCES) {

      HConnectionImplementation connection = HBASE_INSTANCES

          .get(connectionKey);

      if (connection != null) {

        connection.decCount();

        if (connection.isZeroReference() || staleConnection) {

          HBASE_INSTANCES.remove(connectionKey);

          connection.internalClose();

        }

      } else {

        LOG.error("Connection not found in the list, can't delete it "+

          "(connection key="+connectionKey+"). May be the key was modified?");

      }

    }

  }

  /**

   * It is provided for unit test cases which verify the behavior of region

   * location cache prefetch.

   * @return Number of cached regions for the table.

   * @throws ZooKeeperConnectionException

   */

  static int getCachedRegionCount(Configuration conf,

      final byte[] tableName)

  throws IOException {

    return execute(new HConnectable<Integer>(conf) {

      @Override

      public Integer connect(HConnection connection) {

        return ((HConnectionImplementation) connection)

            .getNumberOfCachedRegionLocations(tableName);

      }

    });

  }

  /**

   * It's provided for unit test cases which verify the behavior of region

   * location cache prefetch.

   * @return true if the region where the table and row reside is cached.

   * @throws ZooKeeperConnectionException

   */

  static boolean isRegionCached(Configuration conf,

      final byte[] tableName, final byte[] row) throws IOException {

    return execute(new HConnectable<Boolean>(conf) {

      @Override

      public Boolean connect(HConnection connection) {

        return ((HConnectionImplementation) connection).isRegionCached(tableName, row);

      }

    });

  }

  /**

   * This class makes it convenient for one to execute a command in the context

   * of a {@link HConnection} instance based on the given {@link Configuration}.

   *

   * <p>

   * If you find yourself wanting to use a {@link HConnection} for a relatively

   * short duration of time, and do not want to deal with the hassle of creating

   * and cleaning up that resource, then you should consider using this

   * convenience class.

   *

   * @param <T>

   *          the return type of the {@link HConnectable#connect(HConnection)}

   *          method.

   */

  public static abstract class HConnectable<T> {

    public Configuration conf;

    protected HConnectable(Configuration conf) {

      this.conf = conf;

    }

    public abstract T connect(HConnection connection) throws IOException;

  }

  /**

   * This convenience method invokes the given {@link HConnectable#connect}

   * implementation using a {@link HConnection} instance that lasts just for the

   * duration of that invocation.

   *

   * @param <T> the return type of the connect method

   * @param connectable the {@link HConnectable} instance

   * @return the value returned by the connect method

   * @throws IOException

   */

  public static <T> T execute(HConnectable<T> connectable) throws IOException {

    if (connectable == null || connectable.conf == null) {

      return null;

    }

    Configuration conf = connectable.conf;

    HConnection connection = HConnectionManager.getConnection(conf);

    boolean connectSucceeded = false;

    try {

      T returnValue = connectable.connect(connection);

      connectSucceeded = true;

      return returnValue;

    } finally {

      try {

        connection.close();

      } catch (Exception e) {

        if (connectSucceeded) {

          throw new IOException("The connection to " + connection

              + " could not be deleted.", e);

        }

      }

    }

  }

  /**

   * Denotes a unique key to a {@link HConnection} instance.

   *

   * In essence, this class captures the properties in {@link Configuration}

   * that may be used in the process of establishing a connection. In light of

   * that, if any new such properties are introduced into the mix, they must be

   * added to the {@link HConnectionKey#properties} list.

   *

   */

  public static class HConnectionKey {

    final static String[] CONNECTION_PROPERTIES = new String[] {

        HConstants.ZOOKEEPER_QUORUM, HConstants.ZOOKEEPER_ZNODE_PARENT,

        HConstants.ZOOKEEPER_CLIENT_PORT,

        HConstants.ZOOKEEPER_RECOVERABLE_WAITTIME,

        HConstants.HBASE_CLIENT_PAUSE, HConstants.HBASE_CLIENT_RETRIES_NUMBER,

        HConstants.HBASE_CLIENT_RPC_MAXATTEMPTS,

        HConstants.HBASE_RPC_TIMEOUT_KEY,

        HConstants.HBASE_CLIENT_PREFETCH_LIMIT,

        HConstants.HBASE_META_SCANNER_CACHING,

        HConstants.HBASE_CLIENT_INSTANCE_ID };

    private Map<String, String> properties;

    private String username;

    public HConnectionKey(Configuration conf) {

      Map<String, String> m = new HashMap<String, String>();

      if (conf != null) {

        for (String property : CONNECTION_PROPERTIES) {

          String value = conf.get(property);

          if (value != null) {

            m.put(property, value);

          }

        }

      }

      this.properties = Collections.unmodifiableMap(m);

      try {

        User currentUser = User.getCurrent();

        if (currentUser != null) {

          username = currentUser.getName();

        }

      } catch (IOException ioe) {

        LOG.warn("Error obtaining current user, skipping username in HConnectionKey",

            ioe);

      }

    }

    @Override

    public int hashCode() {

      final int prime = 31;

      int result = 1;

      if (username != null) {

        result = username.hashCode();

      }

      for (String property : CONNECTION_PROPERTIES) {

        String value = properties.get(property);

        if (value != null) {

          result = prime * result + value.hashCode();

        }

      }

      return result;

    }

    @edu.umd.cs.findbugs.annotations.SuppressWarnings (value="ES_COMPARING_STRINGS_WITH_EQ",

        justification="Optimization")

    @Override

    public boolean equals(Object obj) {

      if (this == obj)

        return true;

      if (obj == null)

        return false;

      if (getClass() != obj.getClass())

        return false;

      HConnectionKey that = (HConnectionKey) obj;

      if (this.username != null && !this.username.equals(that.username)) {

        return false;

      } else if (this.username == null && that.username != null) {

        return false;

      }

      if (this.properties == null) {

        if (that.properties != null) {

          return false;

        }

      } else {

        if (that.properties == null) {

          return false;

        }

        for (String property : CONNECTION_PROPERTIES) {

          String thisValue = this.properties.get(property);

          String thatValue = that.properties.get(property);

          //noinspection StringEquality

          if (thisValue == thatValue) {

            continue;

          }

          if (thisValue == null || !thisValue.equals(thatValue)) {

            return false;

          }

        }

      }

      return true;

    }

    @Override

    public String toString() {

      return "HConnectionKey{" +

        "properties=" + properties +

        ", username='" + username + '\'' +

        '}';

    }

  }

  static class HConnectionImplementation implements HConnection, Closeable {

    static final Log LOG = LogFactory.getLog(HConnectionImplementation.class);

    private final Class<? extends AdminProtocol> adminClass;

    private final Class<? extends ClientProtocol> clientClass;

    private final long pause;

    private final int numRetries;

    private final int maxRPCAttempts;

    private final int rpcTimeout;

    private final int prefetchRegionLimit;

    private volatile boolean closed;

    private volatile boolean aborted;

    // package protected for the tests

    ClusterStatusListener clusterStatusListener;

    private final Object userRegionLock = new Object();

    // We have a single lock for master & zk to prevent deadlocks. Having

    //  one lock for ZK and one lock for master is not possible:

    //  When creating a connection to master, we need a connection to ZK to get

    //  its address. But another thread could have taken the ZK lock, and could

    //  be waiting for the master lock => deadlock.

    private final Object masterAndZKLock = new Object();

    private long keepZooKeeperWatcherAliveUntil = Long.MAX_VALUE;

    private final DelayedClosing delayedClosing =

      DelayedClosing.createAndStart(this);

    private final Configuration conf;

    // client RPC

    private RpcClientEngine rpcEngine;

    // Known region ServerName.toString() -> RegionClient/Admin

    private final ConcurrentHashMap<String, Map<String, IpcProtocol>> servers =

      new ConcurrentHashMap<String, Map<String, IpcProtocol>>();

    private final ConcurrentHashMap<String, String> connectionLock =

      new ConcurrentHashMap<String, String>();

    /**

     * Map of table to table {@link HRegionLocation}s.  The table key is made

     * by doing a {@link Bytes#mapKey(byte[])} of the table's name.

     */

    private final Map<Integer, SoftValueSortedMap<byte [], HRegionLocation>>

      cachedRegionLocations =

        new HashMap<Integer, SoftValueSortedMap<byte [], HRegionLocation>>();

    // The presence of a server in the map implies it's likely that there is an

    // entry in cachedRegionLocations that map to this server; but the absence

    // of a server in this map guarentees that there is no entry in cache that

    // maps to the absent server.

    // The access to this attribute must be protected by a lock on cachedRegionLocations

    private final Set<ServerName> cachedServers = new HashSet<ServerName>();

    // region cache prefetch is enabled by default. this set contains all

    // tables whose region cache prefetch are disabled.

    private final Set<Integer> regionCachePrefetchDisabledTables =

      new CopyOnWriteArraySet<Integer>();

    private int refCount;

    // indicates whether this connection's life cycle is managed (by us)

    private final boolean managed;

    /**

     * constructor

     * @param conf Configuration object

     */

    @SuppressWarnings("unchecked")

    public HConnectionImplementation(Configuration conf, boolean managed) throws IOException {

      this.conf = conf;

      this.managed = managed;

      String adminClassName = conf.get(REGION_PROTOCOL_CLASS,

        DEFAULT_ADMIN_PROTOCOL_CLASS);

      this.closed = false;

      try {

        this.adminClass =

          (Class<? extends AdminProtocol>) Class.forName(adminClassName);

      } catch (ClassNotFoundException e) {

        throw new UnsupportedOperationException(

            "Unable to find region server interface " + adminClassName, e);

      }

      String clientClassName = conf.get(CLIENT_PROTOCOL_CLASS,

        DEFAULT_CLIENT_PROTOCOL_CLASS);

      try {

        this.clientClass =

          (Class<? extends ClientProtocol>) Class.forName(clientClassName);

      } catch (ClassNotFoundException e) {

        throw new UnsupportedOperationException(

            "Unable to find client protocol " + clientClassName, e);

      }

      this.pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE,

          HConstants.DEFAULT_HBASE_CLIENT_PAUSE);

      this.numRetries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,

          HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);

      this.maxRPCAttempts = conf.getInt(

          HConstants.HBASE_CLIENT_RPC_MAXATTEMPTS,

          HConstants.DEFAULT_HBASE_CLIENT_RPC_MAXATTEMPTS);

      this.rpcTimeout = conf.getInt(

          HConstants.HBASE_RPC_TIMEOUT_KEY,

          HConstants.DEFAULT_HBASE_RPC_TIMEOUT);

      this.prefetchRegionLimit = conf.getInt(

          HConstants.HBASE_CLIENT_PREFETCH_LIMIT,

          HConstants.DEFAULT_HBASE_CLIENT_PREFETCH_LIMIT);

      retrieveClusterId();

      // ProtobufRpcClientEngine is the main RpcClientEngine implementation,

      // but we maintain access through an interface to allow overriding for tests

      // RPC engine setup must follow obtaining the cluster ID for token authentication to work

      this.rpcEngine = new ProtobufRpcClientEngine(this.conf, this.clusterId);

      // Do we publish the status?

      Class<? extends ClusterStatusListener.Listener> listenerClass =

          conf.getClass(ClusterStatusListener.STATUS_LISTENER_CLASS,

              ClusterStatusListener.DEFAULT_STATUS_LISTENER_CLASS,

              ClusterStatusListener.Listener.class);

      if (listenerClass != null) {

        clusterStatusListener = new ClusterStatusListener(

            new ClusterStatusListener.DeadServerHandler() {

              @Override

              public void newDead(ServerName sn) {

                clearCaches(sn);

                rpcEngine.getClient().cancelConnections(sn.getHostname(), sn.getPort(), null);

              }

            }, conf, listenerClass);

      }

    }

    /**

     * An identifier that will remain the same for a given connection.

     * @return

     */

    public String toString(){

      return "hconnection-0x" + Integer.toHexString(hashCode());

    }

    private String clusterId = null;

    public final void retrieveClusterId(){

      if (clusterId != null) {

        return;

      }

      // No synchronized here, worse case we will retrieve it twice, that's

      //  not an issue.

      ZooKeeperKeepAliveConnection zkw = null;

      try {

        zkw = getKeepAliveZooKeeperWatcher();

        clusterId = ZKClusterId.readClusterIdZNode(zkw);

        if (clusterId == null) {

          LOG.info("ClusterId read in ZooKeeper is null");

        }

      } catch (KeeperException e) {

        LOG.warn("Can't retrieve clusterId from Zookeeper", e);

      } catch (IOException e) {

        LOG.warn("Can't retrieve clusterId from Zookeeper", e);

      } finally {

        if (zkw != null) {

          zkw.close();

        }

      }

      if (clusterId == null) {

        clusterId = HConstants.CLUSTER_ID_DEFAULT;

      }

      LOG.info("ClusterId is " + clusterId);

    }

    @Override

    public Configuration getConfiguration() {

      return this.conf;

    }

    private static class MasterProtocolState {

      public MasterProtocol protocol;

      public int userCount;

      public long keepAliveUntil = Long.MAX_VALUE;

      public final Class<? extends MasterProtocol> protocolClass;

      public MasterProtocolState (

          final Class<? extends MasterProtocol> protocolClass) {

        this.protocolClass = protocolClass;

      }

    }

    /**

     * Create a new Master proxy. Try once only.

     */

    private MasterProtocol createMasterInterface(

        MasterProtocolState masterProtocolState)

        throws IOException, KeeperException, ServiceException {

      ZooKeeperKeepAliveConnection zkw;

      try {

        zkw = getKeepAliveZooKeeperWatcher();

      } catch (IOException e) {

        throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e);

      }

      try {

        checkIfBaseNodeAvailable(zkw);

        ServerName sn = MasterAddressTracker.getMasterAddress(zkw);

        if (sn == null) {

          String msg =

            "ZooKeeper available but no active master location found";

          LOG.info(msg);

          throw new MasterNotRunningException(msg);

        }

        InetSocketAddress isa =

          new InetSocketAddress(sn.getHostname(), sn.getPort());

        MasterProtocol tryMaster = rpcEngine.getProxy(

            masterProtocolState.protocolClass,

            isa, this.conf, this.rpcTimeout);

        if (tryMaster.isMasterRunning(

            null, RequestConverter.buildIsMasterRunningRequest()).getIsMasterRunning()) {

          return tryMaster;

        } else {

          String msg = "Can create a proxy to master, but it is not running";

          LOG.info(msg);

          throw new MasterNotRunningException(msg);

        }

      } finally {

        zkw.close();

      }

    }

    /**

     * Create a master, retries if necessary.

     */

    @edu.umd.cs.findbugs.annotations.SuppressWarnings (value="SWL_SLEEP_WITH_LOCK_HELD")

    private MasterProtocol createMasterWithRetries(

      MasterProtocolState masterProtocolState) throws MasterNotRunningException {

      // The lock must be at the beginning to prevent multiple master creation

      //  (and leaks) in a multithread context

      synchronized (this.masterAndZKLock) {

        Exception exceptionCaught = null;

        MasterProtocol master = null;

        int tries = 0;

        while (

          !this.closed && master == null

          ) {

          tries++;

          try {

            master = createMasterInterface(masterProtocolState);

          } catch (IOException e) {

            exceptionCaught = e;

          } catch (KeeperException e) {

            exceptionCaught = e;

          } catch (ServiceException e) {

            exceptionCaught = e;

          }

          if (exceptionCaught != null)

            // It failed. If it's not the last try, we're going to wait a little

          if (tries < numRetries) {

            // tries at this point is 1 or more; decrement to start from 0.

            long pauseTime = ConnectionUtils.getPauseTime(this.pause, tries - 1);

            LOG.info("getMaster attempt " + tries + " of " + numRetries +

              " failed; retrying after sleep of " +pauseTime + ", exception=" + exceptionCaught);

            try {

              Thread.sleep(pauseTime);

            } catch (InterruptedException e) {

              Thread.currentThread().interrupt();

              throw new RuntimeException(

                "Thread was interrupted while trying to connect to master.", e);

            }

          } else {

            // Enough tries, we stop now

            LOG.info("getMaster attempt " + tries + " of " + numRetries +

              " failed; no more retrying.", exceptionCaught);

            throw new MasterNotRunningException(exceptionCaught);

          }

        }

        if (master == null) {

          // implies this.closed true

          throw new MasterNotRunningException(

            "Connection was closed while trying to get master");

        }

        return master;

      }

    }

    private void checkIfBaseNodeAvailable(ZooKeeperWatcher zkw)

      throws MasterNotRunningException {

      String errorMsg;

      try {

        if (ZKUtil.checkExists(zkw, zkw.baseZNode) == -1) {

          errorMsg = "The node " + zkw.baseZNode+" is not in ZooKeeper. "

            + "It should have been written by the master. "

            + "Check the value configured in 'zookeeper.znode.parent'. "

            + "There could be a mismatch with the one configured in the master.";

          LOG.error(errorMsg);

          throw new MasterNotRunningException(errorMsg);

        }

      } catch (KeeperException e) {

        errorMsg = "Can't get connection to ZooKeeper: " + e.getMessage();

        LOG.error(errorMsg);

        throw new MasterNotRunningException(errorMsg, e);

      }

    }

    /**

     * @return true if the master is running, throws an exception otherwise

     * @throws MasterNotRunningException - if the master is not running

     * @throws ZooKeeperConnectionException

     */

    @Override

    public boolean isMasterRunning()

      throws MasterNotRunningException, ZooKeeperConnectionException {

      // When getting the master proxy connection, we check it's running,

      // so if there is no exception, it means we've been able to get a

      // connection on a running master

      getKeepAliveMasterMonitor().close();

      return true;

    }

    @Override

    public HRegionLocation getRegionLocation(final byte [] name,

        final byte [] row, boolean reload)

    throws IOException {

      return reload? relocateRegion(name, row): locateRegion(name, row);

    }

    @Override

    public boolean isTableEnabled(byte[] tableName) throws IOException {

      return testTableOnlineState(tableName, true);

    }

    @Override

    public boolean isTableDisabled(byte[] tableName) throws IOException {

      return testTableOnlineState(tableName, false);

    }

    @Override

    public boolean isTableAvailable(final byte[] tableName) throws IOException {

      final AtomicBoolean available = new AtomicBoolean(true);