* Copyright 2010 The Apache Software Foundation

*

* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* HRegionServer makes a set of HRegions available to clients.  It checks in with

* the HMaster. There are many HRegionServers in a single HBase deployment.

*/

    HBaseRPCErrorHandler, Runnable, Watcher, Stoppable {

  public static final Log LOG = LogFactory.getLog(HRegionServer.class);

  private static final HMsg REPORT_EXITING = new HMsg(Type.MSG_REPORT_EXITING);

  private static final HMsg REPORT_QUIESCED = new HMsg(Type.MSG_REPORT_QUIESCED);

  private static final HMsg [] EMPTY_HMSG_ARRAY = new HMsg [] {};

  // Set when a report to the master comes back with a message asking us to

  // shutdown.  Also set by call to stop when debugging or running unit tests

  // of HRegionServer in isolation. We use AtomicBoolean rather than

  // plain boolean so we can pass a reference to Chore threads.  Otherwise,

  // Chore threads need to know about the hosting class.

  protected final AtomicBoolean stopRequested = new AtomicBoolean(false);

  protected final AtomicBoolean quiesced = new AtomicBoolean(false);

  // Go down hard.  Used if file system becomes unavailable and also in

  // debugging and unit tests.

  protected volatile boolean abortRequested;

  private volatile boolean killed = false;

  // If false, the file system has become unavailable

  protected volatile boolean fsOk;

  protected HServerInfo serverInfo;

  protected final Configuration conf;

  private final ServerConnection connection;

  protected final AtomicBoolean haveRootRegion = new AtomicBoolean(false);

  private FileSystem fs;

  private Path rootDir;

  private final Random rand = new Random();

  // Key is Bytes.hashCode of region name byte array and the value is HRegion

  // in both of the maps below.  Use Bytes.mapKey(byte []) generating key for

  // below maps.

  protected final Map<Integer, HRegion> onlineRegions =

    new ConcurrentHashMap<Integer, HRegion>();

  protected final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();

  private final LinkedBlockingQueue<HMsg> outboundMsgs =

    new LinkedBlockingQueue<HMsg>();

  final int numRetries;

  protected final int threadWakeFrequency;

  private final int msgInterval;

  protected final int numRegionsToReport;

  private final long maxScannerResultSize;

  // Remote HMaster

  private HMasterRegionInterface hbaseMaster;

  // Server to handle client requests.  Default access so can be accessed by

  // unit tests.

  HBaseServer server;

  // Leases

  private Leases leases;

  // Request counter

  private volatile AtomicInteger requestCount = new AtomicInteger();

  // Info server.  Default access so can be used by unit tests.  REGIONSERVER

  // is name of the webapp and the attribute name used stuffing this instance

  // into web context.

  InfoServer infoServer;

  /** region server process name */

  public static final String REGIONSERVER = "regionserver";

  /*

   * Space is reserved in HRS constructor and then released when aborting

   * to recover from an OOME. See HBASE-706.  TODO: Make this percentage of the

   * heap or a minimum.

   */

  private final LinkedList<byte[]> reservedSpace = new LinkedList<byte []>();

  private RegionServerMetrics metrics;

  // Compactions

  CompactSplitThread compactSplitThread;

  // Cache flushing

  MemStoreFlusher cacheFlusher;

  /* Check for major compactions.

   */

  Chore majorCompactionChecker;

  // HLog and HLog roller.  log is protected rather than private to avoid

  // eclipse warning when accessed by inner classes

  protected volatile HLog hlog;

  LogRoller hlogRoller;

  // flag set after we're done setting up server threads (used for testing)

  protected volatile boolean isOnline;

  final Map<String, InternalScanner> scanners =

    new ConcurrentHashMap<String, InternalScanner>();

  private ZooKeeperWrapper zooKeeperWrapper;

  // A sleeper that sleeps for msgInterval.

  private final Sleeper sleeper;

  private final long rpcTimeout;

  // Address passed in to constructor.  This is not always the address we run

  // with.  For example, if passed port is 0, then we are to pick a port.  The

  // actual address we run with is in the #serverInfo data member.

  private final HServerAddress address;

  // The main region server thread.

  private Thread regionServerThread;

  private final String machineName;

  /**

   * Starts a HRegionServer at the default location

   * @param conf

   * @throws IOException

   */

  public HRegionServer(Configuration conf) throws IOException {

    machineName = DNS.getDefaultHost(

        conf.get("hbase.regionserver.dns.interface","default"),

        conf.get("hbase.regionserver.dns.nameserver","default"));

    String addressStr = machineName + ":" +

      conf.get(HConstants.REGIONSERVER_PORT,

          Integer.toString(HConstants.DEFAULT_REGIONSERVER_PORT));

    // This is not necessarily the address we will run with.  The address we

    // use will be in #serverInfo data member.  For example, we may have been

    // passed a port of 0 which means we should pick some ephemeral port to bind

    // to.

    address = new HServerAddress(addressStr);

    LOG.info("My address is " + address);

    this.abortRequested = false;

    this.fsOk = true;

    this.conf = conf;

    this.connection = ServerConnectionManager.getConnection(conf);

    this.isOnline = false;

    // Config'ed params

    this.numRetries =  conf.getInt("hbase.client.retries.number", 2);

    this.threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY,

        10 * 1000);

    this.msgInterval = conf.getInt("hbase.regionserver.msginterval", 1 * 1000);

    sleeper = new Sleeper(this.msgInterval, this.stopRequested);

    this.maxScannerResultSize = conf.getLong(

            HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,

            HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);

    // Task thread to process requests from Master

    this.worker = new Worker();

    this.numRegionsToReport =

      conf.getInt("hbase.regionserver.numregionstoreport", 10);

    this.rpcTimeout =

      conf.getLong(HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY,

          HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD);

    reinitialize();

  }

  /**

   * Creates all of the state that needs to be reconstructed in case we are

   * doing a restart. This is shared between the constructor and restart().

   * Both call it.

   * @throws IOException

   */

  private void reinitialize() throws IOException {

    this.abortRequested = false;

    this.stopRequested.set(false);

    // Server to handle client requests

    this.server = HBaseRPC.getServer(this, address.getBindAddress(),

      address.getPort(), conf.getInt("hbase.regionserver.handler.count", 10),

      false, conf);

    this.server.setErrorHandler(this);

    // Address is giving a default IP for the moment. Will be changed after

    // calling the master.

    this.serverInfo = new HServerInfo(new HServerAddress(

      new InetSocketAddress(address.getBindAddress(),

      this.server.getListenerAddress().getPort())), System.currentTimeMillis(),

      this.conf.getInt("hbase.regionserver.info.port", 60030), machineName);

    if (this.serverInfo.getServerAddress() == null) {

      throw new NullPointerException("Server address cannot be null; " +

        "hbase-958 debugging");

    }

    reinitializeThreads();

    reinitializeZooKeeper();

    int nbBlocks = conf.getInt("hbase.regionserver.nbreservationblocks", 4);

    for(int i = 0; i < nbBlocks; i++)  {

      reservedSpace.add(new byte[HConstants.DEFAULT_SIZE_RESERVATION_BLOCK]);

    }

  }

  private void reinitializeZooKeeper() throws IOException {

    zooKeeperWrapper = new ZooKeeperWrapper(conf, this);

    watchMasterAddress();

  }

  private void reinitializeThreads() {

    this.workerThread = new Thread(worker);

    // Cache flushing thread.

    this.cacheFlusher = new MemStoreFlusher(conf, this);

    // Compaction thread

    this.compactSplitThread = new CompactSplitThread(this);

    // Log rolling thread

    this.hlogRoller = new LogRoller(this);

    // Background thread to check for major compactions; needed if region

    // has not gotten updates in a while.  Make it run at a lesser frequency.

    int multiplier = this.conf.getInt(HConstants.THREAD_WAKE_FREQUENCY +

        ".multiplier", 1000);

    this.majorCompactionChecker = new MajorCompactionChecker(this,

      this.threadWakeFrequency * multiplier,  this.stopRequested);

    this.leases = new Leases(

        (int) conf.getLong(HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY,

            HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD),

        this.threadWakeFrequency);

  }

  /**

   * We register ourselves as a watcher on the master address ZNode. This is

   * called by ZooKeeper when we get an event on that ZNode. When this method

   * is called it means either our master has died, or a new one has come up.

   * Either way we need to update our knowledge of the master.

   * @param event WatchedEvent from ZooKeeper.

   */

  public void process(WatchedEvent event) {

    EventType type = event.getType();

    KeeperState state = event.getState();

    LOG.info("Got ZooKeeper event, state: " + state + ", type: " +

      type + ", path: " + event.getPath());

    // Ignore events if we're shutting down.

    if (this.stopRequested.get()) {

      LOG.debug("Ignoring ZooKeeper event while shutting down");

      return;

    }

    if (state == KeeperState.Expired) {

      LOG.error("ZooKeeper session expired");

      boolean restart =

        this.conf.getBoolean("hbase.regionserver.restart.on.zk.expire", false);

      if (restart) {

        restart();

      } else {

        abort("ZooKeeper session expired");

      }

    } else if (type == EventType.NodeDeleted) {

      watchMasterAddress();

    } else if (type == EventType.NodeCreated) {

      getMaster();

      // ZooKeeper watches are one time only, so we need to re-register our watch.

      watchMasterAddress();

    }

  }

  private void watchMasterAddress() {

    while (!stopRequested.get() && !zooKeeperWrapper.watchMasterAddress(this)) {

      LOG.warn("Unable to set watcher on ZooKeeper master address. Retrying.");

      sleeper.sleep();

    }

  }

  private void restart() {

    abort("Restarting region server");

    Threads.shutdown(regionServerThread);

    boolean done = false;

    while (!done) {

      try {

        reinitialize();

        done = true;

      } catch (IOException e) {

        LOG.debug("Error trying to reinitialize ZooKeeper", e);

      }

    }

    Thread t = new Thread(this);

    String name = regionServerThread.getName();

    t.setName(name);

    t.start();

  }

  /** @return ZooKeeperWrapper used by RegionServer. */

  public ZooKeeperWrapper getZooKeeperWrapper() {

    return zooKeeperWrapper;

  }

  /**

   * The HRegionServer sticks in this loop until closed. It repeatedly checks

   * in with the HMaster, sending heartbeats & reports, and receiving HRegion

   * load/unload instructions.

   */

  public void run() {

    regionServerThread = Thread.currentThread();

    boolean quiesceRequested = false;

    try {

      MapWritable w = null;

      while (!stopRequested.get()) {

        w = reportForDuty();

        if (w != null) {

          init(w);

          break;

        }

        sleeper.sleep();

        LOG.warn("No response from master on reportForDuty. Sleeping and " +

          "then trying again.");

      }

      List<HMsg> outboundMessages = new ArrayList<HMsg>();

      long lastMsg = 0;

      // Now ask master what it wants us to do and tell it what we have done

      for (int tries = 0; !stopRequested.get() && isHealthy();) {

        // Try to get the root region location from the master.

        if (!haveRootRegion.get()) {

          HServerAddress rootServer = zooKeeperWrapper.readRootRegionLocation();

          if (rootServer != null) {

            // By setting the root region location, we bypass the wait imposed on

            // HTable for all regions being assigned.

            this.connection.setRootRegionLocation(

                new HRegionLocation(HRegionInfo.ROOT_REGIONINFO, rootServer));

            haveRootRegion.set(true);

          }

        }

        long now = System.currentTimeMillis();

        // Drop into the send loop if msgInterval has elapsed or if something

        // to send.  If we fail talking to the master, then we'll sleep below

        // on poll of the outboundMsgs blockingqueue.

        if ((now - lastMsg) >= msgInterval || !outboundMessages.isEmpty()) {

          try {

            doMetrics();

            MemoryUsage memory =

              ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();

            HServerLoad hsl = new HServerLoad(requestCount.get(),

              (int)(memory.getUsed()/1024/1024),

              (int)(memory.getMax()/1024/1024));

            for (HRegion r: onlineRegions.values()) {

              hsl.addRegionInfo(createRegionLoad(r));

            }

            this.serverInfo.setLoad(hsl);

            this.requestCount.set(0);

            addOutboundMsgs(outboundMessages);

            HMsg msgs[] = this.hbaseMaster.regionServerReport(

              serverInfo, outboundMessages.toArray(EMPTY_HMSG_ARRAY),

              getMostLoadedRegions());

            lastMsg = System.currentTimeMillis();

            updateOutboundMsgs(outboundMessages);

            outboundMessages.clear();

            if (this.quiesced.get() && onlineRegions.size() == 0) {

              // We've just told the master we're exiting because we aren't

              // serving any regions. So set the stop bit and exit.

              LOG.info("Server quiesced and not serving any regions. " +

                "Starting shutdown");

              stopRequested.set(true);

              this.outboundMsgs.clear();

              continue;

            }

            // Queue up the HMaster's instruction stream for processing

            boolean restart = false;

            for(int i = 0;

                !restart && !stopRequested.get() && i < msgs.length;

                i++) {

              LOG.info(msgs[i].toString());

              this.connection.unsetRootRegionLocation();

              switch(msgs[i].getType()) {

              case MSG_REGIONSERVER_STOP:

                stopRequested.set(true);

                break;

              case MSG_REGIONSERVER_QUIESCE:

                if (!quiesceRequested) {

                  try {

                    toDo.put(new ToDoEntry(msgs[i]));

                  } catch (InterruptedException e) {

                    throw new RuntimeException("Putting into msgQueue was " +

                        "interrupted.", e);

                  }

                  quiesceRequested = true;

                }

                break;

              default:

                if (fsOk) {

                  try {

                    toDo.put(new ToDoEntry(msgs[i]));

                  } catch (InterruptedException e) {

                    throw new RuntimeException("Putting into msgQueue was " +

                        "interrupted.", e);

                  }

                }

              }

            }

            // Reset tries count if we had a successful transaction.

            tries = 0;

            if (restart || this.stopRequested.get()) {

              toDo.clear();

              continue;

            }

          } catch (Exception e) { // FindBugs REC_CATCH_EXCEPTION

            // Two special exceptions could be printed out here,

            // PleaseHoldException and YouAreDeadException

            if (e instanceof IOException) {

              e = RemoteExceptionHandler.checkIOException((IOException) e);

            }

            if (e instanceof YouAreDeadException) {

              // This will be caught and handled as a fatal error below

              throw e;

            }

            tries++;

            if (tries > 0 && (tries % this.numRetries) == 0) {

              // Check filesystem every so often.

              checkFileSystem();

            }

            if (this.stopRequested.get()) {

              LOG.info("Stop requested, clearing toDo despite exception");

              toDo.clear();

              continue;

            }

            LOG.warn("Attempt=" + tries, e);

            // No point retrying immediately; this is probably connection to

            // master issue.  Doing below will cause us to sleep.

            lastMsg = System.currentTimeMillis();

          }

        }

        now = System.currentTimeMillis();

        HMsg msg = this.outboundMsgs.poll((msgInterval - (now - lastMsg)),

          TimeUnit.MILLISECONDS);

        // If we got something, add it to list of things to send.

        if (msg != null) outboundMessages.add(msg);

        // Do some housekeeping before going back around

        housekeeping();

      } // for

    } catch (Throwable t) {

      if (!checkOOME(t)) {

        abort("Unhandled exception", t);

      }

    }

    this.leases.closeAfterLeasesExpire();

    this.worker.stop();

    this.server.stop();

    if (this.infoServer != null) {

      LOG.info("Stopping infoServer");

      try {

        this.infoServer.stop();

      } catch (Exception e) {

        e.printStackTrace();

      }

    }

    // Send cache a shutdown.

    LruBlockCache c = (LruBlockCache)StoreFile.getBlockCache(this.conf);

    if (c != null) c.shutdown();

    // Send interrupts to wake up threads if sleeping so they notice shutdown.

    // TODO: Should we check they are alive?  If OOME could have exited already

    cacheFlusher.interruptIfNecessary();

    compactSplitThread.interruptIfNecessary();

    hlogRoller.interruptIfNecessary();

    this.majorCompactionChecker.interrupt();

    if (killed) {

      // Just skip out w/o closing regions.

    } else if (abortRequested) {

      if (this.fsOk) {

        // Only try to clean up if the file system is available

        try {

          if (this.hlog != null) {

            this.hlog.close();

            LOG.info("On abort, closed hlog");

          }

        } catch (Throwable e) {

          LOG.error("Unable to close log in abort",

            RemoteExceptionHandler.checkThrowable(e));

        }

        closeAllRegions(); // Don't leave any open file handles

      }

      LOG.info("aborting server at: " + this.serverInfo.getServerName());

    } else {

      ArrayList<HRegion> closedRegions = closeAllRegions();

      try {

        if (this.hlog != null) {

          hlog.closeAndDelete();

        }

      } catch (Throwable e) {

        LOG.error("Close and delete failed",

          RemoteExceptionHandler.checkThrowable(e));

      }

      try {

        HMsg[] exitMsg = new HMsg[closedRegions.size() + 1];

        exitMsg[0] = REPORT_EXITING;

        // Tell the master what regions we are/were serving

        int i = 1;

        for (HRegion region: closedRegions) {

          exitMsg[i++] = new HMsg(HMsg.Type.MSG_REPORT_CLOSE,

              region.getRegionInfo());

        }

        LOG.info("telling master that region server is shutting down at: " +

            serverInfo.getServerName());

        hbaseMaster.regionServerReport(serverInfo, exitMsg, (HRegionInfo[])null);

      } catch (Throwable e) {

        LOG.warn("Failed to send exiting message to master: ",

          RemoteExceptionHandler.checkThrowable(e));

      }

      LOG.info("stopping server at: " + this.serverInfo.getServerName());

    }

    // Make sure the proxy is down.

    if (this.hbaseMaster != null) {

      HBaseRPC.stopProxy(this.hbaseMaster);

      this.hbaseMaster = null;

    }

    if (!killed) {

      this.zooKeeperWrapper.close();

      join();

    }

    LOG.info(Thread.currentThread().getName() + " exiting");

  }

  /*

   * Add to the passed <code>msgs</code> messages to pass to the master.

   * @param msgs Current outboundMsgs array; we'll add messages to this List.

   */

  private void addOutboundMsgs(final List<HMsg> msgs) {

    if (msgs.isEmpty()) {

      this.outboundMsgs.drainTo(msgs);

      return;

    }

    OUTER: for (HMsg m: this.outboundMsgs) {

      for (HMsg mm: msgs) {

        // Be careful don't add duplicates.

        if (mm.equals(m)) {

          continue OUTER;

        }

      }

      msgs.add(m);

    }

  }

  /*

   * Remove from this.outboundMsgs those messsages we sent the master.

   * @param msgs Messages we sent the master.

   */

  private void updateOutboundMsgs(final List<HMsg> msgs) {

    if (msgs.isEmpty()) return;

    for (HMsg m: this.outboundMsgs) {

      for (HMsg mm: msgs) {

        if (mm.equals(m)) {

          this.outboundMsgs.remove(m);

          break;

        }

      }

    }

  }

  /*

   * Run init. Sets up hlog and starts up all server threads.

   * @param c Extra configuration.

   */

  protected void init(final MapWritable c) throws IOException {

    try {

      for (Map.Entry<Writable, Writable> e: c.entrySet()) {

        String key = e.getKey().toString();

        String value = e.getValue().toString();

        if (LOG.isDebugEnabled()) {

          LOG.debug("Config from master: " + key + "=" + value);

        }

        this.conf.set(key, value);

      }

      // Master may have sent us a new address with the other configs.

      // Update our address in this case. See HBASE-719

      String hra = conf.get("hbase.regionserver.address");

      // TODO: The below used to be this.address != null.  Was broken by what

      // looks like a mistake in:

      //

      // HBASE-1215 migration; metautils scan of meta region was broken; wouldn't see first row

      // ------------------------------------------------------------------------

      // r796326 | stack | 2009-07-21 07:40:34 -0700 (Tue, 21 Jul 2009) | 38 lines

      if (hra != null) {

        HServerAddress hsa = new HServerAddress (hra,

          this.serverInfo.getServerAddress().getPort());

        LOG.info("Master passed us address to use. Was=" +

          this.serverInfo.getServerAddress() + ", Now=" + hra);

        this.serverInfo.setServerAddress(hsa);

      }

      // Master sent us hbase.rootdir to use. Should be fully qualified

      // path with file system specification included.  Set 'fs.defaultFS'

      // to match the filesystem on hbase.rootdir else underlying hadoop hdfs

      // accessors will be going against wrong filesystem (unless all is set

      // to defaults).

      this.conf.set("fs.defaultFS", this.conf.get("hbase.rootdir"));

      // Get fs instance used by this RS

      this.fs = FileSystem.get(this.conf);

      this.rootDir = new Path(this.conf.get(HConstants.HBASE_DIR));

      this.hlog = setupHLog();

      // Init in here rather than in constructor after thread name has been set

      this.metrics = new RegionServerMetrics();

      startServiceThreads();

      isOnline = true;

    } catch (Throwable e) {

      this.isOnline = false;

      this.stopRequested.set(true);

      throw convertThrowableToIOE(cleanup(e, "Failed init"),

        "Region server startup failed");

    }

  }

  /*

   * @param r Region to get RegionLoad for.

   * @return RegionLoad instance.

   * @throws IOException

   */

  private HServerLoad.RegionLoad createRegionLoad(final HRegion r) {

    byte[] name = r.getRegionName();

    int stores = 0;

    int storefiles = 0;

    int storefileSizeMB = 0;

    int memstoreSizeMB = (int)(r.memstoreSize.get()/1024/1024);

    int storefileIndexSizeMB = 0;

    synchronized (r.stores) {

      stores += r.stores.size();

      for (Store store: r.stores.values()) {

        storefiles += store.getStorefilesCount();

        storefileSizeMB +=

          (int)(store.getStorefilesSize()/1024/1024);

        storefileIndexSizeMB +=

          (int)(store.getStorefilesIndexSize()/1024/1024);

      }

    }

    return new HServerLoad.RegionLoad(name, stores, storefiles,

      storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB);

  }

  /**

   * @param regionName

   * @return An instance of RegionLoad.

   * @throws IOException

   */

  public HServerLoad.RegionLoad createRegionLoad(final byte [] regionName) {

    return createRegionLoad(this.onlineRegions.get(Bytes.mapKey(regionName)));

  }

  /*

   * Cleanup after Throwable caught invoking method.  Converts <code>t</code>

   * to IOE if it isn't already.

   * @param t Throwable

   * @return Throwable converted to an IOE; methods can only let out IOEs.

   */

  private Throwable cleanup(final Throwable t) {

    return cleanup(t, null);

  }

  /*

   * Cleanup after Throwable caught invoking method.  Converts <code>t</code>

   * to IOE if it isn't already.

   * @param t Throwable

   * @param msg Message to log in error.  Can be null.

   * @return Throwable converted to an IOE; methods can only let out IOEs.

   */

  private Throwable cleanup(final Throwable t, final String msg) {

    if (msg == null) {

      LOG.error("", RemoteExceptionHandler.checkThrowable(t));

    } else {

      LOG.error(msg, RemoteExceptionHandler.checkThrowable(t));

    }

    if (!checkOOME(t)) {