* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* DataNode is a class (and program) that stores a set of

* blocks for a DFS deployment.  A single deployment can

* have one or many DataNodes.  Each DataNode communicates

* regularly with a single NameNode.  It also communicates

* with client code and other DataNodes from time to time.

*

* DataNodes store a series of named blocks.  The DataNode

* allows client code to read these blocks, or to write new

* block data.  The DataNode may also, in response to instructions

* from its NameNode, delete blocks or copy blocks to/from other

* DataNodes.

*

* The DataNode maintains just one critical table:

*   block-> stream of bytes (of BLOCK_SIZE or less)

*

* This info is stored on a local disk.  The DataNode

* reports the table's contents to the NameNode upon startup

* and every so often afterwards.

*

* DataNodes spend their lives in an endless loop of asking

* the NameNode for something to do.  A NameNode cannot connect

* to a DataNode directly; a NameNode simply returns values from

* functions invoked by a DataNode.

*

* DataNodes maintain an open server socket so that client code

* or other DataNodes can read/write data.  The host/port for

* this server is reported to the NameNode, which then sends that

* information to clients or other DataNodes that might be interested.

*

**********************************************************/

    implements InterDatanodeProtocol, ClientDatanodeProtocol, FSConstants, Runnable {

  public static final Log LOG = LogFactory.getLog(DataNode.class);

 

  static{

    Configuration.addDefaultResource("hdfs-default.xml");

    Configuration.addDefaultResource("hdfs-site.xml");

  }

  public static final String DN_CLIENTTRACE_FORMAT =

        "src: %s" +      // src IP

        ", dest: %s" +   // dst IP

        ", bytes: %s" +  // byte count

        ", op: %s" +     // operation

        ", cliID: %s" +  // DFSClient id

        ", offset: %s" + // offset

        ", srvID: %s" +  // DatanodeRegistration

        ", blockid: %s" + // block id

        ", duration: %s"; // duration time

  static final Log ClientTraceLog =

    LogFactory.getLog(DataNode.class.getName() + ".clienttrace");

  /**

   * Use {@link NetUtils#createSocketAddr(String)} instead.

   */

  @Deprecated

  public static InetSocketAddress createSocketAddr(String target

                                                   ) throws IOException {

    return NetUtils.createSocketAddr(target);

  }

 

  public DatanodeProtocol namenode = null;

  public FSDatasetInterface data = null;

  public DatanodeRegistration dnRegistration = null;

  volatile boolean shouldRun = true;

  private LinkedList<Block> receivedBlockList = new LinkedList<Block>();

  /** list of blocks being recovered */

  private final Map<Block, Block> ongoingRecovery = new HashMap<Block, Block>();

  private LinkedList<String> delHints = new LinkedList<String>();

  public final static String EMPTY_DEL_HINT = "";

  AtomicInteger xmitsInProgress = new AtomicInteger();

  Daemon dataXceiverServer = null;

  ThreadGroup threadGroup = null;

  long blockReportInterval;

  //disallow the sending of BR before instructed to do so

  long lastBlockReport = 0;

  boolean resetBlockReportTime = true;

  long initialBlockReportDelay = BLOCKREPORT_INITIAL_DELAY * 1000L;

  long lastHeartbeat = 0;

  long heartBeatInterval;

  private DataStorage storage = null;

  private HttpServer infoServer = null;

  DataNodeMetrics myMetrics;

  private static InetSocketAddress nameNodeAddr;

  private InetSocketAddress selfAddr;

  private static DataNode datanodeObject = null;

  private Thread dataNodeThread = null;

  String machineName;

  private static String dnThreadName;

  int socketTimeout;

  int socketWriteTimeout = 0; 

  boolean transferToAllowed = true;

  int writePacketSize = 0;

  /**

   * Testing hook that allows tests to delay the sending of blockReceived

   * RPCs to the namenode. This can help find bugs in append.

   */

  int artificialBlockReceivedDelay = 0;

  boolean isBlockTokenEnabled;

  BlockTokenSecretManager blockTokenSecretManager;

  boolean isBlockTokenInitialized = false;

 

  public DataBlockScanner blockScanner = null;

  public Daemon blockScannerThread = null;

 

  /** Activated plug-ins. */

  private PluginDispatcher<DatanodePlugin> pluginDispatcher;

 

  private static final Random R = new Random();

 

  public static final String DATA_DIR_KEY = "dfs.data.dir";

  public final static String DATA_DIR_PERMISSION_KEY =

    "dfs.datanode.data.dir.perm";

  private static final String DEFAULT_DATA_DIR_PERMISSION = "755";

 

  // For InterDataNodeProtocol

  public Server ipcServer;

  private SecureResources secureResources = null;

 

  /**

   * Current system time.

   * @return current time in msec.

   */

  static long now() {

    return System.currentTimeMillis();

  }

 

  /**

   * Create the DataNode given a configuration and an array of dataDirs.

   * 'dataDirs' is where the blocks are stored.

   */

  DataNode(final Configuration conf,

           final AbstractList<File> dataDirs) throws IOException {

    this(conf, dataDirs, null);

  }

 

  /**

   * Start a Datanode with specified server sockets for secure environments

   * where they are run with privileged ports and injected from a higher

   * level of capability

   */

  DataNode(final Configuration conf,

           final AbstractList<File> dataDirs, SecureResources resources) throws IOException {

    super(conf);

    UserGroupInformation.setConfiguration(conf);

    SecurityUtil.login(conf, DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY,

        DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY);

    datanodeObject = this;

    try {

      startDataNode(conf, dataDirs, resources);

    } catch (IOException ie) {

      shutdown();

      throw ie;

    }  

  }

   

 

  /**

   * This method starts the data node with the specified conf.

   *

   * @param conf - the configuration

   *  if conf's CONFIG_PROPERTY_SIMULATED property is set

   *  then a simulated storage based data node is created.

   *

   * @param dataDirs - only for a non-simulated storage data node

   * @throws IOException

   */

  void startDataNode(Configuration conf,

                     AbstractList<File> dataDirs, SecureResources resources

                     ) throws IOException {

    if(UserGroupInformation.isSecurityEnabled() && resources == null &&

       conf.getBoolean("dfs.datanode.require.secure.ports", true))

      throw new RuntimeException("Cannot start secure cluster without " +

          "privileged resources.");

   

    this.secureResources = resources;

    // use configured nameserver & interface to get local hostname

    if (conf.get("slave.host.name") != null) {

      machineName = conf.get("slave.host.name");  

    }

    if (machineName == null) {

      machineName = DNS.getDefaultHost(

                                     conf.get("dfs.datanode.dns.interface","default"),

                                     conf.get("dfs.datanode.dns.nameserver","default"));

    }

    InetSocketAddress nameNodeAddr = NameNode.getAddress(conf);

   

    this.socketTimeout =  conf.getInt("dfs.socket.timeout",

                                      HdfsConstants.READ_TIMEOUT);

    this.socketWriteTimeout = conf.getInt("dfs.datanode.socket.write.timeout",

                                          HdfsConstants.WRITE_TIMEOUT);

    /* Based on results on different platforms, we might need set the default

     * to false on some of them. */

    this.transferToAllowed = conf.getBoolean("dfs.datanode.transferTo.allowed",

                                             true);

    this.writePacketSize = conf.getInt("dfs.write.packet.size", 64*1024);

    InetSocketAddress socAddr = DataNode.getStreamingAddr(conf);

    int tmpPort = socAddr.getPort();

    storage = new DataStorage();

    // construct registration

    this.dnRegistration = new DatanodeRegistration(machineName + ":" + tmpPort);

    // connect to name node

    this.namenode = (DatanodeProtocol)

      RPC.waitForProxy(DatanodeProtocol.class,

                       DatanodeProtocol.versionID,

                       nameNodeAddr,

                       conf);

    // get version and id info from the name-node

    NamespaceInfo nsInfo = handshake();

    StartupOption startOpt = getStartupOption(conf);

    assert startOpt != null : "Startup option must be set.";

   

    boolean simulatedFSDataset =

        conf.getBoolean("dfs.datanode.simulateddatastorage", false);

    if (simulatedFSDataset) {

        setNewStorageID(dnRegistration);

        dnRegistration.storageInfo.layoutVersion = FSConstants.LAYOUT_VERSION;

        dnRegistration.storageInfo.namespaceID = nsInfo.namespaceID;

        // it would have been better to pass storage as a parameter to

        // constructor below - need to augment ReflectionUtils used below.

        conf.set("StorageId", dnRegistration.getStorageID());

        try {

          //Equivalent of following (can't do because Simulated is in test dir)

          //  this.data = new SimulatedFSDataset(conf);

          this.data = (FSDatasetInterface) ReflectionUtils.newInstance(

              Class.forName("org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset"), conf);

        } catch (ClassNotFoundException e) {

          throw new IOException(StringUtils.stringifyException(e));

        }

    } else { // real storage

      // read storage info, lock data dirs and transition fs state if necessary

      storage.recoverTransitionRead(nsInfo, dataDirs, startOpt);

      // adjust

      this.dnRegistration.setStorageInfo(storage);

      // initialize data node internal structure

      this.data = new FSDataset(storage, conf);

    }

    // Allow configuration to delay block reports to find bugs

    artificialBlockReceivedDelay = conf.getInt(

      "dfs.datanode.artificialBlockReceivedDelay", 0);

    // find free port or use privileged port provide

    ServerSocket ss;

    if(secureResources == null) {

      ss = (socketWriteTimeout > 0) ?

        ServerSocketChannel.open().socket() : new ServerSocket();

      Server.bind(ss, socAddr, 0);

    } else {

      ss = resources.getStreamingSocket();

    }

    ss.setReceiveBufferSize(DEFAULT_DATA_SOCKET_SIZE);

    // adjust machine name with the actual port

    tmpPort = ss.getLocalPort();

    selfAddr = new InetSocketAddress(ss.getInetAddress().getHostAddress(),

                                     tmpPort);

    this.dnRegistration.setName(machineName + ":" + tmpPort);

    LOG.info("Opened info server at " + tmpPort);

     

    this.threadGroup = new ThreadGroup("dataXceiverServer");

    this.dataXceiverServer = new Daemon(threadGroup,

        new DataXceiverServer(ss, conf, this));

    this.threadGroup.setDaemon(true); // auto destroy when empty

    this.blockReportInterval =

      conf.getLong("dfs.blockreport.intervalMsec", BLOCKREPORT_INTERVAL);

    this.initialBlockReportDelay = conf.getLong("dfs.blockreport.initialDelay",

                                            BLOCKREPORT_INITIAL_DELAY)* 1000L;

    if (this.initialBlockReportDelay >= blockReportInterval) {

      this.initialBlockReportDelay = 0;

      LOG.info("dfs.blockreport.initialDelay is greater than " +

        "dfs.blockreport.intervalMsec." + " Setting initial delay to 0 msec:");

    }

    this.heartBeatInterval = conf.getLong("dfs.heartbeat.interval", HEARTBEAT_INTERVAL) * 1000L;

    DataNode.nameNodeAddr = nameNodeAddr;

    //initialize periodic block scanner

    String reason = null;

    if (conf.getInt("dfs.datanode.scan.period.hours", 0) < 0) {

      reason = "verification is turned off by configuration";

    } else if ( !(data instanceof FSDataset) ) {

      reason = "verifcation is supported only with FSDataset";

    }

    if ( reason == null ) {

      blockScanner = new DataBlockScanner(this, (FSDataset)data, conf);

    } else {

      LOG.info("Periodic Block Verification is disabled because " +

               reason + ".");

    }

    //create a servlet to serve full-file content

    InetSocketAddress infoSocAddr = DataNode.getInfoAddr(conf);

    String infoHost = infoSocAddr.getHostName();

    int tmpInfoPort = infoSocAddr.getPort();

    this.infoServer = (secureResources == null)

       ? new HttpServer("datanode", infoHost, tmpInfoPort, tmpInfoPort == 0,

           conf, SecurityUtil.getAdminAcls(conf, DFSConfigKeys.DFS_ADMIN))

       : new HttpServer("datanode", infoHost, tmpInfoPort, tmpInfoPort == 0,

           conf, SecurityUtil.getAdminAcls(conf, DFSConfigKeys.DFS_ADMIN),

           secureResources.getListener());

    if (conf.getBoolean("dfs.https.enable", false)) {

      boolean needClientAuth = conf.getBoolean("dfs.https.need.client.auth", false);

      InetSocketAddress secInfoSocAddr = NetUtils.createSocketAddr(conf.get(

          "dfs.datanode.https.address", infoHost + ":" + 0));

      Configuration sslConf = new Configuration(false);

      sslConf.addResource(conf.get("dfs.https.server.keystore.resource",

          "ssl-server.xml"));

      this.infoServer.addSslListener(secInfoSocAddr, sslConf, needClientAuth);

    }

    this.infoServer.addInternalServlet(null, "/streamFile/*", StreamFile.class);

    this.infoServer.addInternalServlet(null, "/getFileChecksum/*",

        FileChecksumServlets.GetServlet.class);

    this.infoServer.setAttribute("datanode.blockScanner", blockScanner);

    this.infoServer.setAttribute(JspHelper.CURRENT_CONF, conf);

    this.infoServer.addServlet(null, "/blockScannerReport",

                               DataBlockScanner.Servlet.class);

    this.infoServer.start();

    // adjust info port

    this.dnRegistration.setInfoPort(this.infoServer.getPort());

    myMetrics = new DataNodeMetrics(conf, dnRegistration.getName());

    // BlockTokenSecretManager is created here, but it shouldn't be

    // used until it is initialized in register().

    this.blockTokenSecretManager = new BlockTokenSecretManager(false,

        0, 0);

    //init ipc server

    InetSocketAddress ipcAddr = NetUtils.createSocketAddr(

        conf.get("dfs.datanode.ipc.address"));

    ipcServer = RPC.getServer(this, ipcAddr.getHostName(), ipcAddr.getPort(),

        conf.getInt("dfs.datanode.handler.count", 3), false, conf,

        blockTokenSecretManager);

    // set service-level authorization security policy

    if (conf.getBoolean(

        CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, false)) {

      ipcServer.refreshServiceAcl(conf, new HDFSPolicyProvider());

    }

    dnRegistration.setIpcPort(ipcServer.getListenerAddress().getPort());

    LOG.info("dnRegistration = " + dnRegistration);

   

    pluginDispatcher = PluginDispatcher.createFromConfiguration(

        conf, DFSConfigKeys.DFS_DATANODE_PLUGINS_KEY, DatanodePlugin.class);

    pluginDispatcher.dispatchStart(this);

  }

  /**

   * Determine the http server's effective addr

   */

  public static InetSocketAddress getInfoAddr(Configuration conf) {

    String infoAddr = NetUtils.getServerAddress(conf,

        "dfs.datanode.info.bindAddress",

        "dfs.datanode.info.port",

        "dfs.datanode.http.address");

   

    return NetUtils.createSocketAddr(infoAddr);

  }

  /**

   * Creates either NIO or regular depending on socketWriteTimeout.

   */

  protected Socket newSocket() throws IOException {

    return (socketWriteTimeout > 0) ?

           SocketChannel.open().socket() : new Socket();                                  

  }

 

  private NamespaceInfo handshake() throws IOException {

    NamespaceInfo nsInfo = new NamespaceInfo();

    while (shouldRun) {

      try {

        nsInfo = namenode.versionRequest();

        break;

      } catch(SocketTimeoutException e) {  // namenode is busy

        LOG.info("Problem connecting to server: " + getNameNodeAddr());

        try {

          Thread.sleep(1000);

        } catch (InterruptedException ie) {}

      }

    }

    String errorMsg = null;

    // verify build version

    if( ! nsInfo.getBuildVersion().equals( Storage.getBuildVersion() )) {

      errorMsg = "Incompatible build versions: namenode BV = "

        + nsInfo.getBuildVersion() + "; datanode BV = "

        + Storage.getBuildVersion();

      LOG.fatal( errorMsg );

      try {

        namenode.errorReport( dnRegistration,

                              DatanodeProtocol.NOTIFY, errorMsg );

      } catch( SocketTimeoutException e ) {  // namenode is busy

        LOG.info("Problem connecting to server: " + getNameNodeAddr());

      }

      throw new IOException( errorMsg );

    }

    assert FSConstants.LAYOUT_VERSION == nsInfo.getLayoutVersion() :

      "Data-node and name-node layout versions must be the same."

      + "Expected: "+ FSConstants.LAYOUT_VERSION + " actual "+ nsInfo.getLayoutVersion();

    return nsInfo;

  }

  /** Return the DataNode object

   *

   */

  public static DataNode getDataNode() {

    return datanodeObject;

  }

  public static InterDatanodeProtocol createInterDataNodeProtocolProxy(

      DatanodeID datanodeid, final Configuration conf) throws IOException {

    final InetSocketAddress addr = NetUtils.createSocketAddr(

        datanodeid.getHost() + ":" + datanodeid.getIpcPort());

    if (InterDatanodeProtocol.LOG.isDebugEnabled()) {

      InterDatanodeProtocol.LOG.info("InterDatanodeProtocol addr=" + addr);

    }

    UserGroupInformation loginUgi = UserGroupInformation.getLoginUser();

    try {

      return loginUgi

          .doAs(new PrivilegedExceptionAction<InterDatanodeProtocol>() {

            public InterDatanodeProtocol run() throws IOException {

              return (InterDatanodeProtocol) RPC.getProxy(

                  InterDatanodeProtocol.class, InterDatanodeProtocol.versionID,

                  addr, conf);

            }

          });

    } catch (InterruptedException ie) {

      throw new IOException(ie.getMessage());

    }

  }

  public InetSocketAddress getNameNodeAddr() {

    return nameNodeAddr;

  }

 

  public InetSocketAddress getSelfAddr() {

    return selfAddr;

  }

   

  DataNodeMetrics getMetrics() {

    return myMetrics;

  }

 

  /**

   * Return the namenode's identifier

   */

  public String getNamenode() {

    //return namenode.toString();

    return "<namenode>";

  }

  public static void setNewStorageID(DatanodeRegistration dnReg) {

    /* Return

     * "DS-randInt-ipaddr-currentTimeMillis"

     * It is considered extermely rare for all these numbers to match

     * on a different machine accidentally for the following

     * a) SecureRandom(INT_MAX) is pretty much random (1 in 2 billion), and

     * b) Good chance ip address would be different, and

     * c) Even on the same machine, Datanode is designed to use different ports.

     * d) Good chance that these are started at different times.

     * For a confict to occur all the 4 above have to match!.

     * The format of this string can be changed anytime in future without

     * affecting its functionality.

     */

    String ip = "unknownIP";

    try {

      ip = DNS.getDefaultIP("default");

    } catch (UnknownHostException ignored) {

      LOG.warn("Could not find ip address of \"default\" inteface.");

    }

   

    int rand = 0;

    try {

      rand = SecureRandom.getInstance("SHA1PRNG").nextInt(Integer.MAX_VALUE);

    } catch (NoSuchAlgorithmException e) {

      LOG.warn("Could not use SecureRandom");

      rand = R.nextInt(Integer.MAX_VALUE);

    }

    dnReg.storageID = "DS-" + rand + "-"+ ip + "-" + dnReg.getPort() + "-" +

                      System.currentTimeMillis();

  }

  /**

   * Register datanode

   * <p>

   * The datanode needs to register with the namenode on startup in order

   * 1) to report which storage it is serving now and

   * 2) to receive a registrationID

   * issued by the namenode to recognize registered datanodes.

   *

   * @see FSNamesystem#registerDatanode(DatanodeRegistration)

   * @throws IOException

   */

  private void register() throws IOException {

    if (dnRegistration.getStorageID().equals("")) {

      setNewStorageID(dnRegistration);

    }

    while(shouldRun) {

      try {

        // reset name to machineName. Mainly for web interface.

        dnRegistration.name = machineName + ":" + dnRegistration.getPort();

        dnRegistration = namenode.register(dnRegistration);

        break;

      } catch(SocketTimeoutException e) {  // namenode is busy

        LOG.info("Problem connecting to server: " + getNameNodeAddr());

        try {

          Thread.sleep(1000);

        } catch (InterruptedException ie) {}

      }

    }

    assert ("".equals(storage.getStorageID())

            && !"".equals(dnRegistration.getStorageID()))

            || storage.getStorageID().equals(dnRegistration.getStorageID()) :

            "New storageID can be assigned only if data-node is not formatted";

    if (storage.getStorageID().equals("")) {

      storage.setStorageID(dnRegistration.getStorageID());

      storage.writeAll();

      LOG.info("New storage id " + dnRegistration.getStorageID()

          + " is assigned to data-node " + dnRegistration.getName());

    }

    if(! storage.getStorageID().equals(dnRegistration.getStorageID())) {

      throw new IOException("Inconsistent storage IDs. Name-node returned "

          + dnRegistration.getStorageID()

          + ". Expecting " + storage.getStorageID());

    }

   

    if (!isBlockTokenInitialized) {

      /* first time registering with NN */

      ExportedBlockKeys keys = dnRegistration.exportedKeys;

      this.isBlockTokenEnabled = keys.isBlockTokenEnabled();

      if (isBlockTokenEnabled) {

        long blockKeyUpdateInterval = keys.getKeyUpdateInterval();

        long blockTokenLifetime = keys.getTokenLifetime();

        LOG.info("Block token params received from NN: keyUpdateInterval="

            + blockKeyUpdateInterval / (60 * 1000) + " min(s), tokenLifetime="

            + blockTokenLifetime / (60 * 1000) + " min(s)");

        blockTokenSecretManager.setTokenLifetime(blockTokenLifetime);

      }

      isBlockTokenInitialized = true;

    }

    if (isBlockTokenEnabled) {

      blockTokenSecretManager.setKeys(dnRegistration.exportedKeys);

      dnRegistration.exportedKeys = ExportedBlockKeys.DUMMY_KEYS;

    }

    // random short delay - helps scatter the BR from all DNs

    scheduleBlockReport(initialBlockReportDelay);

  }

  /**

   * Shut down this instance of the datanode.

   * Returns only after shutdown is complete.

   * This method can only be called by the offerService thread.

   * Otherwise, deadlock might occur.

   */

  public void shutdown() {

    if (pluginDispatcher != null) {

      pluginDispatcher.dispatchStop();

    }

    if (infoServer != null) {

      try {

        infoServer.stop();

      } catch (Exception e) {

        LOG.warn("Exception shutting down DataNode", e);

      }

    }

    if (ipcServer != null) {

      ipcServer.stop();

    }

    this.shouldRun = false;

    if (dataXceiverServer != null) {

      ((DataXceiverServer) this.dataXceiverServer.getRunnable()).kill();

      this.dataXceiverServer.interrupt();

      // wait for all data receiver threads to exit

      if (this.threadGroup != null) {

        while (true) {

          this.threadGroup.interrupt();

          LOG.info("Waiting for threadgroup to exit, active threads is " +

                   this.threadGroup.activeCount());

          if (this.threadGroup.activeCount() == 0) {

            break;

          }

          try {

            Thread.sleep(1000);

          } catch (InterruptedException e) {}

        }

      }

      // wait for dataXceiveServer to terminate

      try {

        this.dataXceiverServer.join();

      } catch (InterruptedException ie) {

      }

    }

   

    RPC.stopProxy(namenode); // stop the RPC threads

   

    if(upgradeManager != null)

      upgradeManager.shutdownUpgrade();

    if (blockScannerThread != null) {

      blockScannerThread.interrupt();

      try {

        blockScannerThread.join(3600000L); // wait for at most 1 hour

      } catch (InterruptedException ie) {

      }

    }

    if (storage != null) {

      try {

        this.storage.unlockAll();

      } catch (IOException ie) {

      }

    }

    if (dataNodeThread != null) {

      dataNodeThread.interrupt();

      try {

        dataNodeThread.join();

      } catch (InterruptedException ie) {

      }

    }

    if (data != null) {

      data.shutdown();

    }

    if (myMetrics != null) {

      myMetrics.shutdown();

    }

  }

 

 

  /** Check if there is no space in disk

   *  @param e that caused this checkDiskError call

   **/

  protected void checkDiskError(Exception e ) throws IOException {

   

    LOG.warn("checkDiskError: exception: ", e);

   

    if (e.getMessage() != null &&

        e.getMessage().startsWith("No space left on device")) {

      throw new DiskOutOfSpaceException("No space left on device");

    } else {

      checkDiskError();

    }

  }

 

  /**

   *  Check if there is a disk failure and if so, handle the error

   *

   **/

  protected void checkDiskError( ) {

    try {

      data.checkDataDir();

    } catch(DiskErrorException de) {

      handleDiskError(de.getMessage());

    }

  }

 

  private void handleDiskError(String errMsgr) {

    final boolean hasEnoughResources = data.hasEnoughResources();

    LOG.warn("DataNode.handleDiskError: Keep Running: " + hasEnoughResources);

         

    // If we have enough active valid volumes then we do not want to

    // shutdown the DN completely.

    int dpError = hasEnoughResources ? DatanodeProtocol.DISK_ERROR 

                                     : DatanodeProtocol.FATAL_DISK_ERROR;

    try {

      namenode.errorReport(dnRegistration, dpError, errMsgr);

    } catch(IOException ignored) {

    }

   

    if (hasEnoughResources) {

      scheduleBlockReport(0);

      return; // do not shutdown

    }

   

    LOG.warn("DataNode is shutting down.\n" + errMsgr);

    shouldRun = false;

  }

   

  /** Number of concurrent xceivers per node. */

  int getXceiverCount() {

    return threadGroup == null ? 0 : threadGroup.activeCount();

  }

   

  /**

   * Main loop for the DataNode.  Runs until shutdown,

   * forever calling remote NameNode functions.

   */

  public void offerService() throws Exception {

    

    LOG.info("using BLOCKREPORT_INTERVAL of " + blockReportInterval + "msec" +

       " Initial delay: " + initialBlockReportDelay + "msec");

    //

    // Now loop for a long time....

    //

    while (shouldRun) {

      try {

        long startTime = now();

        //

        // Every so often, send heartbeat or block-report

        //

       

        if (startTime - lastHeartbeat > heartBeatInterval) {

          //

          // All heartbeat messages include following info:

          // -- Datanode name

          // -- data transfer port

          // -- Total capacity

          // -- Bytes remaining

          //

          lastHeartbeat = startTime;

          DatanodeCommand[] cmds = namenode.sendHeartbeat(dnRegistration,

                                                       data.getCapacity(),

                                                       data.getDfsUsed(),

                                                       data.getRemaining(),

                                                       xmitsInProgress.get(),

                                                       getXceiverCount());

          myMetrics.heartbeats.inc(now() - startTime);

          //LOG.info("Just sent heartbeat, with name " + localName);

          if (!processCommand(cmds))

            continue;

        }

           

        // check if there are newly received blocks

        Block [] blockArray=null;

        String [] delHintArray=null;

        synchronized(receivedBlockList) {

          synchronized(delHints) {

            int numBlocks = receivedBlockList.size();

            if (numBlocks > 0) {

              if(numBlocks!=delHints.size()) {

                LOG.warn("Panic: receiveBlockList and delHints are not of the same length" );

              }

              //

              // Send newly-received blockids to namenode

              //

              blockArray = receivedBlockList.toArray(new Block[numBlocks]);

              delHintArray = delHints.toArray(new String[numBlocks]);

            }

          }

        }

        if (blockArray != null) {

          if(delHintArray == null || delHintArray.length != blockArray.length ) {

            LOG.warn("Panic: block array & delHintArray are not the same" );

          }

          namenode.blockReceived(dnRegistration, blockArray, delHintArray);

          synchronized (receivedBlockList) {

            synchronized (delHints) {

              for(int i=0; i<blockArray.length; i++) {

                receivedBlockList.remove(blockArray[i]);

                delHints.remove(delHintArray[i]);

              }

            }

          }

        }

        // send block report

        if (startTime - lastBlockReport > blockReportInterval) {

          //

          // Send latest blockinfo report if timer has expired.

          // Get back a list of local block(s) that are obsolete

          // and can be safely GC'ed.

          //

          long brStartTime = now();

          Block[] bReport = data.getBlockReport();

          DatanodeCommand cmd = namenode.blockReport(dnRegistration,

                  BlockListAsLongs.convertToArrayLongs(bReport));

          long brTime = now() - brStartTime;

          myMetrics.blockReports.inc(brTime);

          LOG.info("BlockReport of " + bReport.length +

              " blocks got processed in " + brTime + " msecs");

          //

          // If we have sent the first block report, then wait a random

          // time before we start the periodic block reports.

          //

          if (resetBlockReportTime) {

            lastBlockReport = startTime - R.nextInt((int)(blockReportInterval));

            resetBlockReportTime = false;

          } else {

            /* say the last block report was at 8:20:14. The current report

             * should have started around 9:20:14 (default 1 hour interval).

             * If current time is :

             *   1) normal like 9:20:18, next report should be at 10:20:14

             *   2) unexpected like 11:35:43, next report should be at 12:20:14

             */

            lastBlockReport += (now() - lastBlockReport) /

                               blockReportInterval * blockReportInterval;

          }

          processCommand(cmd);

        }

        // start block scanner

        if (blockScanner != null && blockScannerThread == null &&

            upgradeManager.isUpgradeCompleted()) {

          LOG.info("Starting Periodic block scanner.");

          blockScannerThread = new Daemon(blockScanner);

          blockScannerThread.start();

        }

           

        //

        // There is no work to do;  sleep until hearbeat timer elapses,

        // or work arrives, and then iterate again.

        //

        long waitTime = heartBeatInterval - (System.currentTimeMillis() - lastHeartbeat);

        synchronized(receivedBlockList) {

          if (waitTime > 0 && receivedBlockList.size() == 0) {

            try {

              receivedBlockList.wait(waitTime);

            } catch (InterruptedException ie) {

            }

            delayBeforeBlockReceived();

          }

        } // synchronized

      } catch(RemoteException re) {

        String reClass = re.getClassName();

        if (UnregisteredDatanodeException.class.getName().equals(reClass) ||

            DisallowedDatanodeException.class.getName().equals(reClass) ||

            IncorrectVersionException.class.getName().equals(reClass)) {

          LOG.warn("DataNode is shutting down: " +

                   StringUtils.stringifyException(re));

          shutdown();

          return;

        }

        try {

          Thread.sleep(1000);

        } catch (InterruptedException ie) {}

        LOG.warn(StringUtils.stringifyException(re));

      } catch (IOException e) {

        LOG.warn(StringUtils.stringifyException(e));

      }

    } // while (shouldRun)

  } // offerService

  /**

   * When a block has been received, we can delay some period of time before

   * reporting it to the DN, for the purpose of testing. This simulates

   * the actual latency of blockReceived on a real network (where the client

   * may be closer to the NN than the DNs).

   */

  private void delayBeforeBlockReceived() {

    if (artificialBlockReceivedDelay > 0 && !receivedBlockList.isEmpty()) {

      try {

        long sleepFor = (long)R.nextInt(artificialBlockReceivedDelay);

        LOG.debug("DataNode " + dnRegistration + " sleeping for " +

                  "artificial delay: " + sleepFor + " ms");

        Thread.sleep(sleepFor);

      } catch (InterruptedException ie) {

        Thread.currentThread().interrupt();

      }

    }

  }

  /**

   * Process an array of datanode commands

   *

   * @param cmds an array of datanode commands

   * @return true if further processing may be required or false otherwise.

   */

  private boolean processCommand(DatanodeCommand[] cmds) {

    if (cmds != null) {

      for (DatanodeCommand cmd : cmds) {

        try {

          if (processCommand(cmd) == false) {

            return false;

          }

        } catch (IOException ioe) {

          LOG.warn("Error processing datanode Command", ioe);

        }

      }

    }

    return true;

  }

 

    /**

     *

     * @param cmd

     * @return true if further processing may be required or false otherwise.

     * @throws IOException

     */

  private boolean processCommand(DatanodeCommand cmd) throws IOException {

    if (cmd == null)

      return true;

    final BlockCommand bcmd = cmd instanceof BlockCommand? (BlockCommand)cmd: null;

    switch(cmd.getAction()) {

    case DatanodeProtocol.DNA_TRANSFER:

      // Send a copy of a block to another datanode

      transferBlocks(bcmd.getBlocks(), bcmd.getTargets());

      myMetrics.blocksReplicated.inc(bcmd.getBlocks().length);

      break;

    case DatanodeProtocol.DNA_INVALIDATE:

      //

      // Some local block(s) are obsolete and can be

      // safely garbage-collected.

      //

      Block toDelete[] = bcmd.getBlocks();

      try {

        if (blockScanner != null) {

          blockScanner.deleteBlocks(toDelete);

        }

        data.invalidate(toDelete);

      } catch(IOException e) {

        checkDiskError();

        throw e;

      }

      myMetrics.blocksRemoved.inc(toDelete.length);

      break;

    case DatanodeProtocol.DNA_SHUTDOWN:

      // shut down the data node

      this.shutdown();

      return false;

    case DatanodeProtocol.DNA_REGISTER:

      // namenode requested a registration - at start or if NN lost contact

      LOG.info("DatanodeCommand action: DNA_REGISTER");

      if (shouldRun) {

        register();

        pluginDispatcher.dispatchCall(

          new SingleArgumentRunnable<DatanodePlugin>() {

            public void run(DatanodePlugin p) { p.reregistrationComplete(); }

          });

      }

      break;

    case DatanodeProtocol.DNA_FINALIZE:

      storage.finalizeUpgrade();

      break;

    case UpgradeCommand.UC_ACTION_START_UPGRADE:

      // start distributed upgrade here

      processDistributedUpgradeCommand((UpgradeCommand)cmd);

      break;

    case DatanodeProtocol.DNA_RECOVERBLOCK:

      recoverBlocks(bcmd.getBlocks(), bcmd.getTargets());

      break;

    case DatanodeProtocol.DNA_ACCESSKEYUPDATE:

      LOG.info("DatanodeCommand action: DNA_ACCESSKEYUPDATE");

      if (isBlockTokenEnabled) {

        blockTokenSecretManager.setKeys(((KeyUpdateCommand) cmd).getExportedKeys());

      }

      break;

    default:

      LOG.warn("Unknown DatanodeCommand action: " + cmd.getAction());

    }

    return true;

  }

  // Distributed upgrade manager

  UpgradeManagerDatanode upgradeManager = new UpgradeManagerDatanode(this);

  private void processDistributedUpgradeCommand(UpgradeCommand comm

                                               ) throws IOException {

    assert upgradeManager != null : "DataNode.upgradeManager is null.";

    upgradeManager.processUpgradeCommand(comm);

  }

  /**

   * Start distributed upgrade if it should be initiated by the data-node.

   */

  private void startDistributedUpgradeIfNeeded() throws IOException {

    UpgradeManagerDatanode um = DataNode.getDataNode().upgradeManager;

    assert um != null : "DataNode.upgradeManager is null.";

    if(!um.getUpgradeState())

      return;

    um.setUpgradeState(false, um.getUpgradeVersion());

    um.startUpgrade();

    return;

  }

  private void transferBlock( Block block,

                              DatanodeInfo xferTargets[]

                              ) throws IOException {

    if (!data.isValidBlock(block)) {

      // block does not exist or is under-construction

      String errStr = "Can't send invalid block " + block;

      LOG.info(errStr);

      namenode.errorReport(dnRegistration,

                           DatanodeProtocol.INVALID_BLOCK,

                           errStr);

      return;

    }

    // Check if NN recorded length matches on-disk length

    long onDiskLength = data.getLength(block);

    if (block.getNumBytes() > onDiskLength) {

      // Shorter on-disk len indicates corruption so report NN the corrupt block

      namenode.reportBadBlocks(new LocatedBlock[]{

          new LocatedBlock(block, new DatanodeInfo[] {

              new DatanodeInfo(dnRegistration)})});

      LOG.info("Can't replicate block " + block

          + " because on-disk length " + onDiskLength

          + " is shorter than NameNode recorded length " + block.getNumBytes());

      return;

    }

   

    int numTargets = xferTargets.length;

    if (numTargets > 0) {

      if (LOG.isInfoEnabled()) {

        StringBuilder xfersBuilder = new StringBuilder();

        for (int i = 0; i < numTargets; i++) {

          xfersBuilder.append(xferTargets[i].getName());

          xfersBuilder.append(" ");

        }

        LOG.info(dnRegistration + " Starting thread to transfer block " +

                 block + " to " + xfersBuilder);                      

      }

      new Daemon(new DataTransfer(xferTargets, block, this)).start();

    }

  }

  private void transferBlocks( Block blocks[],

                               DatanodeInfo xferTargets[][]

                               ) {

    for (int i = 0; i < blocks.length; i++) {

      try {

        transferBlock(blocks[i], xferTargets[i]);