* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* FSNamesystem does the actual bookkeeping work for the

* DataNode.

*

* It tracks several important tables.

*

* 1)  valid fsname --> blocklist  (kept on disk, logged)

* 2)  Set of all valid blocks (inverted #1)

* 3)  block --> machinelist (kept in memory, rebuilt dynamically from reports)

* 4)  machine --> blocklist (inverted #2)

* 5)  LRU cache of updated-heartbeat machines

***************************************************/

  public static final Log LOG = LogFactory.getLog("org.apache.hadoop.fs.FSNamesystem");

  //

  // Stores the correct file name hierarchy

  //

  FSDirectory dir;

  //

  // Stores the block-->datanode(s) map.  Updated only in response

  // to client-sent information.

  // Mapping: Block -> { INode, datanodes, self ref }

  //

  BlocksMap blocksMap = new BlocksMap();

   

  /**

   * Stores the datanode -> block map. 

   * <p>

   * Done by storing a set of {@link DatanodeDescriptor} objects, sorted by

   * storage id. In order to keep the storage map consistent it tracks

   * all storages ever registered with the namenode.

   * A descriptor corresponding to a specific storage id can be

   * <ul>

   * <li>added to the map if it is a new storage id;</li>

   * <li>updated with a new datanode started as a replacement for the old one

   * with the same storage id; and </li>

   * <li>removed if and only if an existing datanode is restarted to serve a

   * different storage id.</li>

   * </ul> <br>

   * The list of the {@link DatanodeDescriptor}s in the map is checkpointed

   * in the namespace image file. Only the {@link DatanodeInfo} part is

   * persistent, the list of blocks is restored from the datanode block

   * reports.

   * <p>

   * Mapping: StorageID -> DatanodeDescriptor

   */

  Map<String, DatanodeDescriptor> datanodeMap =

    new TreeMap<String, DatanodeDescriptor>();

  //

  // Keeps a Collection for every named machine containing

  // blocks that have recently been invalidated and are thought to live

  // on the machine in question.

  // Mapping: StorageID -> ArrayList<Block>

  //

  private Map<String, Collection<Block>> recentInvalidateSets =

    new TreeMap<String, Collection<Block>>();

  //

  // Keeps a TreeSet for every named node.  Each treeset contains

  // a list of the blocks that are "extra" at that location.  We'll

  // eventually remove these extras.

  // Mapping: StorageID -> TreeSet<Block>

  //

  private Map<String, Collection<Block>> excessReplicateMap =

    new TreeMap<String, Collection<Block>>();

  //

  // Stats on overall usage

  //

  long totalCapacity = 0L, totalUsed=0L, totalRemaining = 0L;

  // total number of connections per live datanode

  int totalLoad = 0;

  //

  // For the HTTP browsing interface

  //

  StatusHttpServer infoServer;

  int infoPort;

  String infoBindAddress;

  Date startTime;

   

  //

  Random r = new Random();

  /**

   * Stores a set of DatanodeDescriptor objects.

   * This is a subset of {@link #datanodeMap}, containing nodes that are

   * considered alive.

   * The {@link HeartbeatMonitor} periodically checks for outdated entries,

   * and removes them from the list.

   */

  ArrayList<DatanodeDescriptor> heartbeats = new ArrayList<DatanodeDescriptor>();

  //

  // Store set of Blocks that need to be replicated 1 or more times.

  // We also store pending replication-orders.

  // Set of: Block

  //

  private UnderReplicatedBlocks neededReplications = new UnderReplicatedBlocks();

  private PendingReplicationBlocks pendingReplications;

  //

  // Used for handling lock-leases

  // Mapping: leaseHolder -> Lease

  //

  private Map<StringBytesWritable, Lease> leases = new TreeMap<StringBytesWritable, Lease>();

  // Set of: Lease

  private SortedSet<Lease> sortedLeases = new TreeSet<Lease>();

  //

  // Threaded object that checks to see if we have been

  // getting heartbeats from all clients.

  //

  Daemon hbthread = null;   // HeartbeatMonitor thread

  Daemon lmthread = null;   // LeaseMonitor thread

  Daemon smmthread = null;  // SafeModeMonitor thread

  Daemon replthread = null;  // Replication thread

  volatile boolean fsRunning = true;

  long systemStart = 0;

  //  The maximum number of replicates we should allow for a single block

  private int maxReplication;

  //  How many outgoing replication streams a given node should have at one time

  private int maxReplicationStreams;

  // MIN_REPLICATION is how many copies we need in place or else we disallow the write

  private int minReplication;

  // Default replication

  private int defaultReplication;

  // heartbeatRecheckInterval is how often namenode checks for expired datanodes

  private long heartbeatRecheckInterval;

  // heartbeatExpireInterval is how long namenode waits for datanode to report

  // heartbeat

  private long heartbeatExpireInterval;

  //replicationRecheckInterval is how often namenode checks for new replication work

  private long replicationRecheckInterval;

  //decommissionRecheckInterval is how often namenode checks if a node has finished decommission

  private long decommissionRecheckInterval;

  // default block size of a file

  private long defaultBlockSize = 0;

  private int replIndex = 0; // last datanode used for replication work

  static int REPL_WORK_PER_ITERATION = 32; // max percent datanodes per iteration

  public static FSNamesystem fsNamesystemObject;

  private String localMachine;

  private int port;

  private SafeModeInfo safeMode;  // safe mode information

  private Host2NodesMap host2DataNodeMap = new Host2NodesMap();

   

  // datanode networktoplogy

  NetworkTopology clusterMap = new NetworkTopology();

  // for block replicas placement

  ReplicationTargetChooser replicator;

  private HostsFileReader hostsReader;

  private Daemon dnthread = null;

  // can fs-image be rolled?

  volatile private CheckpointStates ckptState = CheckpointStates.START;

  private static final SimpleDateFormat DATE_FORM =

    new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");

  /**

   * dirs is a list of directories where the filesystem directory state

   * is stored

   */

  public FSNamesystem(String hostname,

                      int port,

                      NameNode nn, Configuration conf) throws IOException {

    fsNamesystemObject = this;

    setConfigurationParameters(conf);

    this.localMachine = hostname;

    this.port = port;

    this.dir = new FSDirectory(this, conf);

    StartupOption startOpt = NameNode.getStartupOption(conf);