Examples of org.apache.hadoop.net.NetworkTopology

org.apache.hadoop.net.NetworkTopology
The class represents a cluster of computer with a tree hierarchical network topology. For example, a cluster may be consists of many data centers filled with racks of computers. In a network topology, leaves represent data nodes (computers) and inner nodes represent switches/routers that manage traffic in/out of data centers or racks. @author hairong

   * dirs is a list of directories where the filesystem directory state
   * is stored
   */
  FSNamesystem(FSImage fsImage, Configuration conf) throws IOException {
    super(conf);
    this.clusterMap = new NetworkTopology(conf);
    this.fsLock = new ReentrantReadWriteLock();
    setConfigurationParameters(conf);
    this.dir = new FSDirectory(fsImage, this, conf);
  }

View Full Code Here

    return goodBlock;
  }


  /* reset all fields in a balancer preparing for the next iteration */
  private void resetData() {
    this.cluster = new NetworkTopology();
    this.overUtilizedDatanodes.clear();
    this.aboveAvgUtilizedDatanodes.clear();
    this.belowAvgUtilizedDatanodes.clear();
    this.underUtilizedDatanodes.clear();
    this.datanodes.clear();

View Full Code Here

    rackA = nodes.subList(0, 3);
    rackB = nodes.subList(3, 6);
  }
  
  private void addNodes(Iterable<DatanodeDescriptor> nodesToAdd) {
    NetworkTopology cluster = bm.getDatanodeManager().getNetworkTopology();
    // construct network topology
    for (DatanodeDescriptor dn : nodesToAdd) {
      cluster.add(dn);
      dn.updateHeartbeat(
          2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
          2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0, 0);
      bm.getDatanodeManager().checkIfClusterIsNowMultiRack(dn);
    }

View Full Code Here

      bm.getDatanodeManager().checkIfClusterIsNowMultiRack(dn);
    }
  }


  private void removeNode(DatanodeDescriptor deadNode) {
    NetworkTopology cluster = bm.getDatanodeManager().getNetworkTopology();
    cluster.remove(deadNode);
    bm.removeBlocksAssociatedTo(deadNode);
  }

View Full Code Here

    long minSize = Math.max(job.getLong("mapred.min.split.size", 1),
                            minSplitSize);


    // generate splits
    ArrayList<FileSplit> splits = new ArrayList<FileSplit>(numSplits);
    NetworkTopology clusterMap = new NetworkTopology();
    for (LocatedFileStatus file: files) {
      Path path = file.getPath();
      FileSystem fs = path.getFileSystem(job);
      long length = file.getLen();
      BlockLocation[] blkLocations = file.getBlockLocations();

View Full Code Here


  public void testPolicyInitAndFirstChoice() throws Exception {
    VerifiablePolicy policy = new VerifiablePolicy();
    Configuration conf = new Configuration();
    TestClusterStats stats = new TestClusterStats();
    NetworkTopology clusterMap = new NetworkTopology();
    TestHostsReader hostsReader = new TestHostsReader();
    TestMapping dnsToSwitchMapping = new TestMapping();




    for (DatanodeDescriptor d: dataNodes) {
      clusterMap.add(d);
    }


    conf.setInt("dfs.replication.rackwindow", 1);
    conf.setInt("dfs.replication.machineWindow", 2);

View Full Code Here


  private VerifiablePolicy initTest() throws Exception {
    VerifiablePolicy policy = new VerifiablePolicy();
    Configuration conf = new Configuration();
    TestClusterStats stats = new TestClusterStats();
    NetworkTopology clusterMap = new NetworkTopology();
    TestHostsReader hostsReader = new TestHostsReader();
    TestMapping dnsToSwitchMapping = new TestMapping();


    for (DatanodeDescriptor d: dataNodes) {
      clusterMap.add(d);
    }


    conf.setInt("dfs.replication.rackwindow", 2);
    conf.setInt("dfs.replication.machineWindow", 2);

View Full Code Here


  public void testChooseTarget() throws Exception {
    VerifiablePolicy policy = new VerifiablePolicy();
    Configuration conf = new Configuration();
    TestClusterStats stats = new TestClusterStats();
    NetworkTopology clusterMap = new NetworkTopology();
    TestHostsReader hostsReader = new TestHostsReader();
    TestMapping dnsToSwitchMapping = new TestMapping();


    for (DatanodeDescriptor d: dataNodes) {
      clusterMap.add(d);
    }


    conf.setInt("dfs.replication.rackwindow", 2);
    conf.setInt("dfs.replication.machineWindow", 2);

View Full Code Here


  public void testFindBest() throws Exception {
    VerifiablePolicy policy = new VerifiablePolicy();
    Configuration conf = new Configuration();
    TestClusterStats stats = new TestClusterStats();
    NetworkTopology clusterMap = new NetworkTopology();
    TestHostsReader hostsReader = new TestHostsReader();
    TestMapping dnsToSwitchMapping = new TestMapping();




    for (DatanodeDescriptor d: dataNodes) {
      clusterMap.add(d);
    }


    conf.setInt("dfs.replication.rackwindow", 2);
    conf.setInt("dfs.replication.machineWindow", 2);

View Full Code Here

  public void testRemainingReplicas() throws Exception {


    VerifiablePolicy policy = new VerifiablePolicy();
    Configuration conf = new Configuration();
    TestClusterStats stats = new TestClusterStats();
    NetworkTopology clusterMap = new NetworkTopology();
    TestHostsReader hostsReader = new TestHostsReader();
    TestMapping dnsToSwitchMapping = new TestMapping();




    for (DatanodeDescriptor d: dataNodes) {
      clusterMap.add(d);
    }


    ArrayList<DatanodeDescriptor> results = new ArrayList<DatanodeDescriptor>();
    HashMap<Node, Node> excludedNodes = new HashMap<Node, Node>();
    int bls = 1024;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.net.NetworkTopology

org.apache.hadoop.dfs.Balancer

org.apache.hadoop.dfs.DatanodeDescriptor

org.apache.hadoop.hdfs.server.balancer.Balancer

org.apache.hadoop.hdfs.server.blockmanagement.TestBlockManager

org.apache.hadoop.hdfs.server.namenode.FSNamesystem

org.apache.hadoop.hdfs.server.namenode.FSNamesystem$SafeModeInfo

org.apache.hadoop.hdfs.server.namenode.TestConfigurableBlockPlacement

org.apache.hadoop.hdfs.server.namenode.TestFsck

org.apache.hadoop.mapred.FileInputFormat

org.apache.hadoop.mapred.TestGetSplitHosts

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.