Examples of DataNodeDescriptor


Examples of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor

                + " because the file exists");
          }
        }
      }

      final DatanodeDescriptor clientNode =
          blockManager.getDatanodeManager().getDatanodeByHost(clientMachine);

      if (append && myFile != null) {
        return prepareFileForWrite(
            src, myFile, holder, clientMachine, clientNode, true);
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor

      IOException {
    checkBlock(previous);
    Block previousBlock = ExtendedBlock.getLocalBlock(previous);
    long fileLength, blockSize;
    int replication;
    DatanodeDescriptor clientNode = null;
    Block newBlock = null;

    if(NameNode.stateChangeLog.isDebugEnabled()) {
      NameNode.stateChangeLog.debug(
          "BLOCK* NameSystem.getAdditionalBlock: file "
          +src+" for "+clientName);
    }

    writeLock();
    try {
      checkOperation(OperationCategory.WRITE);

      if (isInSafeMode()) {
        throw new SafeModeException("Cannot add block to " + src, safeMode);
      }

      // have we exceeded the configured limit of fs objects.
      checkFsObjectLimit();

      INodeFileUnderConstruction pendingFile = checkLease(src, clientName);
      BlockInfo lastBlockInFile = pendingFile.getLastBlock();
      if (!Block.matchingIdAndGenStamp(previousBlock, lastBlockInFile)) {
        // The block that the client claims is the current last block
        // doesn't match up with what we think is the last block. There are
        // three possibilities:
        // 1) This is the first block allocation of an append() pipeline
        //    which started appending exactly at a block boundary.
        //    In this case, the client isn't passed the previous block,
        //    so it makes the allocateBlock() call with previous=null.
        //    We can distinguish this since the last block of the file
        //    will be exactly a full block.
        // 2) This is a retry from a client that missed the response of a
        //    prior getAdditionalBlock() call, perhaps because of a network
        //    timeout, or because of an HA failover. In that case, we know
        //    by the fact that the client is re-issuing the RPC that it
        //    never began to write to the old block. Hence it is safe to
        //    abandon it and allocate a new one.
        // 3) This is an entirely bogus request/bug -- we should error out
        //    rather than potentially appending a new block with an empty
        //    one in the middle, etc

        BlockInfo penultimateBlock = pendingFile.getPenultimateBlock();
        if (previous == null &&
            lastBlockInFile != null &&
            lastBlockInFile.getNumBytes() == pendingFile.getPreferredBlockSize() &&
            lastBlockInFile.isComplete()) {
          // Case 1
          if (NameNode.stateChangeLog.isDebugEnabled()) {
             NameNode.stateChangeLog.debug(
                 "BLOCK* NameSystem.allocateBlock: handling block allocation" +
                 " writing to a file with a complete previous block: src=" +
                 src + " lastBlock=" + lastBlockInFile);
          }
        } else if (Block.matchingIdAndGenStamp(penultimateBlock, previousBlock)) {
          // Case 2
          if (lastBlockInFile.getNumBytes() != 0) {
            throw new IOException(
                "Request looked like a retry to allocate block " +
                lastBlockInFile + " but it already contains " +
                lastBlockInFile.getNumBytes() + " bytes");
          }

          // The retry case ("b" above) -- abandon the old block.
          NameNode.stateChangeLog.info("BLOCK* NameSystem.allocateBlock: " +
              "caught retry for allocation of a new block in " +
              src + ". Abandoning old block " + lastBlockInFile);
          dir.removeBlock(src, pendingFile, lastBlockInFile);
          dir.persistBlocks(src, pendingFile);
        } else {
         
          throw new IOException("Cannot allocate block in " + src + ": " +
              "passed 'previous' block " + previous + " does not match actual " +
              "last block in file " + lastBlockInFile);
        }
      }

      // commit the last block and complete it if it has minimum replicas
      commitOrCompleteLastBlock(pendingFile, previousBlock);

      //
      // If we fail this, bad things happen!
      //
      if (!checkFileProgress(pendingFile, false)) {
        throw new NotReplicatedYetException("Not replicated yet:" + src);
      }
      fileLength = pendingFile.computeContentSummary().getLength();
      blockSize = pendingFile.getPreferredBlockSize();
      clientNode = pendingFile.getClientNode();
      replication = pendingFile.getReplication();
    } finally {
      writeUnlock();
    }

    // choose targets for the new block to be allocated.
    final DatanodeDescriptor targets[] = blockManager.chooseTarget(
        src, replication, clientNode, excludedNodes, blockSize);

    // Allocate a new block and record it in the INode.
    writeLock();
    try {
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor

      final int numAdditionalNodes, final String clientName
      ) throws IOException {
    //check if the feature is enabled
    dtpReplaceDatanodeOnFailure.checkEnabled();

    final DatanodeDescriptor clientnode;
    final long preferredblocksize;
    final List<DatanodeDescriptor> chosen;
    readLock();
    try {
      checkOperation(OperationCategory.WRITE);
      //check safe mode
      if (isInSafeMode()) {
        throw new SafeModeException("Cannot add datanode; src=" + src
            + ", blk=" + blk, safeMode);
      }

      //check lease
      final INodeFileUnderConstruction file = checkLease(src, clientName);
      clientnode = file.getClientNode();
      preferredblocksize = file.getPreferredBlockSize();

      //find datanode descriptors
      chosen = new ArrayList<DatanodeDescriptor>();
      for(DatanodeInfo d : existings) {
        final DatanodeDescriptor descriptor = blockManager.getDatanodeManager(
            ).getDatanode(d);
        if (descriptor != null) {
          chosen.add(descriptor);
        }
      }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor

      final long blocksize, Configuration conf) throws IOException {
    final BlockManager bm = namenode.getNamesystem().getBlockManager();

    if (op == PutOpParam.Op.CREATE) {
      //choose a datanode near to client
      final DatanodeDescriptor clientNode = bm.getDatanodeManager(
          ).getDatanodeByHost(getRemoteAddress());
      if (clientNode != null) {
        final DatanodeDescriptor[] datanodes = bm.getBlockPlacementPolicy()
            .chooseTarget(path, 1, clientNode,
                new ArrayList<DatanodeDescriptor>(), false, null, blocksize);
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor

        doc.startTag("replicas");
        for(final Iterator<DatanodeDescriptor> it = blockManager.datanodeIterator(block);
            it.hasNext(); ) {
          doc.startTag("replica");

          DatanodeDescriptor dd = it.next();

          doc.startTag("host_name");
          doc.pcdata(dd.getHostName());
          doc.endTag();

          boolean isCorrupt = blockManager.getCorruptReplicaBlockIds(0,
                                block.getBlockId()) != null;
         
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor

      builder.add(dataNode.getDisplayName());
    }
    set = builder.build();

    for (int i = 0; i < 10; i++) {
      DatanodeDescriptor dnDescriptor = NamenodeJspHelper
          .getRandomDatanode(nameNode);
      assertTrue("testGetRandomDatanode error",
          set.contains(dnDescriptor.toString()));
    }
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor

      decommissionedNodes.add(downnode);
      Thread.sleep(5000);
      final List<DatanodeDescriptor> decommissioningNodes = dm.getDecommissioningNodes();
      if (iteration == 0) {
        assertEquals(decommissioningNodes.size(), 1);
        DatanodeDescriptor decommNode = decommissioningNodes.get(0);
        checkDecommissionStatus(decommNode, 4, 0, 2);
        checkDFSAdminDecommissionStatus(decommissioningNodes.subList(0, 1),
            fileSys, admin);
      } else {
        assertEquals(decommissioningNodes.size(), 2);
        DatanodeDescriptor decommNode1 = decommissioningNodes.get(0);
        DatanodeDescriptor decommNode2 = decommissioningNodes.get(1);
        checkDecommissionStatus(decommNode1, 4, 4, 2);
        checkDecommissionStatus(decommNode2, 4, 4, 2);
        checkDFSAdminDecommissionStatus(decommissioningNodes.subList(0, 2),
            fileSys, admin);
      }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor

      final List<Long> blockIds) throws IOException {
    namesystem.writeLock();
    final long startTime = Time.monotonicNow();
    final long endTime;
    try {
      final DatanodeDescriptor datanode =
          blockManager.getDatanodeManager().getDatanode(datanodeID);
      if (datanode == null || !datanode.isAlive) {
        throw new IOException(
            "processCacheReport from dead or unregistered datanode: " +
            datanode);
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor

      final long blocksize) throws IOException {
    final BlockManager bm = namenode.getNamesystem().getBlockManager();

    if (op == PutOpParam.Op.CREATE) {
      //choose a datanode near to client
      final DatanodeDescriptor clientNode = bm.getDatanodeManager(
          ).getDatanodeByHost(getRemoteAddress());
      if (clientNode != null) {
        final DatanodeStorageInfo[] storages = bm.getBlockPlacementPolicy()
            .chooseTarget(path, 1, clientNode,
                new ArrayList<DatanodeStorageInfo>(), false, null, blocksize,
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor

              + " on client " + clientMachine
              + " because the file exists");
        }
      }

      final DatanodeDescriptor clientNode =
          blockManager.getDatanodeManager().getDatanodeByHost(clientMachine);

      if (append && myFile != null) {
        return prepareFileForWrite(
            src, myFile, holder, clientMachine, clientNode, true);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.