* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* DFSOutputStream creates files from a stream of bytes.

*

* The client application writes data that is cached internally by

* this stream. Data is broken up into packets, each packet is

* typically 64K in size. A packet comprises of chunks. Each chunk

* is typically 512 bytes and has an associated checksum with it.

*

* When a client application fills up the currentPacket, it is

* enqueued into dataQueue.  The DataStreamer thread picks up

* packets from the dataQueue, sends it to the first datanode in

* the pipeline and moves it from the dataQueue to the ackQueue.

* The ResponseProcessor receives acks from the datanodes. When an

* successful ack for a packet is received from all datanodes, the

* ResponseProcessor removes the corresponding packet from the

* ackQueue.

*

* In case of error, all outstanding packets and moved from

* ackQueue. A new pipeline is setup by eliminating the bad

* datanode from the original pipeline. The DataStreamer now

* starts sending packets from the dataQueue.

****************************************************************/

  private final DFSClient dfsClient;

  private Socket[] s;

  boolean closed = false;

  private String src;

  private MultiDataOutputStream blockStream;

  private MultiDataInputStream blockReplyStream;

  private Block block;

  final private long blockSize;

  private boolean pktIncludeVersion = false;

  final private int packetVersion;

  private DataChecksum checksum;

  private LinkedList<DFSOutputStreamPacket> dataQueue = new LinkedList<DFSOutputStreamPacket>();

  private LinkedList<DFSOutputStreamPacket> ackQueue = new LinkedList<DFSOutputStreamPacket>();

  private int numPendingHeartbeats = 0;

  private long lastPacketSentTime = 0;

  private final long packetTimeout;

  private DFSOutputStreamPacket currentPacket = null;

  private int maxPackets = 80; // each packet 64K, total 5MB

  // private int maxPackets = 1000; // each packet 64K, total 64MB

  private DataStreamer streamer;

  private ResponseProcessor response = null;

  private long currentSeqno = 0;

  private long lastQueuedSeqno = -1;

  private long lastAckedSeqno = -1;

  private long bytesCurBlock = 0; // bytes writen in current block

  private int packetSize = 0; // write packet size, including the header.

  private int chunksPerPacket = 0;

  DatanodeInfo[] nodes = null; // list of targets for current block

  private DatanodeInfo[] favoredNodes = null; // put replicas here if possible

  private volatile boolean hasError = false;

  private volatile int errorIndex = 0;

  volatile IOException lastException = null;

  private long artificialSlowdown = 0;

  private long lastFlushOffset = 0; // offset when flush was invoked

  private boolean persistBlocks = false; // persist blocks on namenode

  private int recoveryErrorCount = 0; // number of times block recovery failed

  private final int maxRecoveryErrorCount;

  private volatile boolean appendChunk = false;   // appending to existing partial block

  private long initialFileSize = 0; // at time of file open

  private Progressable progress;

  private short blockReplication; // replication factor of file

  private long lastBlkOffset = 0; // end pos of last block already sent

  private boolean forceSync;

  private boolean doParallelWrites = false;

   

  private final WriteOptions options;

  private void setLastException(IOException e) {

    if (lastException == null) {

      lastException = e;

    }

  }

 

  public void setOffsets(long offset) {

    DFSClient.LOG.info("set last block offsets in file: " + src + " pos: " + offset);

    lastBlkOffset = offset;

  }

  /** Decide if the write pipeline supports bidirectional heartbeat or not */

  private boolean supportClientHeartbeat() throws IOException {

    return dfsClient.getDataTransferProtocolVersion() >=

                 DataTransferProtocol.CLIENT_HEARTBEAT_VERSION;

  }

  /**

   * Check if the last outstanding packet has not received an ack before

   * it is timed out.

   * If true, for now just log it.

   * We will provide a decent solution to this later on.

   */

  private void checkIfLastPacketTimeout() {

     synchronized (ackQueue) {

             if( !ackQueue.isEmpty()  && (

                             System.currentTimeMillis() - lastPacketSentTime > packetTimeout) ) {

               DFSClient.LOG.warn("Packet " + ackQueue.getLast().seqno +

                             " of " + block + " is timed out");

             }

     }

  }

  //

  // The DataStreamer class is responsible for sending data packets to the

  // datanodes in the pipeline. It retrieves a new blockid and block locations

  // from the namenode, and starts streaming packets to the pipeline of

  // Datanodes. Every packet has a sequence number associated with

  // it. When all the packets for a block are sent out and acks for each

  // if them are received, the DataStreamer closes the current block.

  //

  private class DataStreamer extends Daemon {

    private volatile boolean closed = false;

    private long lastPacket;

    private boolean doSleep;

    DataStreamer() throws IOException {

      // explicitly invoke RPC so avoiding RPC in waitForWork

      // that might cause timeout

      dfsClient.getDataTransferProtocolVersion();

    }

    private void waitForWork() throws IOException {

      if ( supportClientHeartbeat() ) {  // send heart beat

        long now = System.currentTimeMillis();

        while ((!closed && !hasError && dfsClient.clientRunning

            && dataQueue.size() == 0  &&

            (blockStream == null || (

                blockStream != null && now - lastPacket < dfsClient.timeoutValue/2)))

                || doSleep) {

          long timeout = dfsClient.timeoutValue/2 - (now-lastPacket);

          timeout = timeout <= 0 ? 1000 : timeout;

          try {

            dataQueue.wait(timeout);

            checkIfLastPacketTimeout();

            now = System.currentTimeMillis();

          } catch (InterruptedException  e) {

          }

          doSleep = false;

        }

      } else { // no sending heart beat

        while ((!closed && !hasError && dfsClient.clientRunning

            && dataQueue.size() == 0) || doSleep) {

          try {

            dataQueue.wait(1000);

          } catch (InterruptedException  e) {

          }

          doSleep = false;

        }

      }

    }

    public void run() {

      while (!closed && dfsClient.clientRunning) {

        // if the Responder encountered an error, shutdown Responder

        if (hasError && response != null) {

          try {

            response.close();

            response.join();

            response = null;

          } catch (InterruptedException  e) {

          }

        }

        DFSOutputStreamPacket one = null;

        // process IO errors if any

        doSleep = processDatanodeError(hasError, false);

        try {

          synchronized (dataQueue) {

            // wait for a packet to be sent.

            waitForWork();

            if (closed || hasError || !dfsClient.clientRunning) {

              continue;

            }

            InjectionHandler

                .processEventIO(InjectionEvent.DFSCLIENT_DATASTREAM_AFTER_WAIT, blockStream);

           

            // get packet to be sent.

            if (dataQueue.isEmpty()) {

              one = DFSOutputStreamPacketFactory.getHeartbeatPacket(

                  DFSOutputStream.this, ifPacketIncludeVersion(),

                  getPacketVersion()); // heartbeat

                                                                     // packet

            } else {

              one = dataQueue.getFirst(); // regular data packet

              one.eventPopFromDataQueue();

            }

          }

                 

          long offsetInBlock = one.offsetInBlock;

          // get new block from namenode.

          if (blockStream == null) {

            DFSClient.LOG.debug("Allocating new block: " + src + "  pos: " + lastBlkOffset);

            nodes = nextBlockOutputStream(src);

            this.setName("DataStreamer for file " + src +

                " block " + block);

            response = new ResponseProcessor(nodes);

            response.start();

          }

          if (offsetInBlock > blockSize

              || (offsetInBlock == blockSize && (one.dataLength > 0 || !one.lastPacketInBlock))) {

            throw new IOException("BlockSize " + blockSize +

                                  " is smaller than data size. " +

                                  " Offset of packet in block " +

                                  offsetInBlock +

                                  " Aborting file " + src);

          }

          ByteBuffer buf = one.getBuffer();

          InjectionHandler.processEventIO(

              InjectionEvent.DFSCLIENT_DATASTREAM_BEFORE_WRITE, blockStream);

         

          // write out data to remote datanode

          blockStream.write(buf.array(), buf.position(), buf.remaining());

          if (one.lastPacketInBlock) {

            blockStream.writeInt(0); // indicate end-of-block

          }

          blockStream.flush();

          lastPacket = System.currentTimeMillis();

          if (DFSClient.LOG.isDebugEnabled()) {

            DFSClient.LOG.debug("DataStreamer block " + block +

                      " wrote packet seqno:" + one.seqno +

                      " size:" + buf.remaining() +

                      " offsetInBlock:" + one.offsetInBlock +

                      " lastPacketInBlock:" + one.lastPacketInBlock);

          }

          // move packet from dataQueue to ackQueue

          synchronized (dataQueue) {

            if (!one.isHeartbeatPacket()) {

              dataQueue.removeFirst();

              dataQueue.notifyAll();

              synchronized (ackQueue) {

                ackQueue.addLast(one);

                one.eventAddToAckQueue();

                lastPacketSentTime = System.currentTimeMillis();

                ackQueue.notifyAll();

              }

            } else {

              synchronized (ackQueue) {

                numPendingHeartbeats++;

                ackQueue.notifyAll();                 

              }

              DFSClient.LOG.info("Sending a heartbeat packet for block " + block);

            }

          }

        } catch (Throwable e) {

          dfsClient.incWriteExpCntToStats();

          DFSClient.LOG.warn("DataStreamer Exception: ", e);

          if (e instanceof IOException) {

            setLastException((IOException)e);

          }

          hasError = true;

          if (blockStream != null) {

            // find the first datanode to which we could not write data.

            int possibleError =  blockStream.getErrorIndex();

            if (possibleError != -1) {

              errorIndex = possibleError;

              DFSClient.LOG.warn("DataStreamer bad datanode in pipeline:" +

                         possibleError);

            }

          }

        }

        if (closed || hasError || !dfsClient.clientRunning) {

          continue;

        }

        // Is this block full?

        if (one.lastPacketInBlock) {

          synchronized (ackQueue) {

            while (!hasError && ackQueue.size() != 0 && dfsClient.clientRunning) {

              try {

                ackQueue.wait();   // wait for acks to arrive from datanodes

              } catch (InterruptedException  e) {

              }

            }

          }

          DFSClient.LOG.debug("Closing old block " + block);

          this.setName("DataStreamer for file " + src);

          response.close();        // ignore all errors in Response

          try {

            response.join();

            response = null;

          } catch (InterruptedException  e) {

          }

         

          if (closed || hasError || !dfsClient.clientRunning) {

            continue;

          }

          synchronized (dataQueue) {

            try {

              blockStream.close();

              blockReplyStream.close();

            } catch (IOException e) {

            }

            nodes = null;

            response = null;

            blockStream = null;

            blockReplyStream = null;

          }

        }

       

        if (progress != null) { progress.progress(); }

        // This is used by unit test to trigger race conditions.

        if (artificialSlowdown != 0 && dfsClient.clientRunning) {

          DFSClient.sleepForUnitTest(artificialSlowdown);

        }

      }

    }

    // shutdown thread

    void close() {

      closed = true;

      synchronized (dataQueue) {

        dataQueue.notifyAll();

      }

      synchronized (ackQueue) {

        ackQueue.notifyAll();

      }

      this.interrupt();

    }

  }

  //

  // Processes reponses from the datanodes.  A packet is removed

  // from the ackQueue when its response arrives.

  //

  private class ResponseProcessor extends Thread {

    private volatile boolean closed = false;

    private DatanodeInfo[] targets = null;

    private boolean lastPacketInBlock = false;

    ResponseProcessor (DatanodeInfo[] targets) {

      this.targets = targets;

    }

    public void run() {

      this.setName("ResponseProcessor for block " + block);

      while (!closed && dfsClient.clientRunning && !lastPacketInBlock) {

        // process responses from datanodes.

        int recordError = 0;

        try {

          long seqno = 0;

          synchronized (ackQueue) {

            while (!closed && dfsClient.clientRunning && ackQueue.isEmpty() &&

                   numPendingHeartbeats == 0) {

              try {

                ackQueue.wait();

              } catch (InterruptedException e) {

                // If the thread is being interrupted when waiting for

                // packet, we log the exception and treat it as a normal

                // exception.

                //

                DFSClient.LOG.info("ResponseProcessor thread interrupted when " +

                         "waiting for new packets");

                throw e;

              }

            }

          }

          if (closed || !dfsClient.clientRunning) {

            break;

          }

          eventStartReceiveAck();

          PipelineAck pipelineAck = null;

          if (!doParallelWrites) {

            // verify seqno from datanode

            if (supportClientHeartbeat()) {

              pipelineAck = new PipelineAck();

              pipelineAck.readFields(blockReplyStream.get(0), targets.length,

                  profileData != null);

             

              seqno = pipelineAck.getSeqno();

             

              if (!pipelineAck.isSuccess()) {

                for (int i = 0; i < targets.length && dfsClient.clientRunning; i++) {

                  short reply = pipelineAck.getReply(i);

                  if (reply != DataTransferProtocol.OP_STATUS_SUCCESS) {

                    recordError = i; // first bad datanode

                    throw new IOException("Bad response " + reply + " for block "

                        + block + " from datanode " + targets[i].getName());

                  }

                }               

              }

            } else {

              // Backward compatibility codes.

              seqno = blockReplyStream.get(0).readLong();

              DFSClient.LOG.debug("DFSClient received ack for seqno " + seqno);

              if (seqno == DFSOutputStreamPacket.HEART_BEAT_SEQNO) {

                continue;

              }

              // regular ack

              // processes response status from all datanodes.

              for (int i = 0; i < targets.length && dfsClient.clientRunning; i++) {

                short reply = blockReplyStream.get(0).readShort();

                if (reply != DataTransferProtocol.OP_STATUS_SUCCESS) {

                  recordError = i; // first bad datanode

                  throw new IOException("Bad response " + reply + " for block "

                      + block + " from datanode " + targets[i].getName());

                }

              }

            }

          } else {

            // The client is writing to all replicas in parallel. It also

            // expects an ack from all replicas.

            long lastsn = 0;

            assert blockReplyStream.size() > 0;

            for (int i = 0; i < blockReplyStream.size(); i++) {

              recordError = i; // remember the current slot

              seqno = blockReplyStream.get(i).readLong();

              if (DFSClient.LOG.isDebugEnabled()) {

                DFSClient.LOG.debug("DFSClient for block " + block + " " + seqno);

              }

              if (i != 0 && seqno != -2 && seqno != lastsn) {

                String msg = "Responses from datanodes do not match "

                    + " this replica acked " + seqno

                    + " but previous replica acked " + lastsn;

                DFSClient.LOG.warn(msg);

                throw new IOException(msg);

              }

              short reply = blockReplyStream.get(i).readShort();

              if (reply != DataTransferProtocol.OP_STATUS_SUCCESS) {

                recordError = i; // first bad datanode

                throw new IOException("Bad parallel response " + reply

                    + " for block " + block + " from datanode "

                    + targets[i].getName());

              }

              lastsn = seqno;

            }

          }

          assert seqno != -2 :

            "Ack for unkown seqno should be a failed ack!";

          if (seqno == DFSOutputStreamPacket.HEART_BEAT_SEQNO) {  // a heartbeat ack

            assert supportClientHeartbeat();

            synchronized(ackQueue) {

              assert numPendingHeartbeats > 0;

              numPendingHeartbeats--;

            }

            continue;

          }

          DFSOutputStreamPacket one = null;

          synchronized (ackQueue) {

            assert !ackQueue.isEmpty();

            one = ackQueue.getFirst();

          }

          if (one.seqno != seqno) {

            throw new IOException("Responseprocessor: Expecting seqno " +

                " for block " + block +

                one.seqno + " but received " + seqno);

          }

                   

          lastPacketInBlock = one.lastPacketInBlock;

          if (lastPacketInBlock) {

            if (DFSClient.LOG.isDebugEnabled()) {

              DFSClient.LOG

                  .debug("Update pos in file: " + src + " curBlckOffset: "

                      + lastBlkOffset + " blockSize: "

                      + one.getEndPosInCurrBlk());

            }

            lastBlkOffset += one.getEndPosInCurrBlk();

          }

          synchronized (ackQueue) {

            assert seqno == lastAckedSeqno + 1;

            lastAckedSeqno = seqno;

           

            ackQueue.removeFirst();

            ackQueue.notifyAll();

          }

          one.eventAckReceived();

          if (getProfileData() != null) {

            getProfileData().finishPacket(one.profile, pipelineAck);

            long slowWriteProfileThreshold = options

                .getLogSlowWriteProfileDataThreshold();

            long totalTime = getProfileData().recentPacketProfile.getTotalTime();

            if (slowWriteProfileThreshold > 0

                && totalTime > slowWriteProfileThreshold) {

              DFSClient.LOG.warn("Slow Write Packet for block : " + block +

                  ", packet seqno : " + one.seqno +  ", total time : " +

                  totalTime + " \n" + getProfileData().recentPacketProfile);

            }

          }

        } catch (Exception e) {

          if (!closed) {

            hasError = true;

            errorIndex = recordError;

            if (e instanceof IOException) {

              setLastException((IOException)e);

            }

            DFSClient.LOG.warn("DFSOutputStream ResponseProcessor exception " +

                     " for block " + block +

                      StringUtils.stringifyException(e));

            closed = true;

          }

        }

        synchronized (dataQueue) {

          dataQueue.notifyAll();

        }

        synchronized (ackQueue) {

          ackQueue.notifyAll();

        }

      }

    }

    void close() {

      closed = true;

      this.interrupt();

    }

  }

  // If this stream has encountered any errors so far, shutdown

  // threads and mark stream as closed. Returns true if we should

  // sleep for a while after returning from this call.

  //

  private boolean processDatanodeError(boolean hasError, boolean isAppend) {

    if (!hasError) {

      return false;

    }

    if (response != null) {

      DFSClient.LOG.info("Error Recovery for block " + block +

               " waiting for responder to exit. ");

      return true;

    }

    dfsClient.incWriteExpCntToStats();

    if (errorIndex >= 0) {

      DFSClient.LOG.warn("Error Recovery for block " + block

          + " bad datanode[" + errorIndex + "] "

          + (nodes == null? "nodes == null": nodes[errorIndex].getName()));

    }

    if (blockStream != null) {

      try {

        blockStream.close();

        blockReplyStream.close();

      } catch (IOException e) {

      }

    }

    blockStream = null;

    blockReplyStream = null;

    // move packets from ack queue to front of the data queue

    synchronized (dataQueue) {

      synchronized (ackQueue) {

        if (!ackQueue.isEmpty()) {

          DFSClient.LOG.info("First unacked packet in " + block + " starts at "

              + ackQueue.getFirst().offsetInBlock);

          dataQueue.addAll(0, ackQueue);

          ackQueue.clear();

        }

        numPendingHeartbeats = 0;

      }

    }

    boolean success = false;

    while (!success && dfsClient.clientRunning) {

      DatanodeInfo[] newnodes = null;

      if (nodes == null) {

        String msg = "Could not get block locations. " +

                                        "Source file \"" + src

                                        + "\" - Aborting...";

        DFSClient.LOG.warn(msg);

        setLastException(new IOException(msg));

        closed = true;

        if (streamer != null) streamer.close();

        return false;

      }

      StringBuilder pipelineMsg = new StringBuilder();

      for (int j = 0; j < nodes.length; j++) {

        pipelineMsg.append(nodes[j].getName());

        if (j < nodes.length - 1) {

          pipelineMsg.append(", ");

        }

      }

      // remove bad datanode from list of datanodes.

      // If errorIndex was not set (i.e. appends), then do not remove

      // any datanodes

      //

      if (errorIndex < 0) {

        newnodes = nodes;

      } else {

        if (nodes.length <= 1) {

          lastException = new IOException("All datanodes " + pipelineMsg +

                                          " are bad. Aborting...");

          closed = true;

          if (streamer != null) streamer.close();

          return false;

        }

        DFSClient.LOG.warn("Error Recovery for block " + block +

                 " in pipeline " + pipelineMsg +

                 ": bad datanode " + nodes[errorIndex].getName());

        newnodes =  new DatanodeInfo[nodes.length-1];

        System.arraycopy(nodes, 0, newnodes, 0, errorIndex);

        System.arraycopy(nodes, errorIndex+1, newnodes, errorIndex,

            newnodes.length-errorIndex);

      }

      // Tell the primary datanode to do error recovery

      // by stamping appropriate generation stamps.

      //

      LocatedBlock newBlock = null;

      DatanodeInfo primaryNode = null;

      boolean clientAdRecoveryPrimaryProtocolSupported = false;

      try {

        clientAdRecoveryPrimaryProtocolSupported = dfsClient.namenodeProtocolProxy

            .isMethodSupported("nextGenerationStamp", Block.class,

                boolean.class);

      } catch (InterruptedIOException iie) {

        return false;

      } catch (IOException ioe) {

        DFSClient.LOG.warn(

            "Error when trying to determine whether namenode protocol "

                + "supports client as block recovoery coordinator.", ioe);

      }

      boolean clientAsRecoveryPrimary = dfsClient.conf.getBoolean(

          "dfs.client.as.block.recovery.primary", true)

          && clientAdRecoveryPrimaryProtocolSupported;

      try {

        if (clientAsRecoveryPrimary) {

          BlockRecoveryCoordinator brc = new BlockRecoveryCoordinator(

              DFSClient.LOG, dfsClient.conf, dfsClient.socketTimeout, null,

              new BlockSyncer(dfsClient.getNamespaceId(),

                  dfsClient.getNameNodeRPC(), DFSClient.LOG), null);

          newBlock = brc.recoverBlock(dfsClient.getNamespaceId(), block, false,

              newnodes, false, System.currentTimeMillis() + dfsClient.socketTimeout * 8000);

        } else {

          // Pick the "least" datanode as the primary datanode to avoid

          // deadlock.

          primaryNode = Collections.min(Arrays.asList(newnodes));

          newBlock = recoverBlockFromPrimaryDataNode(primaryNode, newnodes,

              isAppend);

        }

       

        if (newBlock == null) {

          throw new IOException("all datanodes do not have the block");

        }

        boolean isEmpty;

        long nextByteToSend;

        long newBlockSize = newBlock.getBlockSize();

        int numPktRemoved;

        synchronized (dataQueue) {

          numPktRemoved = adjustDataQueueAfterBlockRecovery(newBlockSize);

          isEmpty = dataQueue.isEmpty();

          if (isEmpty) {

            if (currentPacket != null) {

              nextByteToSend = currentPacket.offsetInBlock;

            } else {

              nextByteToSend = bytesCurBlock;

            }

          } else {

            nextByteToSend = dataQueue.getFirst().offsetInBlock;

          }

        }

        if (numPktRemoved > 0) {

          DFSClient.LOG.info("Remove " + numPktRemoved

              + " packets in the packet queue after block recovery");

          if (nextByteToSend > newBlockSize) {

            DFSClient.LOG

                .warn("Missing bytes after removing packets! It should never happen. nextByteToSend "

                    + nextByteToSend + " new block size " + newBlockSize);

          }

        } else if (nextByteToSend > newBlockSize) {

          DFSClient.LOG.warn("Missing bytes! Error Recovery for block " + block

              + " end up with " + newBlock.getBlockSize()

              + " bytes but client already sent " + nextByteToSend

              + " bytes and data queue is " + (isEmpty ? "" : "not ")

              + "empty.");

        } else if (DFSClient.LOG.isDebugEnabled()) {

          DFSClient.LOG.debug("Didn't remove any block. nextByteToSend "

                    + nextByteToSend + " new block size " + newBlockSize);

        }

      } catch (BlockAlreadyCommittedException e) {

        dfsClient.incWriteExpCntToStats();

        DFSClient.LOG

            .warn("Error Recovery for block "

                + block

                + " failed "

                + " because block is already committed according to primary datanode "

                + primaryNode + ". " + " Pipeline was " + pipelineMsg

                + ". Aborting...", e);

        lastException = e;

        closed = true;

        if (streamer != null) streamer.close();

        return false;       // abort with IOexception

      } catch (IOException e) {

        dfsClient.incWriteExpCntToStats();

        DFSClient.LOG.warn("Failed recovery attempt #" + recoveryErrorCount +

            " from primary datanode " + primaryNode, e);

        recoveryErrorCount++;

        // For client as primary, no need to retry as all failures thrown by

        // data nodes are already handled.

        if (clientAsRecoveryPrimary || recoveryErrorCount > maxRecoveryErrorCount) {

          if (!clientAsRecoveryPrimary && nodes.length > 1) {

            // if the primary datanode failed, remove it from the list.

            // The original bad datanode is left in the list because it is

            // conservative to remove only one datanode in one iteration.

            for (int j = 0; j < nodes.length; j++) {

              if (nodes[j].equals(primaryNode)) {

                errorIndex = j; // forget original bad node.

              }

            }

            // remove primary node from list

            newnodes =  new DatanodeInfo[nodes.length-1];

            System.arraycopy(nodes, 0, newnodes, 0, errorIndex);

            System.arraycopy(nodes, errorIndex+1, newnodes, errorIndex,

                             newnodes.length-errorIndex);

            nodes = newnodes;

            DFSClient.LOG.warn("Error Recovery for block " + block + " failed " +

                     " because recovery from primary datanode " +

                     primaryNode + " failed " + recoveryErrorCount +

                     " times. " + " Pipeline was " + pipelineMsg +

                     ". Marking primary datanode as bad.");

            recoveryErrorCount = 0;

            errorIndex = -1;

            return true;          // sleep when we return from here

          }

          String emsg = "Error Recovery for block " + block + " failed " +

                        " because recovery from primary datanode " +

                        primaryNode + " failed " + recoveryErrorCount +

                        " times. "  + " Pipeline was " + pipelineMsg +

                        ". Aborting...";

          DFSClient.LOG.warn(emsg);

          lastException = new IOException(emsg);

          closed = true;

          if (streamer != null) streamer.close();

          return false;       // abort with IOexception

        }

        DFSClient.LOG.warn("Error Recovery for block " + block + " failed " +

                 " because recovery from primary datanode " +

                 primaryNode + " failed " + recoveryErrorCount +

                 " times. "  + " Pipeline was " + pipelineMsg +

                 ". Will retry...");

        return true;          // sleep when we return from here

      } finally {

      }

      recoveryErrorCount = 0; // block recovery successful

      // If the block recovery generated a new generation stamp, use that

      // from now on.  Also, setup new pipeline

      //

      if (newBlock != null) {

        block = newBlock.getBlock();

        nodes = newBlock.getLocations();

      }

      this.hasError = false;

      lastException = null;

      errorIndex = 0;

      success = createBlockOutputStream(nodes, dfsClient.clientName,

          true, false);

    }

    response = new ResponseProcessor(nodes);

    response.start();

    return false; // do not sleep, continue processing

  }

 

  LocatedBlock recoverBlockFromPrimaryDataNode(DatanodeInfo primaryNode,

      DatanodeInfo[] newnodes, boolean isAppend) throws IOException {

    ProtocolProxy<ClientDatanodeProtocol> primary = null;

    try {

    // Copied from org.apache.hadoop.ipc.Client

    int connectTimeout = dfsClient.conf.getInt(

        Client.CONNECT_TIMEOUT_KEY, Client.CONNECT_TIMEOUT_DEFAULT);

    int maxRetries = dfsClient.conf.getInt(

        Client.CONNECT_MAX_RETRIES_KEY, Client.CONNECT_MAX_RETRIES_DEFAULT);

    /*

     * considering pipeline recovery needs 3 RPCs to DataNodes and 2 RPCs to

     * NameNode; So rpcTimeout sets to be 5 times of client socketTimeout.

     * Also each datanode RPC might take upto (connectTimeout * maxRetries)

     * to establish connection.

     */

    int recoverTimeout = 5 * dfsClient.socketTimeout + 3

        * (connectTimeout * maxRetries);

    primary = DFSClient.createClientDNProtocolProxy(primaryNode,

        dfsClient.conf, recoverTimeout);

    try {

      if (primary.isMethodSupported("recoverBlock", int.class, Block.class,

          boolean.class, DatanodeInfo[].class, long.class)) {

        // The deadline is up to RPC time out minus one socket timeout

        // to be more conservative.

        return primary.getProxy().recoverBlock(dfsClient.namespaceId, block,

            isAppend, newnodes,

            System.currentTimeMillis() + recoverTimeout -

            dfsClient.socketTimeout - (maxRetries * connectTimeout));

      } else if (primary.isMethodSupported("recoverBlock", int.class, Block.class, boolean.class, DatanodeInfo[].class)) {

        return primary.getProxy().recoverBlock(

            dfsClient.namespaceId, block, isAppend, newnodes);

      } else {

        return primary.getProxy().recoverBlock(block, isAppend, newnodes);

      }

    } catch (RemoteException re) {

      if (re.unwrapRemoteException() instanceof BlockAlreadyCommittedException) {

        throw new BlockAlreadyCommittedException(re);

      } else {

        throw re;

      }

    }

    } finally {

      if (primary != null) {

        RPC.stopProxy(primary.getProxy());

      }

    }

  }

 

 

  private int adjustDataQueueAfterBlockRecovery(long newBlockSize) {

    // New block size should be one of the packet's ending position

    // If the block offset of the first packet is not the new block

    // size, we should be able to remove several packets in packet

    // queue and make sure the first packet is the new block size.

    // Otherwise, something went wrong.

    //

    // We are conservative here: if the first unacked packet starts

    // with a full chunk, it can always be a clean checkpoint. We

    // keep the packets starting from it.

    //

    int bytesPerChecksum = checksum.getBytesPerChecksum();

    int numPktRemoved = 0;

    long newAckedSeqno = -1;

    while (!dataQueue.isEmpty()) {

      DFSOutputStreamPacket first = dataQueue.getFirst();

      long endOffsetOfBlock = first.getEndPosInCurrBlk();

      if (first.isHeartbeatPacket()) {

        dataQueue.removeFirst();

        numPktRemoved++;

      } else if (first.offsetInBlock % bytesPerChecksum == 0) {

        // The first unacked packet starts with a full chunk.

        //

        break;

      } else if (endOffsetOfBlock <= newBlockSize) {

        if (first.lastPacketInBlock) {

          // Last block is already acked in all remaining replicas

          // Resend an empty one to force the stream to finish.

          //

          if (endOffsetOfBlock != newBlockSize) {

            DFSClient.LOG.warn("Packet is the last packet in block with "

                + endOffsetOfBlock

                + " but new block length after block recovery is "

                + newBlockSize + ". Something went wrong.");

          }

         

          first.cleanup();

          first.offsetInBlock = endOffsetOfBlock;

          DFSClient.LOG

              .info("Resend last packet in block and make it empty, new offsetInBlock "

                  + endOffsetOfBlock);

          break;

        } else {

          dataQueue.removeFirst();

          numPktRemoved++;

          if (first.seqno > lastAckedSeqno) {

            newAckedSeqno = first.seqno;

          }

        }

      } else {

        if (first.offsetInBlock != newBlockSize) {

          DFSClient.LOG.warn("Packet has start offset " + first.offsetInBlock

              + " and end offset " + endOffsetOfBlock

              + " but new block length after block recovery is " + newBlockSize

              + ". Something went wrong.");         

        }

        break;

      }

    }

   

    if (numPktRemoved > 0 && newAckedSeqno != -1) {

      synchronized (ackQueue) {

        lastAckedSeqno = newAckedSeqno;

        ackQueue.notifyAll();

      }

    }

    return numPktRemoved;

  }

  private void isClosed() throws IOException {

    if ((closed || !dfsClient.clientRunning) && lastException != null) {

        throw lastException;

    }

  }

  //

  // returns the list of targets, if any, that is being currently used.

  //

  DatanodeInfo[] getPipeline() {

    synchronized (dataQueue) {

      if (nodes == null) {

        return null;

      }

      DatanodeInfo[] value = new DatanodeInfo[nodes.length];

      for (int i = 0; i < nodes.length; i++) {

        value[i] = nodes[i];

      }

      return value;

    }

  }

 

  static private DFSWriteProfilingData getProfile(DFSClient dfsClient) {

    DFSWriteProfilingData profile = DFSClient.getAndResetProfileDataForNextOutputStream();

    if (dfsClient != null) {

      boolean ifAutoPrint = dfsClient.conf.getBoolean(

          FSConstants.FS_OUTPUT_STREAM_AUTO_PRINT_PROFILE, false);

      if (ifAutoPrint) {

        if (profile == null) {

          profile = new DFSWriteProfilingData();

        }

        profile.setAutoPrintWhileClose(true);

      }

    }

    return profile;