* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* A Map-reduce program to recursively copy directories between

* different file-systems.

*/

  public static final Log LOG = LogFactory.getLog(DistCp.class);

  private static final String NAME = "distcp";

  private static final String usage = NAME

    + " [OPTIONS] <srcurl>* <desturl>" +

    "\n\nOPTIONS:" +

    "\n-p[rbugp]              Preserve status" +

    "\n                       r: replication number" +

    "\n                       b: block size" +

    "\n                       u: user" +

    "\n                       g: group" +

    "\n                       p: permission" +

    "\n                       t: modification and access times" +

    "\n                       -p alone is equivalent to -prbugpt" +

    "\n-i                     Ignore failures" +

    "\n-basedir <basedir>     Use <basedir> as the base directory when copying files from <srcurl>" +

    "\n-log <logdir>          Write logs to <logdir>" +

    "\n-m <num_maps>          Maximum number of simultaneous copies" +

    "\n-r <num_reducers>      Maximum number of reducers" +

    "\n-overwrite             Overwrite destination" +

    "\n-update                Overwrite if src size different from dst size" +

    "\n-skipcrccheck          Do not use CRC check to determine if src is " +

    "\n                       different from dest. Relevant only if -update" +

    "\n                       is specified" +

    "\n-copybychunk           Chop files in chunk and copy" +

    "\n-f <urilist_uri>       Use list at <urilist_uri> as src list" +

    "\n-filelimit <n>         Limit the total number of files to be <= n" +

    "\n-sizelimit <n>         Limit the total size to be <= n bytes" +

    "\n-delete                Delete the files existing in the dst but not in src" +

    "\n-mapredSslConf <f>     Filename of SSL configuration for mapper task" +

    "\n-usefastcopy           Use FastCopy (applicable to DFS only)" +

   

    "\n\nNOTE 1: if -overwrite or -update are set, each source URI is " +

    "\n      interpreted as an isomorphic update to an existing directory." +

    "\nFor example:" +

    "\nhadoop " + NAME + " -p -update \"hdfs://A:8020/user/foo/bar\" " +

    "\"hdfs://B:8020/user/foo/baz\"\n" +

    "\n     would update all descendants of 'baz' also in 'bar'; it would " +

    "\n     *not* update /user/foo/baz/bar" +

    "\n\nNOTE 2: The parameter <n> in -filelimit and -sizelimit can be " +

    "\n     specified with symbolic representation.  For examples," +

    "\n       1230k = 1230 * 1024 = 1259520" +

    "\n       891g = 891 * 1024^3 = 956703965184" +

   

    "\n";

 

  private static final long BYTES_PER_MAP =  256 * 1024 * 1024;

  private static final int MAX_MAPS_PER_NODE = 20;

  private static final int MAX_MAPS_DEFAULT = 4000;

  private static final int SYNC_FILE_MAX = 10;

  static enum Counter {

    COPY, SKIP, FAIL, BYTESCOPIED, BYTESEXPECTED, BLOCKSCOPIED

  }

  public static enum Options {

    DELETE("-delete", NAME + ".delete"),

    FILE_LIMIT("-filelimit", NAME + ".limit.file"),

    SIZE_LIMIT("-sizelimit", NAME + ".limit.size"),

    IGNORE_READ_FAILURES("-i", NAME + ".ignore.read.failures"),

    PRESERVE_STATUS("-p", NAME + ".preserve.status"),

    OVERWRITE("-overwrite", NAME + ".overwrite.always"),

    UPDATE("-update", NAME + ".overwrite.ifnewer"),

    SKIPCRC("-skipcrccheck", NAME + ".skip.crc.check"),

    COPYBYCHUNK("-copybychunk", NAME + ".copy.by.chunk"),

    USEFASTCOPY("-usefastcopy", NAME + ".use.fastcopy");

    final String cmd, propertyname;

    private Options(String cmd, String propertyname) {

      this.cmd = cmd;

      this.propertyname = propertyname;

    }

   

    private long parseLong(String[] args, int offset) {

      if (offset ==  args.length) {

        throw new IllegalArgumentException("<n> not specified in " + cmd);

      }

      long n = StringUtils.TraditionalBinaryPrefix.string2long(args[offset]);

      if (n <= 0) {

        throw new IllegalArgumentException("n = " + n + " <= 0 in " + cmd);

      }

      return n;

    }

  }

  static enum FileAttribute {

    BLOCK_SIZE, REPLICATION, USER, GROUP, PERMISSION, TIMES;

    final char symbol;

    private FileAttribute() {symbol = toString().toLowerCase().charAt(0);}

   

    static EnumSet<FileAttribute> parse(String s) {

      if (s == null || s.length() == 0) {

        return EnumSet.allOf(FileAttribute.class);

      }

      EnumSet<FileAttribute> set = EnumSet.noneOf(FileAttribute.class);

      FileAttribute[] attributes = values();

      for(char c : s.toCharArray()) {

        int i = 0;

        for(; i < attributes.length && c != attributes[i].symbol; i++);

        if (i < attributes.length) {

          if (!set.contains(attributes[i])) {

            set.add(attributes[i]);

          } else {

            throw new IllegalArgumentException("There are more than one '"

                + attributes[i].symbol + "' in " + s);

          }

        } else {

          throw new IllegalArgumentException("'" + c + "' in " + s

              + " is undefined.");

        }

      }

      return set;

    }

  }

  static final String TMP_DIR_LABEL = NAME + ".tmp.dir";

  static final String DST_DIR_LABEL = NAME + ".dest.path";

  static final String JOB_DIR_LABEL = NAME + ".job.dir";

  static final String MAX_MAPS_LABEL = NAME + ".max.map.tasks";

  static final String MAX_REDUCE_LABEL = NAME + ".max.reduce.tasks";

  static final String SRC_LIST_LABEL = NAME + ".src.list";

  static final String SRC_COUNT_LABEL = NAME + ".src.count";

  static final String TOTAL_SIZE_LABEL = NAME + ".total.size";

  static final String TOTAL_BLOCKS_LABEL = NAME + ".total.blocks";

  static final String DST_DIR_LIST_LABEL = NAME + ".dst.dir.list";

  static final String DST_CHUNK_FILE_LIST_LABEL = NAME + ".dst.chunk.file.list";

  static final String SPLIT_LIST_LABEL = NAME + ".file.split.list";

  public static final String BYTES_PER_MAP_LABEL = NAME + ".bytes.per.map";

  static final String PRESERVE_STATUS_LABEL

      = Options.PRESERVE_STATUS.propertyname + ".value";

  private JobConf conf;

  public void setConf(Configuration conf) {

    if (conf instanceof JobConf) {

      this.conf = (JobConf) conf;

    } else {

      this.conf = new JobConf(conf);

    }

  }

  public Configuration getConf() {

    return conf;

  }

  public DistCp(Configuration conf) {

    setConf(conf);

  }

  /**

   * An input/output pair of filenames.

   * offset and length field are used for chopping files into chunk files

   * and copying file chunks instead of one big file

   * offset tells where is the starting point of the chuck file in the

   * original file

   * length tells the length of the chuck file

   * If the FilePair are directories or we don't want to copy file by chunks,

   * both offset and length will be set to 0

   * Also we use the index to tell the order of the chunks

   *

   */

  static class FileChunkPair implements Writable {

    FileStatus input = new FileStatus();

    String output;

    long offset;

    long length;

    int chunkIndex;

    FileChunkPair() { }

    FileChunkPair(FileStatus input, String output, long offset,

        long length, int chunkIndex) {

      this.input = input;

      this.output = output;

      this.offset = offset;

      this.length = length;

      this.chunkIndex = chunkIndex;

    }

    public void readFields(DataInput in) throws IOException {

      input.readFields(in);

      output = Text.readString(in);

      offset = in.readLong();

      length = in.readLong();

      chunkIndex = in.readInt();

    }

    public void write(DataOutput out) throws IOException {

      input.write(out);

      Text.writeString(out, output);

      out.writeLong(offset);

      out.writeLong(length);

      out.writeInt(chunkIndex);

    }

    public String toString() {

      return input + " : " + output + " : " + offset + " : " + length 

      + " : " + chunkIndex ;

    }

  }

 

  /**

   * An input/output pair of filenames.

   */ 

  static class FilePairComparable implements WritableComparable {

    FileStatus input = new FileStatus();

    String output;

    FilePairComparable() { }

    FilePairComparable(FileStatus input, String output) {

      this.input = input;

      this.output = output;

    }

    public int compareTo(Object otherObj) {

      FilePairComparable other = (FilePairComparable) otherObj;

      return this.input.compareTo(other.input);

    }

    public void readFields(DataInput in) throws IOException {

      input.readFields(in);

      output = Text.readString(in);

    }

    public void write(DataOutput out) throws IOException {

      input.write(out);

      Text.writeString(out, output);

    }

    public String toString() {

      return input + " : " + output;

    }

  }

  /**

   * Computes the total number of blocks in a file.

   *

   * @param len

   *          the length of the file.

   * @param blockSize

   *          the block size for the file.

   * @return the total number of blocks.

   */

  private static long getBlocks(long len, long blockSize) {

    return (len == 0 || blockSize == 0) ? 0 : (len - 1) / blockSize + 1;

  }

  /**

   * InputFormat of a distcp job responsible for generating splits of the src

   * file list when FastCopy is being used for copying.

   */

  static class FastCopyInputFormat extends CopyInputFormat {

    @Override

    protected long getIncrement(LongWritable key, FilePairComparable value) {

      return getBlocks(value.input.getLen(), value.input.getBlockSize());

    }

    @Override

    protected long getTargetSize(JobConf job, int numSplits) {

      long totalBlocks = job.getLong(TOTAL_BLOCKS_LABEL, -1);

      return totalBlocks / numSplits;

    }

  }

 

  /**

   * InputFormat of a distcp job responsible for generating splits of the src

   * file list.

   */

  static class CopyInputFormat implements InputFormat<Text, Text> {

    protected long getIncrement(LongWritable key, FilePairComparable value) {

      return key.get();

    }

    protected long getTargetSize(JobConf job, int numSplits) {

      long cbsize = job.getLong(TOTAL_SIZE_LABEL, -1);

      return cbsize / numSplits;

    }

    /**

     * Produce splits such that each is no greater than the quotient of the

     * total size and the number of splits requested.

     * @param job The handle to the JobConf object

     * @param numSplits Number of splits requested

     */

    public InputSplit[] getSplits(JobConf job, int numSplits)

        throws IOException {

      int cnfiles = job.getInt(SRC_COUNT_LABEL, -1);

      long cbsize = job.getLong(TOTAL_SIZE_LABEL, -1);

      long blocks = job.getLong(TOTAL_BLOCKS_LABEL, -1);

      String srcfilelist = job.get(SRC_LIST_LABEL, "");

      if (cnfiles < 0 || cbsize < 0 || blocks < 0 || "".equals(srcfilelist)) {

        throw new RuntimeException("Invalid metadata: #files(" + cnfiles +

                                   ") total_size(" + cbsize + ") listuri(" +

                                   srcfilelist + ")");

      }

      Path src = new Path(srcfilelist);

      FileSystem fs = src.getFileSystem(job);

      FileStatus srcst = fs.getFileStatus(src);

      ArrayList<FileSplit> splits = new ArrayList<FileSplit>(numSplits);

      LongWritable key = new LongWritable();

      FilePairComparable value = new FilePairComparable();

      final long targetsize = getTargetSize(job, numSplits);

      long pos = 0L;

      long last = 0L;

      long acc = 0L;

      long cbrem = srcst.getLen();

      SequenceFile.Reader sl = null;

      try {

        sl = new SequenceFile.Reader(fs, src, job);

        for (; sl.next(key, value); last = sl.getPosition()) {

          // if adding this split would put this split past the target size,

          // cut the last split and put this next file in the next split.

          long increment = getIncrement(key, value);

          if (acc + increment > targetsize && acc != 0) {

            long splitsize = last - pos;

            splits.add(new FileSplit(src, pos, splitsize, (String[])null));

            cbrem -= splitsize;

            pos = last;

            acc = 0L;

          }

          acc += increment;

        }

      }

      finally {

        checkAndClose(sl);

      }

      if (cbrem != 0) {

        splits.add(new FileSplit(src, pos, cbrem, (String[])null));

      }

      return splits.toArray(new FileSplit[splits.size()]);

    }

    /**

     * Returns a reader for this split of the src file list.

     */

    public RecordReader<Text, Text> getRecordReader(InputSplit split,

        JobConf job, Reporter reporter) throws IOException {

      return new SequenceFileRecordReader<Text, Text>(job, (FileSplit)split);

    }

  }

 

  /**

   * InputFormat of a distcp job responsible for generating splits from the

   * split list we create during the setup phase.

   *

   */

  static class CopyByChunkInputFormat extends CopyInputFormat

  implements InputFormat<Text, Text> {

    /**

     * Produce splits such that each is no greater than the quotient of the

     * total size and the number of splits requested.

     * @param job The handle to the JobConf object

     * @param numSplits Number of splits requested

     */

    public InputSplit[] getSplits(JobConf job, int numSplits)

        throws IOException {

      int cnfiles = job.getInt(SRC_COUNT_LABEL, -1);

      long cbsize = job.getLong(TOTAL_SIZE_LABEL, -1);

      String srcFileList = job.get(SRC_LIST_LABEL, "");

      Path srcFileListPath = new Path(srcFileList);

      if (cnfiles < 0 || cbsize < 0 || "".equals(srcFileList)) {

        throw new RuntimeException("Invalid metadata: #files(" + cnfiles +

            ") total_size(" + cbsize + ") src_chunk_file_list_uri(" +

            srcFileList + ")");

      }

      ArrayList<FileSplit> splits = new ArrayList<FileSplit>(numSplits);

      SequenceFile.Reader sl = null;

      String splitList = job.get(SPLIT_LIST_LABEL, "");

      if("".equals(splitList)) {

        throw new RuntimeException("Invalid metadata: split_list_uri(" +

            srcFileList + ")");

      }

      //split file list which contains start pos and split length pairs

      //they are used to split srcChunkFileList

      Path splitListPath = new Path(splitList);       

      FileSystem splitListFs = splitListPath.getFileSystem(job);

      try{

        sl = new SequenceFile.Reader(splitListFs, splitListPath, job);

        LongWritable startpos = new LongWritable();

        LongWritable length = new LongWritable();

        while (sl.next(startpos, length)) {

          splits.add(new FileSplit(srcFileListPath, startpos.get(),

              length.get(), (String[])null));

        }

      }

      finally{

        checkAndClose(sl);

      }

      return splits.toArray(new FileSplit[splits.size()]);

    }

  }

  /**

   * FSCopyFilesTask: The mapper for copying files between FileSystems.

   */

  static class CopyFilesTask

      implements Mapper<LongWritable, FilePairComparable, FilePairComparable, Text>,

                 Reducer<FilePairComparable, Text, FilePairComparable, Text> {

    // config

    private int sizeBuf = 128 * 1024;

    private FileSystem destFileSys = null;

    private boolean ignoreReadFailures;

    private boolean preserve_status;

    private EnumSet<FileAttribute> preserved;

    private boolean overwrite;

    private boolean update;

    private Path destPath = null;

    private Path tmpRoot = null;

    private Path attemptTmpRoot = null;

    private byte[] buffer = null;

    private JobConf job;

    private boolean skipCRCCheck = false;

    private boolean useFastCopy = false;

   

    private FastCopy fc = null;

    // stats

    private int failcount = 0;

    private int skipcount = 0;

    private int copycount = 0;

    private String getCountString() {

      return "Copied: " + copycount + " Skipped: " + skipcount

      + " Failed: " + failcount;

    }

    private void updateStatus(Reporter reporter) {

      reporter.setStatus(getCountString());

    }

    /**

     * Return true if dst should be replaced by src and the update flag is set.

     * Right now, this merely checks that the src and dst len are not equal.

     * This should be improved on once modification times, CRCs, etc. can

     * be meaningful in this context.

     * @throws IOException

     */

    private boolean needsUpdate(FileStatus srcstatus,

        FileSystem dstfs, Path dstpath) throws IOException {

      return update && !sameFile(srcstatus.getPath().getFileSystem(job),

          srcstatus, dstfs, dstpath, skipCRCCheck);

    }

    private FSDataOutputStream create(Path f, Reporter reporter,

        FileStatus srcstat) throws IOException {

      if (destFileSys.exists(f)) {

        destFileSys.delete(f, false);

      }

      if (!preserve_status) {

        return destFileSys.create(f, true, sizeBuf, reporter);

      }

      FsPermission permission = preserved.contains(FileAttribute.PERMISSION)?

          srcstat.getPermission(): null;

      short replication = preserved.contains(FileAttribute.REPLICATION)?

          srcstat.getReplication(): destFileSys.getDefaultReplication();

      long blockSize = preserved.contains(FileAttribute.BLOCK_SIZE)?

          srcstat.getBlockSize(): destFileSys.getDefaultBlockSize();

      return destFileSys.create(f, permission, true, sizeBuf, replication,

          blockSize, reporter);

    }

    /**

     * Copy a file to a destination without breaking file into chunks

     * @param filePair the pair of source and dest

     * @param outc map output collector

     * @param reporter

     */

    private void copy(FilePairComparable filePair,

        OutputCollector<FilePairComparable, Text> outc, Reporter reporter)

        throws IOException {

      FileStatus srcstat = filePair.input;

      Path relativedst = new Path(filePair.output);

      Path absdst = new Path(destPath, relativedst);

      Path tmpFile = new Path(attemptTmpRoot, relativedst);

      Path finalTmpFile = new Path(tmpRoot, relativedst);

      int totfiles = job.getInt(SRC_COUNT_LABEL, -1);

      assert totfiles >= 0 : "Invalid file count " + totfiles;

      // if a directory, ensure created even if empty

      if (srcstat.isDir()) {

        if (destFileSys.exists(absdst)) {

          if (!destFileSys.getFileStatus(absdst).isDir()) {

            throw new IOException("Failed to mkdirs: " + absdst+" is a file.");

          }

        }

        else if (!destFileSys.mkdirs(absdst)) {

          throw new IOException("Failed to mkdirs " + absdst);

        }

        // TODO: when modification times can be set, directories should be

        // emitted to reducers so they might be preserved. Also, mkdirs does

        // not currently return an error when the directory already exists;

        // if this changes, all directory work might as well be done in reduce

        return;

      }

      if ((destFileSys.exists(absdst) && !overwrite

          && !needsUpdate(srcstat, destFileSys, absdst)) ||

            destFileSys.exists(finalTmpFile)) {

        outc.collect(filePair, new Text("SKIP: " + srcstat.getPath()));

        ++skipcount;

        reporter.incrCounter(Counter.SKIP, 1);

        updateStatus(reporter);

        return;

      }

     

      FileSystem srcFileSys = srcstat.getPath().getFileSystem(job);

      reporter.incrCounter(Counter.BYTESEXPECTED, srcstat.getLen());

      if (useFastCopy) {

        try {

          if (fc == null) {

            throw new IOException("FastCopy object has not been instantiated.");

          }

          LOG.info("Use FastCopy to copy File from " + srcstat.getPath() +" to " + tmpFile);