Examples of DistCp

org.apache.hadoop.tools.DistCp
DistCp is the main driver-class for DistCpV2. For command-line use, DistCp::main() orchestrates the parsing of command-line parameters and the launch of the DistCp job. For programmatic use, a DistCp object can be constructed by specifying options (in a DistCpOptions object), and DistCp::execute() may be used to launch the copy-job. DistCp may alternatively be sub-classed to fine-tune behaviour.
org.springframework.data.hadoop.fs.DistCp
p.apache.org/common/docs/current/distcp.html">distcp as an embeddable API. Due to the number of options available in DistCp, one can either specify them in a command-like style (through one or multiple {@link String}s) through {@link #copy(String)} or specify individual arguments through the rest of the methods. @author Costin Leau @author Thomas Risberg

Examples of org.apache.hadoop.tools.DistCp

      if (namenode.startsWith("hdfs://")) {
        
        FileSystem fs = FileSystem.get(URI.create(namenode), new Configuration());
        fs.mkdirs(new Path("/empty"));


        ToolRunner.run(new DistCp(conf), new String[] {
                                         "-log",
                                         namenode+"/logs",
                                         namenode+"/empty",
                                         namenode+"/dest"});
        fs = FileSystem.get(URI.create(namenode+"/destdat"), conf);

View Full Code Here

Examples of org.apache.hadoop.tools.DistCp

      cluster = new MiniDFSCluster(conf, 1, true, null);
      final FileSystem hdfs = cluster.getFileSystem();
      final String namenode = hdfs.getUri().toString();
      if (namenode.startsWith("hdfs://")) {
        MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat");
        ToolRunner.run(new DistCp(conf), new String[] {
                                         "-log",
                                         namenode+"/logs",
                                         "file:///"+TEST_ROOT_DIR+"/srcdat",
                                         namenode+"/destdat"});
        assertTrue("Source and destination directories do not match.",

View Full Code Here

Examples of org.apache.hadoop.tools.DistCp

      cluster = new MiniDFSCluster(conf, 1, true, null);
      final FileSystem hdfs = cluster.getFileSystem();
      final String namenode = FileSystem.getDefaultUri(conf).toString();
      if (namenode.startsWith("hdfs://")) {
        MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
        ToolRunner.run(new DistCp(conf), new String[] {
                                         "-log",
                                         "/logs",
                                         namenode+"/srcdat",
                                         "file:///"+TEST_ROOT_DIR+"/destdat"});
        assertTrue("Source and destination directories do not match.",

View Full Code Here

Examples of org.apache.hadoop.tools.DistCp

      cluster = new MiniDFSCluster(conf, 2, true, null);
      final FileSystem hdfs = cluster.getFileSystem();
      final String namenode = hdfs.getUri().toString();
      if (namenode.startsWith("hdfs://")) {
        MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
        ToolRunner.run(new DistCp(conf), new String[] {
                                         "-p",
                                         "-log",
                                         namenode+"/logs",
                                         namenode+"/srcdat",
                                         namenode+"/destdat"});
        assertTrue("Source and destination directories do not match.",
                   checkFiles(hdfs, "/destdat", files));
        FileSystem fs = FileSystem.get(URI.create(namenode+"/logs"), conf);
        assertTrue("Log directory does not exist.",
                    fs.exists(new Path(namenode+"/logs")));


        FileStatus[] dchkpoint = getFileStatus(hdfs, "/destdat", files);
        final int nupdate = NFILES>>2;
        updateFiles(cluster.getFileSystem(), "/srcdat", files, nupdate);
        deldir(hdfs, "/logs");


        ToolRunner.run(new DistCp(conf), new String[] {
                                         "-prbugp", // no t to avoid preserving mod. times
                                         "-update",
                                         "-log",
                                         namenode+"/logs",
                                         namenode+"/srcdat",
                                         namenode+"/destdat"});
        assertTrue("Source and destination directories do not match.",
                   checkFiles(hdfs, "/destdat", files));
        assertTrue("Update failed to replicate all changes in src",
                 checkUpdate(hdfs, dchkpoint, "/destdat", files, nupdate));


        deldir(hdfs, "/logs");
        ToolRunner.run(new DistCp(conf), new String[] {
                                         "-prbugp", // no t to avoid preserving mod. times
                                         "-overwrite",
                                         "-log",
                                         namenode+"/logs",
                                         namenode+"/srcdat",

View Full Code Here

Examples of org.apache.hadoop.tools.DistCp

        out = fs.create(destPath, true);
        out.writeUTF(destData);
        out.close();
        
        // Run with -skipcrccheck option
        ToolRunner.run(new DistCp(conf), new String[] {
          "-p",
          "-update",
          "-skipcrccheck",
          "-log",
          namenode+"/logs",
          namenode+"/srcdat",
          namenode+"/destdat"});
        
        // File should not be overwritten
        FSDataInputStream in = hdfs.open(destPath);
        String s = in.readUTF();
        System.out.println("Dest had: " + s);
        assertTrue("Dest got over written even with skip crc",
            s.equalsIgnoreCase(destData));
        in.close();
        
        deldir(hdfs, "/logs");


        // Run without the option        
        ToolRunner.run(new DistCp(conf), new String[] {
          "-p",
          "-update",
          "-log",
          namenode+"/logs",
          namenode+"/srcdat",

View Full Code Here

Examples of org.apache.hadoop.tools.DistCp

    exec("addsplits row5", true);
    exec("config -t t -s table.split.threshold=345M", true);
    exec("offline t", true);
    String export = folder.newFolder().toString();
    exec("exporttable -t t " + export, true);
    DistCp cp = newDistCp();
    String import_ = folder.newFolder().toString();
    cp.run(new String[] {"-f", export + "/distcp.txt", import_});
    exec("importtable t2 " + import_, true);
    exec("config -t t2 -np", true, "345M", true);
    exec("getsplits -t t2", true, "row5", true);
    exec("constraint --list -t t2", true, "VisibilityConstraint=1", true);
    exec("onlinetable t", true);

View Full Code Here

Examples of org.apache.hadoop.tools.DistCp

    exec("addsplits row5", true);
    exec("config -t " + table + " -s table.split.threshold=345M", true);
    exec("offline " + table, true);
    String export = folder.newFolder().toString();
    exec("exporttable -t " + table + " " + export, true);
    DistCp cp = newDistCp();
    String import_ = folder.newFolder().toString();
    cp.run(new String[] {"-f", export + "/distcp.txt", import_});
    exec("importtable " + table2 + " " + import_, true);
    exec("config -t " + table2 + " -np", true, "345M", true);
    exec("getsplits -t " + table2, true, "row5", true);
    exec("constraint --list -t " + table2, true, "VisibilityConstraint=1", true);
    exec("onlinetable " + table, true);

View Full Code Here

Examples of org.apache.hadoop.tools.DistCp


      final Configuration dfsConf = new Configuration();
      cluster = new MiniDFSCluster(dfsConf, 2, true, null);
      cluster.waitActive();


      final DistCp distcp = new DistCp(dfsConf);
      final FileSystem localfs = FileSystem.get(LOCAL_FS, dfsConf);
      final FileSystem hdfs = cluster.getFileSystem();
      final Configuration proxyConf = new Configuration(false);
      proxyConf.set("hdfsproxy.dfs.namenode.address", hdfs.getUri().getHost() + ":"
          + hdfs.getUri().getPort());

View Full Code Here

Examples of org.apache.hadoop.tools.DistCp

  /** copy files from local file system to local file system */
  public void testCopyFromLocalToLocal() throws Exception {
    Configuration conf = new Configuration();
    FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
    MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat");
    ToolRunner.run(new DistCp(new Configuration()),
                           new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                                         "file:///"+TEST_ROOT_DIR+"/destdat"});
    assertTrue("Source and destination directories do not match.",
               checkFiles(localfs, TEST_ROOT_DIR+"/destdat", files));
    deldir(localfs, TEST_ROOT_DIR+"/destdat");

View Full Code Here

Examples of org.apache.hadoop.tools.DistCp

      cluster = new MiniDFSCluster(conf, 2, true, null);
      final FileSystem hdfs = cluster.getFileSystem();
      namenode = FileSystem.getDefaultUri(conf).toString();
      if (namenode.startsWith("hdfs://")) {
        MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
        ToolRunner.run(new DistCp(conf), new String[] {
                                         "-log",
                                         namenode+"/logs",
                                         namenode+"/srcdat",
                                         namenode+"/destdat"});
        assertTrue("Source and destination directories do not match.",

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.