List of copyFromLocalFile() Examples

Examples of copyFromLocalFile()

org.apache.hadoop.fs.FileSystem.copyFromLocalFile()
The src file is on the local disk. Add it to FS at the given dst name and the source is kept intact afterwards

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

    private void copy(File result, String destination) throws IOException {
        assert result != null;
        assert destination != null;
        Path target = new Path(destination);
        FileSystem fs = target.getFileSystem(configuration);
        fs.copyFromLocalFile(new Path(result.toURI()), target);
    }


    private void delete(File target) {
        assert target != null;
        if (target.isDirectory()) {

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

      // use jar name if job is not named. 
      if ("".equals(job.getJobName())){
        job.setJobName(new Path(originalJarPath).getName());
      }
      job.setJar(submitJarFile.toString());
      fs.copyFromLocalFile(new Path(originalJarPath), submitJarFile);
      fs.setReplication(submitJarFile, replication);
      fs.setPermission(submitJarFile, new FsPermission(JOB_FILE_PERMISSION));
    } else {
      LOG.warn("No job jar file set.  User classes may not be found. "+
               "See JobConf(Class) or JobConf#setJar(String).");

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

            if (outputSpecs != null) {
                for (int i=1; i < outputSpecs.size(); ++i) {
                    String fileName = outputSpecs.get(i).getName();
                    try {
                        int partition = job.getInt("mapred.task.partition", -1);
                        fs.copyFromLocalFile(false, true, new Path(fileName), 
                                             new Path(
                                                     new Path(scriptOutputDir, 
                                                              fileName), 
                                                     getOutputName(partition))
                                            );

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

                // DistributedCache
                if (shipToCluster) {
                    Path dst = 
                        new Path(FileLocalizer.getTemporaryPath(pigContext).toString());
                    FileSystem fs = dst.getFileSystem(conf);
                    fs.copyFromLocalFile(src, dst);
                    
                    // Construct the dst#srcName uri for DistributedCache
                    URI dstURI = null;
                    try {
                        dstURI = new URI(dst.toString() + "#" + src.getName());

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()


    File jarFile1 = buildCoprocessorJar(cpName1);
    File jarFile2 = buildCoprocessorJar(cpName2);


    // copy the jars into dfs
    fs.copyFromLocalFile(new Path(jarFile1.getPath()),
      new Path(fs.getUri().toString() + Path.SEPARATOR));
    String jarFileOnHDFS1 = fs.getUri().toString() + Path.SEPARATOR +
      jarFile1.getName();
    Path pathOnHDFS1 = new Path(jarFileOnHDFS1);
    assertTrue("Copy jar file to HDFS failed.",

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

    Path pathOnHDFS1 = new Path(jarFileOnHDFS1);
    assertTrue("Copy jar file to HDFS failed.",
      fs.exists(pathOnHDFS1));
    LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS1);


    fs.copyFromLocalFile(new Path(jarFile2.getPath()),
        new Path(fs.getUri().toString() + Path.SEPARATOR));
    String jarFileOnHDFS2 = fs.getUri().toString() + Path.SEPARATOR +
      jarFile2.getName();
    Path pathOnHDFS2 = new Path(jarFileOnHDFS2);
    assertTrue("Copy jar file to HDFS failed.",

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

    out.close();
    stream.close();
    LOG.info("Adding jar file to outer jar file completed");


    // copy the jars into dfs
    fs.copyFromLocalFile(new Path(outerJarFile.getPath()),
      new Path(fs.getUri().toString() + Path.SEPARATOR));
    String jarFileOnHDFS = fs.getUri().toString() + Path.SEPARATOR +
      outerJarFile.getName();
    assertTrue("Copy jar file to HDFS failed.",
      fs.exists(new Path(jarFileOnHDFS)));

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

                // DistributedCache
                if (shipToCluster) {
                    Path dst = 
                        new Path(FileLocalizer.getTemporaryPath(null, pigContext).toString());
                    FileSystem fs = dst.getFileSystem(conf);
                    fs.copyFromLocalFile(src, dst);
                    
                    // Construct the dst#srcName uri for DistributedCache
                    URI dstURI = null;
                    try {
                        dstURI = new URI(dst.toString() + "#" + src.getName());

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

                                     LauncherSecurityManager.class, LauncherException.class, LauncherMainTester.class);


        FileSystem fs = getFileSystem();


        Path launcherJar = new Path(actionDir, "launcher.jar");
        fs.copyFromLocalFile(new Path(jar.toString()), launcherJar);


        JobConf jobConf = Services.get().get(HadoopAccessorService.class).
            createJobConf(new URI(getNameNodeUri()).getAuthority());
//        jobConf.setJar(jar.getAbsolutePath());
        jobConf.set("user.name", getTestUser());

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

                    if (errorMessage == null) {
                        File outputData = new File(System.getProperty("oozie.action.output.properties"));
                        FileSystem fs = FileSystem.get(getJobConf());
                        if (outputData.exists()) {


                            fs.copyFromLocalFile(new Path(outputData.toString()), new Path(actionDir,
                                                                                           ACTION_OUTPUT_PROPS));
                            reporter.incrCounter(COUNTER_GROUP, COUNTER_OUTPUT_DATA, 1);


                            int maxOutputData = getJobConf().getInt(CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, 2 * 1024);
                            if (outputData.length() > maxOutputData) {

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.