List of copyFromLocalFile() Examples

Examples of copyFromLocalFile()

org.apache.hadoop.fs.FileSystem.copyFromLocalFile()
The src file is on the local disk. Add it to FS at the given dst name and the source is kept intact afterwards

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

                            props.load(new FileReader(newId));
                            if (props.getProperty("id") == null) {
                                throw new IllegalStateException("ID swap file does not have [id] property");
                            }
                            fs = FileSystem.get(getJobConf());
                            fs.copyFromLocalFile(new Path(newId.toString()), new Path(actionDir, ACTION_NEW_ID_PROPS));
                            reporter.incrCounter(COUNTER_GROUP, COUNTER_DO_ID_SWAP, 1);


                            System.out.println("Oozie Launcher, copying new Hadoop job id to file: "
                                    + new Path(actionDir, ACTION_NEW_ID_PROPS).toUri());

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()


        Path wfParent = new Path("/falcon/test");
        fs.delete(wfParent, true);
        Path wfPath = new Path(wfParent, "workflow");
        fs.mkdirs(wfPath);
        fs.copyFromLocalFile(false, true,
                new Path(TestContext.class.getResource("/fs-workflow.xml").getPath()),
                new Path(wfPath, "workflow.xml"));
        fs.mkdirs(new Path(wfParent, "input/2012/04/20/00"));
        Path outPath = new Path(wfParent, "output");
        fs.mkdirs(outPath);

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

      // use jar name if job is not named. 
      if ("".equals(job.getJobName())){
        job.setJobName(new Path(originalJarPath).getName());
      }
      job.setJar(submitJarFile.toString());
      fs.copyFromLocalFile(new Path(originalJarPath), submitJarFile);
      fs.setReplication(submitJarFile, replication);
    } else {
      LOG.warn("No job jar file set.  User classes may not be found. "+
               "See JobConf(Class) or JobConf#setJar(String).");
    }

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

    job.setNumMapTasks(numMaps);
    job.setNumReduceTasks(numReduces);
    {
      FileSystem fs = dfs.getFileSystem();
      fs.delete(wordExec.getParent(), true);
      fs.copyFromLocalFile(program, wordExec);                                         
      Submitter.setExecutable(job, fs.makeQualified(wordExec).toString());
      Submitter.setIsJavaRecordReader(job, true);
      Submitter.setIsJavaRecordWriter(job, true);
      FileInputFormat.setInputPaths(job, inputPath);
      FileOutputFormat.setOutputPath(job, outputPath);

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

    Path wordExec = new Path("/testing/bin/application");
    Path jobXml = new Path(testDir, "job.xml");
    {
      FileSystem fs = dfs.getFileSystem();
      fs.delete(wordExec.getParent(), true);
      fs.copyFromLocalFile(program, wordExec);
    }
    DataOutputStream out = local.create(new Path(inDir, "part0"));
    out.writeBytes("i am a silly test\n");
    out.writeBytes("you are silly\n");
    out.writeBytes("i am a cat test\n");

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

    Path scriptPath = new Path(debugDir,"testscript.txt");
    Path cachePath = new Path("/cacheDir");
    if (!debugFs.mkdirs(cachePath)) {
      throw new IOException("Mkdirs failed to create " + cachePath.toString());
    }
    debugFs.copyFromLocalFile(scriptPath,cachePath);
    
    URI uri = debugFs.getUri().resolve(cachePath+"/testscript.txt#testscript");
    DistributedCache.createSymlink(conf);
    DistributedCache.addCacheFile(uri, conf);

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

      String shellPathSuffix =
          appName + "/" + appId.getId() + "/"
              + (Shell.WINDOWS ? windowBatPath : linuxShellPath);
      Path shellDst =
          new Path(fs.getHomeDirectory(), shellPathSuffix);
      fs.copyFromLocalFile(false, true, shellSrc, shellDst);
      hdfsShellScriptLocation = shellDst.toUri().toString(); 
      FileStatus shellFileStatus = fs.getFileStatus(shellDst);
      hdfsShellScriptLen = shellFileStatus.getLen();
      hdfsShellScriptTimestamp = shellFileStatus.getModificationTime();
    }

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

          Path pathDst = new Path(hdfs.getWorkingDirectory() + "/temp",
              pathSrc.getName());


          // LOG.info("WorkingDirectory: " + hdfs.getWorkingDirectory());
          LOG.debug("copyToHDFSFile: " + pathDst);
          hdfs.copyFromLocalFile(pathSrc, pathDst);
          hdfs.deleteOnExit(pathDst);


          finalPath = pathDst.makeQualified(hdfs).toString();
          finalArr[i] = finalPath;
        }

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

      // use jar name if job is not named. 
      if ("".equals(job.getJobName())){
        job.setJobName(new Path(originalJarPath).getName());
      }
      job.setJar(submitJarFile.toString());
      fs.copyFromLocalFile(new Path(originalJarPath), submitJarFile);
      fs.setReplication(submitJarFile, replication);
      fs.setPermission(submitJarFile, new FsPermission(JOB_FILE_PERMISSION));
    } else {
      LOG.warn("No job jar file set.  User classes may not be found. "+
               "See JobConf(Class) or JobConf#setJar(String).");

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

    Path scriptPath = new Path(debugDir,"testscript.txt");
    Path cachePath = new Path("/cacheDir");
    if (!debugFs.mkdirs(cachePath)) {
      throw new IOException("Mkdirs failed to create " + cachePath.toString());
    }
    debugFs.copyFromLocalFile(scriptPath,cachePath);
    
    // add debug script as cache file 
    String fileSys = debugFs.getName();
    String scr = null;
    if (fileSys.equals("local")) {

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.