Examples of org.apache.hadoop.fs.FileSystem.copyToLocalFile()

org.apache.hadoop.fs.FileSystem.copyToLocalFile()
The src file is under FS, and the dst is on the local disk. Copy it from FS control to the local dst name.

          Path pathSrc = new Path(uri.getPath());
          if (hdfs.exists(pathSrc)) {
            LocalFileSystem local = FileSystem.getLocal(conf);
            Path pathDst = new Path(local.getWorkingDirectory(),
                pathSrc.getName());
            hdfs.copyToLocalFile(pathSrc, pathDst);
            files.append(pathDst.toUri().getPath());
          }
          first = false;
        }
      }

View Full Code Here

    this.localJarFile = master.getLocalPath(BSPMaster.SUBDIR + "/" + jobId
        + ".jar");


    Path jobDir = master.getSystemDirectoryForJob(jobId);
    FileSystem fs = jobDir.getFileSystem(conf);
    fs.copyToLocalFile(jobFile, localJobFile);
    BSPJob job = new BSPJob(jobId, localJobFile.toString());
    this.jobSplit = job.getConfiguration().get("bsp.job.split.file");


    this.numBSPTasks = job.getNumBspTask();
    this.taskCompletionEvents = new ArrayList<TaskCompletionEvent>(

View Full Code Here

    status.setUsername(job.getUser());
    status.setStartTime(startTime);


    String jarFile = job.getJar();
    if (jarFile != null) {
      fs.copyToLocalFile(new Path(jarFile), localJarFile);
    }


    failedTasksTillNow = new HashSet<Task>(2 * tasks.length);


  }

View Full Code Here

            " is not publicly accessable and as such cannot be part of the" +
            " public cache.");
      }
    }
    
    sourceFs.copyToLocalFile(sCopy, dCopy);
    return dCopy;
  }


  private long unpack(File localrsrc, File dst, Pattern pattern) throws IOException {
    switch (resource.getType()) {

View Full Code Here


          if(dstFs.delete(targetPath, true) || !dstFs.exists(targetPath)) {
            console.printInfo(mesg, mesg_detail);
          // if source exists, rename. Otherwise, create a empty directory
          if (fs.exists(sourcePath))
            fs.copyToLocalFile(sourcePath, targetPath);
          else
            dstFs.mkdirs(targetPath);
          } else {
            console.printInfo("Unable to delete the existing destination directory: " + targetPath);
          }

View Full Code Here

    this.localJarFile = default_job_conf.getLocalPath(JobTracker.SUBDIR
                                                      +"/"+ jobid + ".jar");
    Path sysDir = new Path(this.jobtracker.getSystemDir());
    FileSystem fs = sysDir.getFileSystem(default_conf);
    jobFile = new Path(sysDir, jobid + "/job.xml");
    fs.copyToLocalFile(jobFile, localJobFile);
    conf = new JobConf(localJobFile);
    this.priority = conf.getJobPriority();
    this.status.setJobPriority(this.priority);
    this.profile = new JobProfile(conf.getUser(), jobid, 
                                  jobFile.toString(), url, conf.getJobName(),

View Full Code Here

    this.profile = new JobProfile(conf.getUser(), jobid, 
                                  jobFile.toString(), url, conf.getJobName(),
                                  conf.getQueueName());
    String jarFile = conf.getJar();
    if (jarFile != null) {
      fs.copyToLocalFile(new Path(jarFile), localJarFile);
      conf.setJar(localJarFile.toString());
    }


    this.numMapTasks = conf.getNumMapTasks();
    this.numReduceTasks = conf.getNumReduceTasks();

View Full Code Here

      FileSystem fs = path.getFileSystem(this.conf);
      File parentDir = new File(this.conf.get("hbase.local.dir") + COPROCESSOR_JARS_DIR);
      parentDir.mkdirs();
      File dst = new File(parentDir, "." + pathPrefix +
          "." + className + "." + System.currentTimeMillis() + ".jar");
      fs.copyToLocalFile(path, new Path(dst.toString()));
      dst.deleteOnExit();


      // TODO: code weaving goes here


      // TODO: wrap heap allocations and enforce maximum usage limits

View Full Code Here

    try {
      Path dataModelPath = new Path(dataModelFile);
      FileSystem fs = FileSystem.get(dataModelPath.toUri(), jobConf);
      File tempDataFile = File.createTempFile("mahout-taste-hadoop", "txt");
      tempDataFile.deleteOnExit();
      fs.copyToLocalFile(dataModelPath, new Path(tempDataFile.getAbsolutePath()));
      fileDataModel = new FileDataModel(tempDataFile);
    } catch (IOException ioe) {
      throw new IllegalStateException(ioe);
    }
    try {

View Full Code Here

    }


    FileSystem fs = path.getFileSystem(conf);
    File dst = new File(parentDirStr, "." + pathPrefix + "."
      + path.getName() + "." + System.currentTimeMillis() + ".jar");
    fs.copyToLocalFile(path, new Path(dst.toString()));
    dst.deleteOnExit();


    addURL(dst.getCanonicalFile().toURI().toURL());


    JarFile jarFile = new JarFile(dst.toString());

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.