Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.FileSystem.copyToLocalFile()

98
99
100
101
102
103
104
105
106
107
108
      throw new IOException("Resource " + sCopy +
          " changed on src filesystem (expected " + resource.getTimestamp() +
          ", was " + sStat.getModificationTime());
    }

    sourceFs.copyToLocalFile(sCopy, dCopy);
    return dCopy;
  }

  private long unpack(File localrsrc, File dst, Pattern pattern) throws IOException {
    switch (resource.getType()) {
View Full Code Here
244
245
246
247
248
249
250
251
252
253
254
    this.localJarFile = default_job_conf.getLocalPath(JobTracker.SUBDIR
                                                      +"/"+ jobid + ".jar");
    Path jobDir = jobtracker.getSystemDirectoryForJob(jobId);
    FileSystem fs = jobDir.getFileSystem(default_conf);
    jobFile = new Path(jobDir, "job.xml");
    fs.copyToLocalFile(jobFile, localJobFile);
    conf = new JobConf(localJobFile);
    this.priority = conf.getJobPriority();
    this.status.setJobPriority(this.priority);
    this.profile = new JobProfile(conf.getUser(), jobid,
                                  jobFile.toString(), url, conf.getJobName(),
View Full Code Here
253
254
255
256
257
258
259
260
261
262
263
    this.profile = new JobProfile(conf.getUser(), jobid,
                                  jobFile.toString(), url, conf.getJobName(),
                                  conf.getQueueName());
    String jarFile = conf.getJar();
    if (jarFile != null) {
      fs.copyToLocalFile(new Path(jarFile), localJarFile);
      conf.setJar(localJarFile.toString());
    }

    this.numMapTasks = conf.getNumMapTasks();
    this.numReduceTasks = conf.getNumReduceTasks();
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
    FileDataModel fileDataModel;
    try {
      FileSystem fs = FileSystem.get(jobConf);
      File tempDataFile = File.createTempFile("mahout-taste-hadoop", "txt");
      tempDataFile.deleteOnExit();
      fs.copyToLocalFile(new Path(dataModelFile), new Path(tempDataFile.getAbsolutePath()));
      fileDataModel = new FileDataModel(tempDataFile);
    } catch (IOException ioe) {
      throw new RuntimeException(ioe);
    }
    try {
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
      throws IOException {
    FileSystem fs = FileSystem.get(uri, conf);
    Path cwd = new Path(System.getenv(Environment.PWD.name()));
    Path dFile = new Path(cwd, destName);
    Path srcPath = new Path(uri);
    fs.copyToLocalFile(srcPath, dFile);
    return dFile.makeQualified(FileSystem.getLocal(conf).getUri(), cwd);
  }

  public static byte[] getLocalSha(Path path, Configuration conf) throws IOException {
    InputStream is = null;
View Full Code Here
185
186
187
188
189
190
191
192
193
194
195
            " is not publicly accessable and as such cannot be part of the" +
            " public cache.");
      }
    }
   
    sourceFs.copyToLocalFile(sCopy, dCopy);
    return dCopy;
  }

  private long unpack(File localrsrc, File dst, Pattern pattern) throws IOException {
    switch (resource.getType()) {
View Full Code Here
91
92
93
94
95
96
97
98
99
100
101
    this.localJarFile = master.getLocalPath(BSPMaster.SUBDIR + "/" + jobId
        + ".jar");

    Path jobDir = master.getSystemDirectoryForJob(jobId);
    FileSystem fs = jobDir.getFileSystem(conf);
    fs.copyToLocalFile(jobFile, localJobFile);
    BSPJob job = new BSPJob(jobId, localJobFile.toString());
    this.numBSPTasks = job.getNumBspTask();

    this.profile = new JobProfile(job.getUser(), jobId, jobFile.toString(),
        job.getJobName());
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
    status.setUsername(job.getUser());
    status.setStartTime(startTime);

    String jarFile = job.getJar();
    if (jarFile != null) {
      fs.copyToLocalFile(new Path(jarFile), localJarFile);
    }

  }

  public JobProfile getProfile() {
View Full Code Here
795
796
797
798
799
800
801
802
803
804
805
            } else {
                // fetch from remote:
                File dest = new File(localTempDir, filename);
                dest.deleteOnExit();
                try {
                    srcFs.copyToLocalFile(file.getPath(), new Path(dest.getAbsolutePath()));
                } catch (IOException e) {
                    throw new ExecException("Could not copy " + filePath + " to local destination " + dest, 101, PigException.INPUT, e);
                }
                fetchFiles[idx++] = new FetchFileRet(dest, true);
            }
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
    String recommenderClassName = jobConf.get(RECOMMENDER_CLASS_NAME);
    Path dataModelPath = new Path(dataModelFile);
    FileSystem fs = FileSystem.get(dataModelPath.toUri(), jobConf);
    File tempDataFile = File.createTempFile("mahout-taste-hadoop", "txt");
    tempDataFile.deleteOnExit();
    fs.copyToLocalFile(dataModelPath, new Path(tempDataFile.getAbsolutePath()));
    FileDataModel fileDataModel = new FileDataModel(tempDataFile);

    try {
      Class<? extends Recommender> recommenderClass = Class.forName(recommenderClassName).asSubclass(
        Recommender.class);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.