Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.FileSystem.makeQualified()

85
86
87
88
89
90
91
92
93
94
95
        FileSystem fs = path.getFileSystem(config);
        Path output = path.getParent();
        Path target;
        if (output == null) {
            LOG.warn("エクスポート先のディレクトリはベースディレクトリなので削除されません: {}", path);
            target = fs.makeQualified(path);
        } else {
            LOG.warn("エクスポート先をディレクトリごと削除します: {}", output);
            target = fs.makeQualified(output);
        }
        LOG.debug("ファイルを削除します: {}", target);
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
        if (output == null) {
            LOG.warn("エクスポート先のディレクトリはベースディレクトリなので削除されません: {}", path);
            target = fs.makeQualified(path);
        } else {
            LOG.warn("エクスポート先をディレクトリごと削除します: {}", output);
            target = fs.makeQualified(output);
        }
        LOG.debug("ファイルを削除します: {}", target);
        boolean succeed = fs.delete(target, true);
        LOG.debug("ファイルを削除しました (succeed={}): {}", succeed, target);
    }
View Full Code Here
565
566
567
568
569
570
571
572
573
574
575
    // Create a number of filenames in the JobTracker's fs namespace
    FileSystem fs = getFs();
    LOG.debug("default FileSystem: " + fs.getUri());
    fs.delete(submitJobDir, true);
    submitJobDir = fs.makeQualified(submitJobDir);
    submitJobDir = new Path(submitJobDir.toUri().getPath());
    FsPermission mapredSysPerms = new FsPermission(JOB_DIR_PERMISSION);
    FileSystem.mkdirs(fs, submitJobDir, mapredSysPerms);
    Path filesDir = new Path(submitJobDir, "files");
    Path archivesDir = new Path(submitJobDir, "archives");
View Full Code Here
273
274
275
276
277
278
279
280
281
282
283
      Path path = new Path("/");
      assertTrue("/ should be a directory", fs.getFileStatus(path)
          .isDirectory() == true);
     
      byte[] fileData = AppendTestUtil.randomBytes(seed, size);
      Path file1 = fs.makeQualified(new Path("filelocal.dat"));
      FSDataOutputStream stm = createFile(fs, file1, 1);
      stm.write(fileData);
      stm.close();
     
      URI uri = cluster.getURI();
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
                                              Path matrixOutputPath,
                                              int numInputRows) throws IOException {
    JobConf conf = new JobConf(TransposeJob.class);
    conf.setJobName("TransposeJob: " + matrixInputPath + " transpose -> " + matrixOutputPath);
    FileSystem fs = FileSystem.get(conf);
    matrixInputPath = fs.makeQualified(matrixInputPath);
    matrixOutputPath = fs.makeQualified(matrixOutputPath);
    conf.setInt(NUM_ROWS_KEY, numInputRows);

    FileInputFormat.addInputPath(conf, matrixInputPath);
    conf.setInputFormat(SequenceFileInputFormat.class);
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
                                              int numInputRows) throws IOException {
    JobConf conf = new JobConf(TransposeJob.class);
    conf.setJobName("TransposeJob: " + matrixInputPath + " transpose -> " + matrixOutputPath);
    FileSystem fs = FileSystem.get(conf);
    matrixInputPath = fs.makeQualified(matrixInputPath);
    matrixOutputPath = fs.makeQualified(matrixOutputPath);
    conf.setInt(NUM_ROWS_KEY, numInputRows);

    FileInputFormat.addInputPath(conf, matrixInputPath);
    conf.setInputFormat(SequenceFileInputFormat.class);
    FileOutputFormat.setOutputPath(conf, matrixOutputPath);
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
                                                  Class<? extends TimesSquaredMapper> mapClass,
                                                  Class<? extends VectorSummingReducer> redClass) throws IOException {
    JobConf conf = new JobConf(TimesSquaredJob.class);
    conf.setJobName("TimesSquaredJob: " + matrixInputPath + " timesSquared(" + v.getName() + ')');
    FileSystem fs = FileSystem.get(conf);
    matrixInputPath = fs.makeQualified(matrixInputPath);
    outputVectorPathBase = fs.makeQualified(outputVectorPathBase);

    long now = System.nanoTime();
    Path inputVectorPath = new Path(outputVectorPathBase, INPUT_VECTOR + '/' + now);
    SequenceFile.Writer inputVectorPathWriter = new SequenceFile.Writer(fs,
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
                                                  Class<? extends VectorSummingReducer> redClass) throws IOException {
    JobConf conf = new JobConf(TimesSquaredJob.class);
    conf.setJobName("TimesSquaredJob: " + matrixInputPath + " timesSquared(" + v.getName() + ')');
    FileSystem fs = FileSystem.get(conf);
    matrixInputPath = fs.makeQualified(matrixInputPath);
    outputVectorPathBase = fs.makeQualified(outputVectorPathBase);

    long now = System.nanoTime();
    Path inputVectorPath = new Path(outputVectorPathBase, INPUT_VECTOR + '/' + now);
    SequenceFile.Writer inputVectorPathWriter = new SequenceFile.Writer(fs,
            conf, inputVectorPath, NullWritable.class, VectorWritable.class);
View Full Code Here
21
22
23
24
25
26
27
28
29
30
31
  @Override
  public int run(String[] strings) throws Exception {
    Map<String,String> parsedArgs = parseArguments(strings);
    Configuration conf = getConf();
    FileSystem fs = FileSystem.get(conf);
    Path inputPath = fs.makeQualified(new Path(parsedArgs.get("--input")));
    Path outputPath = fs.makeQualified(new Path(parsedArgs.get("--output")));
    Path indexPath = new Path(outputPath, "docIndex");
    Path matrixPath = new Path(outputPath, "matrix");
    SequenceFile.Writer indexWriter = SequenceFile.createWriter(fs,
                                                                conf,
View Full Code Here
22
23
24
25
26
27
28
29
30
31
32
  public int run(String[] strings) throws Exception {
    Map<String,String> parsedArgs = parseArguments(strings);
    Configuration conf = getConf();
    FileSystem fs = FileSystem.get(conf);
    Path inputPath = fs.makeQualified(new Path(parsedArgs.get("--input")));
    Path outputPath = fs.makeQualified(new Path(parsedArgs.get("--output")));
    Path indexPath = new Path(outputPath, "docIndex");
    Path matrixPath = new Path(outputPath, "matrix");
    SequenceFile.Writer indexWriter = SequenceFile.createWriter(fs,
                                                                conf,
                                                                indexPath,
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.