Examples of org.apache.hadoop.fs.FileSystem.makeQualified()

org.apache.hadoop.fs.FileSystem.makeQualified()
Make sure that a path specifies a FileSystem.

      Path dirPath = new Path(scheme, authority,
          scratchDir + "-" + TaskRunner.getTaskRunnerID());
      if (mkdir) {
        try {
          FileSystem fs = dirPath.getFileSystem(conf);
          dirPath = new Path(fs.makeQualified(dirPath).toString());
          if (!fs.mkdirs(dirPath)) {
            throw new RuntimeException("Cannot make directory: "
                                       + dirPath.toString());
          } else {
            FsPermission fsPermission = new FsPermission(Short.parseShort(scratchDirPermission.trim(), 8));

View Full Code Here


    // Check the output specification
    job.getOutputFormat().checkOutputSpecs(fs, job);


    // Create the splits for the job
    LOG.debug("Creating splits at " + fs.makeQualified(submitSplitFile));
    InputSplit[] splits = 
      job.getInputFormat().getSplits(job, job.getNumMapTasks());
    // sort the splits into order based on size, so that the biggest
    // go first
    Arrays.sort(splits, new Comparator<InputSplit>() {

View Full Code Here

      HConstants.DEFAULT_MAX_FILE_SIZE) <= 1024 * 1024);


    FileSystem fs = (cluster.getDFSCluster() == null) ?
      localFs : cluster.getDFSCluster().getFileSystem();
    assertNotNull(fs);
    Path d = fs.makeQualified(new Path(conf.get(HConstants.HBASE_DIR)));


    // Get connection on the meta table and get count of rows.
    HTable meta = new HTable(conf, HConstants.META_TABLE_NAME);
    int count = count(meta, tableName);
    HTable t = new HTable(conf, new Text(tableName));

View Full Code Here

    for (Path p: dirs) {
      FileSystem fs = p.getFileSystem(job); 
      Path[] matches =
        fs.listPaths(fs.globPaths(p, hiddenFileFilter), hiddenFileFilter);
      for (Path match: matches) {
        result.add(fs.makeQualified(match));
      }
    }


    return result.toArray(new Path[result.size()]);
  }

View Full Code Here

      {
        FileSystem localfs = FileSystem.getLocal(jobConf);
        String TEST_ROOT_DIR =
          new File(System.getProperty("test.build.data","/tmp"))
          .toString().replace(' ', '+');
        Path localIn = localfs.makeQualified
                          (new Path(TEST_ROOT_DIR + "/local/in"));
        Path localOut = localfs.makeQualified
                          (new Path(TEST_ROOT_DIR + "/local/out"));
        result = launchWordCount(jobConf, localIn, localOut,
                                 "all your base belong to us", 1, 1);

View Full Code Here

        String TEST_ROOT_DIR =
          new File(System.getProperty("test.build.data","/tmp"))
          .toString().replace(' ', '+');
        Path localIn = localfs.makeQualified
                          (new Path(TEST_ROOT_DIR + "/local/in"));
        Path localOut = localfs.makeQualified
                          (new Path(TEST_ROOT_DIR + "/local/out"));
        result = launchWordCount(jobConf, localIn, localOut,
                                 "all your base belong to us", 1, 1);
        assertEquals("all\t1\nbase\t1\nbelong\t1\nto\t1\nus\t1\nyour\t1\n", 
                     result.output);

View Full Code Here

        recursivels(fs, src, allFiles);
        for (FileStatus stat: allFiles) {
          String toWrite = "";
          long len = stat.isDir()? 0:stat.getLen();
          if (stat.isDir()) {
            toWrite = "" + fs.makeQualified(stat.getPath()) + " dir ";
            //get the children 
            FileStatus[] list = fs.listStatus(stat.getPath());
            StringBuffer sbuff = new StringBuffer();
            sbuff.append(toWrite);
            for (FileStatus stats: list) {

View Full Code Here

              sbuff.append(stats.getPath().getName() + " ");
            }
            toWrite = sbuff.toString();
          }
          else {
            toWrite +=  fs.makeQualified(stat.getPath()) + " file ";
          }
          srcWriter.append(new LongWritable(len), new 
              Text(toWrite));
          srcWriter.sync();
          numFiles++;

View Full Code Here

      List<Path> globPaths = new ArrayList<Path>();
      for (Path p: srcPaths) {
        FileSystem fs = p.getFileSystem(getConf());
        FileStatus[] statuses = fs.globStatus(p);
        for (FileStatus status: statuses) {
          globPaths.add(fs.makeQualified(status.getPath()));
        }
      }
      archive(globPaths, archiveName, destPath);
    } catch(IOException ie) {
      System.err.println(ie.getLocalizedMessage());

View Full Code Here

    job.setNumReduceTasks(numReduces);
    {
      FileSystem fs = dfs.getFileSystem();
      fs.delete(wordExec.getParent(), true);
      fs.copyFromLocalFile(program, wordExec);                                         
      Submitter.setExecutable(job, fs.makeQualified(wordExec).toString());
      Submitter.setIsJavaRecordReader(job, true);
      Submitter.setIsJavaRecordWriter(job, true);
      FileInputFormat.setInputPaths(job, inputPath);
      FileOutputFormat.setOutputPath(job, outputPath);
      RunningJob rJob = null;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.