Examples of org.apache.hadoop.fs.FileSystem.globStatus()

org.apache.hadoop.fs.FileSystem.globStatus()

Return all the files that match filePattern and are not checksum files. Results are sorted by their names.
A filename pattern is composed of regular characters and special pattern matching characters, which are:

?
Matches any single character.

*
Matches zero or more characters.

[abc]
Matches a single character from character set {a,b,c}.

[a-b]
Matches a single character from the character range {a...b}. Note that character a must be lexicographically less than or equal to character b.

[^a]
Matches a single character that is not from character set or range {a}. Note that the ^ character must occur immediately to the right of the opening bracket.

\c
Removes (escapes) any special meaning of character c.

{ab,cd}
Matches a string from the string set {ab, cd}

{ab,c{de,fh}}
Matches a string from the string set {ab, cde, cfh}

@param pathPattern a regular expression specifying a pth pattern @return an array of paths that match the path pattern @throws IOException

      String pathPattern = getPartitionPath().toString();
      if (getBucketCount() > 0) {
        pathPattern = pathPattern + "/*";
      }
      LOG.info("Path pattern = " + pathPattern);
      FileStatus srcs[] = fs.globStatus(new Path(pathPattern));
      Arrays.sort(srcs);
      for (FileStatus src : srcs) {
        LOG.info("Got file: " + src.getPath());
      }
      if (srcs.length == 0) {

View Full Code Here

          HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS);


      // check if srcf contains nested sub-directories
      FileStatus[] srcs;
      try {
        srcs = fs.globStatus(srcf);
      } catch (IOException e) {
        throw new HiveException("Getting globStatus " + srcf.toString(), e);
      }
      if (srcs == null) {
        LOG.info("No sources specified to move: " + srcf);

View Full Code Here

      String pathPattern = getPath().toString();
      if (getNumBuckets() > 0) {
        pathPattern = pathPattern + "/*";
      }
      LOG.info("Path pattern = " + pathPattern);
      FileStatus srcs[] = fs.globStatus(new Path(pathPattern));
      Arrays.sort(srcs);
      for (FileStatus src : srcs) {
        LOG.info("Got file: " + src.getPath());
      }
      if (srcs.length == 0) {

View Full Code Here

          FileStatus[] dirs;
          ArrayList<FileStatus> files;
          FileSystem fs;
          try {
            fs = FileSystem.get(table.getDataLocation(), conf);
            dirs = fs.globStatus(new Path(tbd.getSourceDir()));
            files = new ArrayList<FileStatus>();
            for (int i = 0; (dirs != null && i < dirs.length); i++) {
              files.addAll(Arrays.asList(fs.listStatus(dirs[i].getPath())));
              // We only check one file, so exit the loop when we have at least
              // one.

View Full Code Here

      console.printInfo("Use sampling data created in previous MR");
      // merges sampling data from previous MR and make paritition keys for total sort
      for (String path : paths) {
        Path inputPath = new Path(path);
        FileSystem fs = inputPath.getFileSystem(job);
        for (FileStatus status : fs.globStatus(new Path(inputPath, ".sampling*"))) {
          sampler.addSampleFile(status.getPath(), job);
        }
      }
    } else if (mWork.getSamplingType() == MapWork.SAMPLING_ON_START) {
      console.printInfo("Creating sampling data..");

View Full Code Here

        final CatalogStorage tableStorage = (CatalogStorage) storage;
        String stagingDir = FeedHelper.getStagingDir(cluster, feed, tableStorage, Tag.REPLICATION);
        Path stagingPath = new Path(stagingDir + "/*/*/*");  // stagingDir/dataOutPartitionValue/nominal-time/data
        FileSystem fs = getFileSystem(cluster);
        try {
            FileStatus[] paths = fs.globStatus(stagingPath);
            delete(cluster, feed, retention, paths);
        } catch (IOException e) {
            throw new FalconException(e);
        }
    }

View Full Code Here

        String stagingPath = ClusterHelper.getLocation(cluster, "staging");
        Path logPath = getLogPath(entity, stagingPath);
        FileSystem fs = getFileSystem(cluster);
        FileStatus[] paths;
        try {
            paths = fs.globStatus(logPath);
        } catch (IOException e) {
            throw new FalconException(e);
        }
        return paths;
    }

View Full Code Here

        return usage;
    }


    private long usage(Path inPath, Configuration conf) throws IOException {
        FileSystem fs = inPath.getFileSystem(conf);
        FileStatus[] fileStatuses = fs.globStatus(inPath);
        if (fileStatuses == null || fileStatuses.length == 0) {
            return 0;
        }
        long totalSize = 0;
        for (FileStatus fileStatus : fileStatuses) {

View Full Code Here

                equals(sourcePath.toString()) : "Source path is not a subset of include path";


        String relativePath = includePath.toString().substring(sourcePath.toString().length());
        String fixedPath = getFixedPath(relativePath);


        FileStatus[] files = fs.globStatus(new Path(targetPath.toString() + "/" + fixedPath));
        if (files != null) {
            for (FileStatus file : files) {
                fs.create(new Path(file.getPath(), FileOutputCommitter.SUCCEEDED_FILE_NAME)).close();
                LOG.info("Created " + new Path(file.getPath(), FileOutputCommitter.SUCCEEDED_FILE_NAME));
            }

View Full Code Here

      }
      // do a glob on the srcPaths and then pass it on
      List<Path> globPaths = new ArrayList<Path>();
      for (Path p: srcPaths) {
        FileSystem fs = p.getFileSystem(getConf());
        FileStatus[] statuses = fs.globStatus(p);
        for (FileStatus status: statuses) {
          globPaths.add(fs.makeQualified(status.getPath()));
        }
      }
      archive(globPaths, archiveName, destPath);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.