Examples of org.apache.hadoop.fs.FileSystem.globStatus()

org.apache.hadoop.fs.FileSystem.globStatus()

Return all the files that match filePattern and are not checksum files. Results are sorted by their names.
A filename pattern is composed of regular characters and special pattern matching characters, which are:

?
Matches any single character.

*
Matches zero or more characters.

[abc]
Matches a single character from character set {a,b,c}.

[a-b]
Matches a single character from the character range {a...b}. Note that character a must be lexicographically less than or equal to character b.

[^a]
Matches a single character that is not from character set or range {a}. Note that the ^ character must occur immediately to the right of the opening bracket.

\c
Removes (escapes) any special meaning of character c.

{ab,cd}
Matches a string from the string set {ab, cd}

{ab,c{de,fh}}
Matches a string from the string set {ab, cde, cfh}

@param pathPattern a regular expression specifying a pth pattern @return an array of paths that match the path pattern @throws IOException

      if (results.hasOption("cachefiles")) {
        FileSystem fs = FileSystem.get(getConf());
        String[] optionValues = results.getOptionValues("cachefiles");
        for (String s : optionValues) {
          Path path = new Path(s);
          FileStatus[] globStatus = fs.globStatus(path);
          for (FileStatus f : globStatus) {
            if (!f.isDir()) {
              DistributedCache.addCacheFile(f.getPath().toUri(),
                  job.getConfiguration());
            } else {

View Full Code Here

    for (Path p : dirs) {
      FileSystem fs = p.getFileSystem(job.getConfiguration());


      FileStatus[] matches = null;
      try {
        matches = fs.globStatus(p, inputFilter);
      } catch (Exception e) {
        LOG.info(p + "\n" + e.toString());
      }


      if (matches == null) {

View Full Code Here

          */
         List<Path> result = new ArrayList<Path>(paths.length);
         Configuration conf = j.getConfiguration();
         for (Path p : paths) {
             FileSystem fs = p.getFileSystem(conf);
             FileStatus[] matches = fs.globStatus(p);
             if( matches == null ) {
                 throw new IOException("Input path does not exist: " + p);
             } else if (matches.length == 0) {
                 LOG.warn("Input Pattern " + p + " matches 0 files");
             } else {

View Full Code Here

    return sum;
  }


  private long getSizeInternal(Configuration conf, Path path) throws IOException {
    FileSystem fs = path.getFileSystem(conf);
    FileStatus[] statuses = fs.globStatus(path, HFileInputFormat.HIDDEN_FILE_FILTER);
    if (statuses == null) {
      return 0;
    }
    long sum = 0;
    for (FileStatus status : statuses) {

View Full Code Here


      ArrayList<Integer> fileNumberList  = new ArrayList<Integer>();
      int index = 0;
      for (Path p: dirs) {
        FileSystem fs = p.getFileSystem(job); 
        FileStatus[] matches = fs.globStatus(p, inputFilter);
        if (matches == null) {
          errors.add(new IOException("Input path does not exist: " + p));
        } else if (matches.length == 0) {
          errors.add(new IOException("Input Pattern " + p + " matches 0 files"));
        } else {

View Full Code Here


      ArrayList<Integer> fileNumberList  = new ArrayList<Integer>();
      int index = 0;
      for (Path p: dirs) {
        FileSystem fs = p.getFileSystem(job); 
        FileStatus[] matches = fs.globStatus(p, inputFilter);
        if (matches == null) {
          errors.add(new IOException("Input path does not exist: " + p));
        } else if (matches.length == 0) {
          errors.add(new IOException("Input Pattern " + p + " matches 0 files"));
        } else {

View Full Code Here

            // separate them out
            for(String location : LoadFunc.getPathStrings(input)){
                if(! UriUtil.isHDFSFileOrLocal(location)){
                    continue;
                }
                FileStatus[] status=fs.globStatus(new Path(location));
                if (status != null){
                    for (FileStatus s : status){
                        size += getPathLength(fs, s);
                    }
                }

View Full Code Here


  private static final Log LOG = LogFactory.getLog(SKJobFactory.class);
    
  public static void addDependencies(Configuration conf) throws IOException {
    FileSystem fs = FileSystem.get(conf);
    FileStatus[] jars = fs.globStatus(new Path("/texaspete/lib/*.jar"));
    if (jars.length > 0) {
      for (FileStatus jar: jars) {
        LOG.info("Adding jar to DC/CP: " + jar.getPath());
        DistributedCache.addFileToClassPath(jar.getPath(), conf, fs);
      }

View Full Code Here

      }
      // do a glob on the srcPaths and then pass it on
      List<Path> globPaths = new ArrayList<Path>();
      for (Path p: srcPaths) {
        FileSystem fs = p.getFileSystem(getConf());
        FileStatus[] statuses = fs.globStatus(p);
        for (FileStatus status: statuses) {
          globPaths.add(fs.makeQualified(status.getPath()));
        }
      }
      archive(parentPath, globPaths, archiveName, destPath);

View Full Code Here

   */
  public List<Path> getSrcPathExpanded() throws IOException {
    FileSystem fs = srcPath.getFileSystem(conf);


    // globbing on srcPath
    FileStatus[] gpaths = fs.globStatus(srcPath);
    if (gpaths == null) {
      return Collections.emptyList();
    }
    List<Path> results = new ArrayList<Path>(gpaths.length);
    for (FileStatus f : gpaths) {

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.