Examples of globStatus()


Examples of org.apache.hadoop.fs.FileSystem.globStatus()

      else
        fs = FileSystem.getLocal(conf);
    }
    ArrayList<Path> filesToSearch = new ArrayList<Path>();
    for(int i=firstArg; i < args.length; ++i){
      Path[] globbedPaths = FileUtil.stat2Paths(fs.globStatus(new Path(args[i])));
      for(Path p: globbedPaths)
        filesToSearch.add(p);
    }
    int tot = filesToSearch.size();
    int i=1;
View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

      ArrayList<Integer> fileNumberList  = new ArrayList<Integer>();
      int index = 0;
      for (Path p: dirs) {
        FileSystem fs = p.getFileSystem(job);
        FileStatus[] matches = fs.globStatus(p, inputFilter);
        if (matches == null) {
          errors.add(new IOException("Input path does not exist: " + p));
        } else if (matches.length == 0) {
          errors.add(new IOException("Input Pattern " + p + " matches 0 files"));
        } else {
View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

          */
         List<Path> result = new ArrayList<Path>(paths.length);
         Configuration conf = j.getConfiguration();
         for (Path p : paths) {
             FileSystem fs = p.getFileSystem(conf);
             FileStatus[] matches = fs.globStatus(p);
             if( matches == null ) {
                 throw new IOException("Input path does not exist: " + p);
             } else if (matches.length == 0) {
                 LOG.warn("Input Pattern " + p + " matches 0 files");
             } else {
View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

      FileSystem fs = srcf.getFileSystem(conf);

      // check if srcf contains nested sub-directories
      FileStatus[] srcs;
      try {
        srcs = fs.globStatus(srcf);
      } catch (IOException e) {
        throw new HiveException("Getting globStatus " + srcf.toString(), e);
      }
      if (srcs == null) {
        LOG.info("No sources specified to move: " + srcf);
View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

      } else {
        Path tablePath = new Path(table.getSd().getLocation());
        FileSystem fs = tablePath.getFileSystem(job.getConfiguration());

        if ( fs.exists(tablePath) ) {
          FileStatus[] status = fs.globStatus(new Path(tablePath, "*"), hiddenFileFilter);

          if( status.length > 0 ) {
            throw new HowlException(ErrorType.ERROR_NON_EMPTY_TABLE,
                      table.getDbName() + "." + table.getTableName());
          }
View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

          FileStatus [] dirs;
          ArrayList<FileStatus> files;
          try {
            fs = FileSystem.get(db.getTable(tbd.getTable().getTableName()).getDataLocation(),
                Hive.get().getConf());
            dirs = fs.globStatus(new Path(tbd.getSourceDir()));
            files = new ArrayList<FileStatus>();
            for (int i=0; (dirs != null && i<dirs.length); i++) {
              files.addAll(Arrays.asList(fs.listStatus(dirs[i].getPath())));
              // We only check one file, so exit the loop when we have at least one.
              if (files.size()>0) break;
View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

      String pathPattern = this.partPath.toString();
      if (getBucketCount() > 0) {
        pathPattern = pathPattern + "/*";
      }
      LOG.info("Path pattern = " + pathPattern);
      FileStatus srcs[] = fs.globStatus(new Path(pathPattern));
      Arrays.sort(srcs);
      for (FileStatus src: srcs) {
        LOG.info("Got file: " + src.getPath());
      }
      return srcs[bucketNum].getPath();
View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

      Path tablePath = new Path(table.getTTable().getSd().getLocation());
      FileSystem fs = tablePath.getFileSystem(context.getConfiguration());

      if (fs.exists(tablePath)) {
        FileStatus[] status = fs.globStatus(new Path(tablePath, "*"), hiddenFileFilter);

        if (status.length > 0) {
          throw new HCatException(ErrorType.ERROR_NON_EMPTY_TABLE,
            table.getDbName() + "." + table.getTableName());
        }
View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

      String dynPathSpec = loadPath.toUri().getPath();
      dynPathSpec = dynPathSpec.replaceAll("__HIVE_DEFAULT_PARTITION__", "*");

      //      LOG.info("Searching for "+dynPathSpec);
      Path pathPattern = new Path(dynPathSpec);
      FileStatus[] status = fs.globStatus(pathPattern);

      partitionsDiscoveredByPath = new LinkedHashMap<String, Map<String, String>>();
      contextDiscoveredByPath = new LinkedHashMap<String, JobContext>();


View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

      String dynPathSpec = loadPath.toUri().getPath();
      dynPathSpec = dynPathSpec.replaceAll("__HIVE_DEFAULT_PARTITION__", "*");

      //      LOG.info("Searching for "+dynPathSpec);
      Path pathPattern = new Path(dynPathSpec);
      FileStatus[] status = fs.globStatus(pathPattern);

      partitionsDiscoveredByPath = new LinkedHashMap<String, Map<String, String>>();
      contextDiscoveredByPath = new LinkedHashMap<String, JobContext>();


View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.