Examples of globStatus()


Examples of org.apache.hadoop.fs.FileSystem.globStatus()

            LOG.debug(MessageFormat.format(
                    "Listing temporary input: {0} (fs={1})",
                    pathPattern,
                    fs.getUri()));
        }
        FileStatus[] statusList = fs.globStatus(pathPattern);
        if (statusList == null || statusList.length == 0) {
            return Collections.emptyList();
        }
        List<Path> results = new ArrayList<Path>();
        for (FileStatus status : statusList) {
View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

    private List<InputSplit> getSplits(Configuration configuration, List<Path> paths) throws IOException {
        long splitSize = configuration.getLong(KEY_DEFAULT_SPLIT_SIZE, DEFAULT_SPLIT_SIZE);
        List<InputSplit> results = new ArrayList<InputSplit>();
        for (Path path : paths) {
            FileSystem fs = path.getFileSystem(configuration);
            FileStatus[] statuses = fs.globStatus(path);
            if (statuses == null) {
                continue;
            }
            for (FileStatus status : statuses) {
                BlockMap blockMap = BlockMap.create(
View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

            context.setError();
            return false;
        }
        List<FileStatus> files;
        try {
            files = asList(fs.globStatus(path));
        } catch (Exception e) {
            LOG.error(MessageFormat.format(
                    "[OT-CLEAN-E01002] Failed to glob path pattern: {0}",
                    path), e);
            context.setError();
View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

            TsvIoFactory<T> factory = new TsvIoFactory<T>(targetTableModel);
            Configuration conf = new Configuration();
            fs = FileSystem.get(new URI(filePath), conf);

            // ディレクトリ内のファイルを取得し、ファイルを出力する
            FileStatus[] status = fs.globStatus(new Path(filePath));
            Path[] listedPaths = FileUtil.stat2Paths(status);
            if (listedPaths == null) {
                LOG.info("TG-COLLECTOR-02006",
                        tableName, filePath);
                return -1;
View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

    }

    private void loadResult(String tablename, String excelFileName) throws IOException {
        Configuration conf = ConfigurationFactory.getDefault().newInstance();
        FileSystem fs = FileSystem.get(conf);
        FileStatus[] status = fs.globStatus(new Path(computeOutputPath(fs, excelFileName)));
        Path[] listedPaths = FileUtil.stat2Paths(status);
        for (Path path : listedPaths) {
            if (isSystemFile(path)) {
                continue;
            }
View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

    }

    private List<TestDataModel> collect(CacheStorage storage, Path contents) throws IOException {
        List<TestDataModel> results = new ArrayList<TestDataModel>();
        FileSystem fs = storage.getFileSystem();
        for (FileStatus status : fs.globStatus(contents)) {
            results.addAll(collectContent(fs, status));
        }
        Collections.sort(results);
        return results;
    }
View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

   */
  public static PathData[] expandAsGlob(String pattern, Configuration conf)
  throws IOException {
    Path globPath = new Path(pattern);
    FileSystem fs = globPath.getFileSystem(conf);   
    FileStatus[] stats = fs.globStatus(globPath);
    PathData[] items = null;
   
    if (stats == null) {
      // remove any quoting in the glob pattern
      pattern = pattern.replaceAll("\\\\(.)", "$1");
View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

    LongWritable value = new LongWritable();
    int minSupport = Integer.valueOf(params.get("minSupport", "3"));
    Configuration conf = new Configuration();
   
    FileSystem fs = FileSystem.get(new Path(params.get("output") + "/parallelcounting").toUri(), conf);
    FileStatus[] outputFiles = fs.globStatus(new Path(params.get("output") + "/parallelcounting/part-*"));
   
    PriorityQueue<Pair<String,Long>> queue = new PriorityQueue<Pair<String,Long>>(11,
        new Comparator<Pair<String,Long>>() {
         
          @Override
View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

  public static List<Pair<String,TopKStringPatterns>> readFrequentPattern(Parameters params) throws IOException {
   
    Configuration conf = new Configuration();
   
    FileSystem fs = FileSystem.get(new Path(params.get("output") + "/frequentPatterns").toUri(), conf);
    FileStatus[] outputFiles = fs.globStatus(new Path(params.get("output") + "/frequentPatterns/part-*"));
   
    List<Pair<String,TopKStringPatterns>> ret = new ArrayList<Pair<String,TopKStringPatterns>>();
    for (FileStatus fileStatus : outputFiles) {
      Path path = fileStatus.getPath();
      ret.addAll(FPGrowth.readFrequentPattern(fs, conf, path));
View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

   
    Writable key = new Text();
    Configuration conf = new Configuration();
   
    FileSystem fs = FileSystem.get(wordCountPath.toUri(), conf);
    FileStatus[] outputFiles = fs.globStatus(new Path(wordCountPath.toString()
                                                      + OUTPUT_FILES_PATTERN));
   
    long chunkSizeLimit = chunkSizeInMegabytes * 1024 * 1024;
    int chunkIndex = 0;
    Path chunkPath = getPath(dictionaryPathBase + DICTIONARY_FILE,
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.