Package org.apache.hadoop.fs.FileSystem

Examples of org.apache.hadoop.fs.FileSystem.Statistics

204
205
206
207
208
209
210
211
212
213
214
  protected static synchronized Map<URI, Statistics> getAllStatistics() {
    Map<URI, Statistics> statsMap = new HashMap<URI, Statistics>(
        STATISTICS_TABLE.size());
    for (Map.Entry<URI, Statistics> pair : STATISTICS_TABLE.entrySet()) {
      URI key = pair.getKey();
      Statistics value = pair.getValue();
      Statistics newStatsObj = new Statistics(value);
      statsMap.put(URI.create(key.toString()), newStatsObj);
    }
    return statsMap;
  }
View Full Code Here
151
152
153
154
155
156
157
158
159
160
161
162
   * @param cls the class to lookup
   * @return a statistics object
   */
  public static synchronized Statistics getStatistics(String scheme,
      Class<? extends AbstractFileSystem> cls) {
    Statistics result = STATISTICS_TABLE.get(cls);
    if (result == null) {
      result = new Statistics(scheme);
      STATISTICS_TABLE.put(cls, result);
    }
    return result;
  }
View Full Code Here
162
163
164
165
166
167
168
169
170
171
172
173
    if (scheme == null) {
      throw new IllegalArgumentException("Scheme not defined in the uri: "
          + uri);
    }
    URI baseUri = getBaseUri(uri);
    Statistics result = STATISTICS_TABLE.get(baseUri);
    if (result == null) {
      result = new Statistics(scheme);
      STATISTICS_TABLE.put(baseUri, result);
    }
    return result;
  }
View Full Code Here
203
204
205
206
207
208
209
210
211
212
213
  protected static synchronized Map<URI, Statistics> getAllStatistics() {
    Map<URI, Statistics> statsMap = new HashMap<URI, Statistics>(
        STATISTICS_TABLE.size());
    for (Map.Entry<URI, Statistics> pair : STATISTICS_TABLE.entrySet()) {
      URI key = pair.getKey();
      Statistics value = pair.getValue();
      Statistics newStatsObj = new Statistics(value);
      statsMap.put(URI.create(key.toString()), newStatsObj);
    }
    return statsMap;
  }
View Full Code Here
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
   *          the path.
   * @return a Statistics instance, or null if none is found for the scheme.
   */
  protected static Statistics getFsStatistics(Path path, Configuration conf)
      throws IOException {
    Statistics matchedStats = null;
    path = path.getFileSystem(conf).makeQualified(path);
    for (Statistics stats : FileSystem.getAllStatistics()) {
      if (stats.getScheme().equals(path.toUri().getScheme())) {
        matchedStats = stats;
        break;
View Full Code Here
436
437
438
439
440
441
442
443
444
445
446
        JobConf job, TaskReporter reporter, String finalName)
        throws IOException {
      this.outputRecordCounter = outputRecordCounter;
      this.fileOutputByteCounter = reporter
          .getCounter(FileOutputFormat.Counter.BYTES_WRITTEN);
      Statistics matchedStats = null;
      if (job.getOutputFormat() instanceof FileOutputFormat) {
        matchedStats = getFsStatistics(FileOutputFormat.getOutputPath(job), job);
      }
      fsStats = matchedStats;
View Full Code Here
555
556
557
558
559
560
561
562
563
564
565
        org.apache.hadoop.mapreduce.TaskAttemptContext taskContext)
        throws InterruptedException, IOException {
      this.outputRecordCounter = recordCounter;
      this.fileOutputByteCounter = reporter
          .getCounter(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.Counter.BYTES_WRITTEN);
      Statistics matchedStats = null;
      // TaskAttemptContext taskContext = new TaskAttemptContext(job,
      // getTaskID());
      if (outputFormat instanceof org.apache.hadoop.mapreduce.lib.output.FileOutputFormat) {
        matchedStats = getFsStatistics(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat
            .getOutputPath(taskContext), taskContext.getConfiguration());
View Full Code Here
163
164
165
166
167
168
169
170
171
172
173
174
    if (scheme == null) {
      throw new IllegalArgumentException("Scheme not defined in the uri: "
          + uri);
    }
    URI baseUri = getBaseUri(uri);
    Statistics result = STATISTICS_TABLE.get(baseUri);
    if (result == null) {
      result = new Statistics(scheme);
      STATISTICS_TABLE.put(baseUri, result);
    }
    return result;
  }
View Full Code Here
204
205
206
207
208
209
210
211
212
213
214
  protected static synchronized Map<URI, Statistics> getAllStatistics() {
    Map<URI, Statistics> statsMap = new HashMap<URI, Statistics>(
        STATISTICS_TABLE.size());
    for (Map.Entry<URI, Statistics> pair : STATISTICS_TABLE.entrySet()) {
      URI key = pair.getKey();
      Statistics value = pair.getValue();
      Statistics newStatsObj = new Statistics(value);
      statsMap.put(URI.create(key.toString()), newStatsObj);
    }
    return statsMap;
  }
View Full Code Here
185
186
187
188
189
190
191
192
193
194
195
      inputRecordCounter = reporter.getCounter(MAP_INPUT_RECORDS);
      inputByteCounter = reporter.getCounter(MAP_INPUT_BYTES);
      fileInputByteCounter = reporter
          .getCounter(FileInputFormat.Counter.BYTES_READ);

      Statistics matchedStats = null;
      if (split instanceof FileSplit) {
        matchedStats = getFsStatistics(((FileSplit) split).getPath(), job);
      }
      fsStats = matchedStats;
     
View Full Code Here
TOP

Related Classes of org.apache.hadoop.fs.FileSystem.Statistics

  • org.apache.hadoop.fs.AbstractFileSystem
  • org.apache.hadoop.fs.FCStatisticsBaseTest
  • org.apache.hadoop.mapred.MapTask$DirectMapOutputCollector
  • org.apache.hadoop.mapred.MapTask$NewDirectOutputCollector
  • org.apache.hadoop.mapred.MapTask$NewTrackingRecordReader
  • org.apache.hadoop.mapred.MapTask$TrackedRecordReader
  • org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter
  • org.apache.hadoop.mapred.ReduceTask$OldTrackingRecordWriter
  • org.apache.hadoop.mapred.Task
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.