Package org.apache.hadoop.mapreduce

Examples of org.apache.hadoop.mapreduce.Counters

203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
              .newInstance(job.getOutputFormatClass(),
                  job.getConfiguration()).getClass()
              .getName());
      job.waitForCompletion(false);
      if (job.isComplete()) {
        Counters counters = job.getCounters();
        totalRecordsRead = counters.findCounter(
            DedupRecordCounter.TOTAL_RECORDS_READ).getValue();
        badRecords = counters.findCounter(
            DedupRecordCounter.BAD_RECORD).getValue();
        output = counters.findCounter(DedupRecordCounter.OUTPUT)
            .getValue();
        duplicateRecords = totalRecordsRead - output;
        logger.info("Total records read are: " + totalRecordsRead);
        logger.info("Bad Records are: " + badRecords);
        logger.info("Output records are: " + output);
View Full Code Here
49
50
51
52
53
54
55
  private Mapper<K1, V1, K2, V2> myMapper;
  private Counters counters;

  public MapDriver(final Mapper<K1, V1, K2, V2> m) {
    myMapper = m;
    counters = new Counters();
  }
View Full Code Here
53
54
55
56
57
58
59
    myMapper = m;
    counters = new Counters();
  }

  public MapDriver() {
    counters = new Counters();
  }
View Full Code Here
50
51
52
53
54
55
56
  private Reducer<K1, V1, K2, V2> myReducer;
  private Counters counters;

  public ReduceDriver(final Reducer<K1, V1, K2, V2> r) {
    myReducer = r;
    counters = new Counters();
  }
View Full Code Here
54
55
56
57
58
59
60
    myReducer = r;
    counters = new Counters();
  }

  public ReduceDriver() {
    counters = new Counters();
  }
View Full Code Here
282
283
284
285
286
287
288
289
290
291
292
293
294
295
            }
          }

          @Override
          public boolean dump(Job job) {
              Counters counters;
            try {
              counters = job.getCounters();
                    long wsize=counters.findCounter("higo", "dumpcount").getValue();
                    if(wsize>0)
                    {
                      try {
                  TableJoin.LOG.info("update "+tableName+ ",dump");
                  TableJoin.updatePercent(tableName, "Stage-2 map = 100%,  reduce = 100%", "DUMP")  ;
View Full Code Here
780
781
782
783
784
785
786
787
788
789
790
791
          || state == JobStateInternal.KILLED || state == JobStateInternal.SUCCEEDED) {
        this.mayBeConstructFinalFullCounters();
        return fullCounters;
      }

      Counters counters = new Counters();
      counters.incrAllCounters(jobCounters);
      return incrTaskCounters(counters, tasks.values());

    } finally {
      readLock.unlock();
    }
View Full Code Here
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
    }
  }

  @Private
  public void constructFinalFullcounters() {
    this.fullCounters = new Counters();
    this.finalMapCounters = new Counters();
    this.finalReduceCounters = new Counters();
    this.fullCounters.incrAllCounters(jobCounters);
    for (Task t : this.tasks.values()) {
      Counters counters = t.getCounters();
      switch (t.getType()) {
      case MAP:
        this.finalMapCounters.incrAllCounters(counters);
        break;
      case REDUCE:
View Full Code Here
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
  @Override
  public Counters getCounters() {
    readLock.lock();
    try {
      Counters counters = reportedStatus.counters;
      if (counters == null) {
        counters = EMPTY_COUNTERS;
      }
      return counters;
    } finally {
View Full Code Here
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
  }
 
  private void updateProgressSplits() {
    double newProgress = reportedStatus.progress;
    newProgress = Math.max(Math.min(newProgress, 1.0D), 0.0D);
    Counters counters = reportedStatus.counters;
    if (counters == null)
      return;

    WrappedProgressSplitsBlock splitsBlock = getProgressSplitBlock();
    if (splitsBlock != null) {
      long now = clock.getTime();
      long start = getLaunchTime(); // TODO Ensure not 0

      if (start != 0 && now - start <= Integer.MAX_VALUE) {
        splitsBlock.getProgressWallclockTime().extend(newProgress,
            (int) (now - start));
      }

      Counter cpuCounter = counters.findCounter(TaskCounter.CPU_MILLISECONDS);
      if (cpuCounter != null && cpuCounter.getValue() <= Integer.MAX_VALUE) {
        splitsBlock.getProgressCPUTime().extend(newProgress,
            (int) cpuCounter.getValue()); // long to int? TODO: FIX. Same below
      }

      Counter virtualBytes = counters
        .findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES);
      if (virtualBytes != null) {
        splitsBlock.getProgressVirtualMemoryKbytes().extend(newProgress,
            (int) (virtualBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
      }

      Counter physicalBytes = counters
        .findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES);
      if (physicalBytes != null) {
        splitsBlock.getProgressPhysicalMemoryKbytes().extend(newProgress,
            (int) (physicalBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
      }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.Counters

  • co.cask.cdap.internal.app.runtime.batch.MapReduceMetricsWriter
  • co.nubetech.hiho.dedup.DedupJob
  • co.nubetech.hiho.dedup.TestDedupKeyMapper
  • co.nubetech.hiho.dedup.TestDedupKeyReducer
  • co.nubetech.hiho.dedup.TestDedupValueMapper
  • co.nubetech.hiho.dedup.TestDedupValueReducer
  • co.nubetech.hiho.merge.MergeJob
  • co.nubetech.hiho.merge.TestMergeKeyMapper
  • co.nubetech.hiho.merge.TestMergeKeyReducer
  • co.nubetech.hiho.merge.TestMergeValueMapper
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.