Package org.apache.hadoop.mapreduce

Examples of org.apache.hadoop.mapreduce.Counters.findCounter()

208
209
210
211
212
213
214
215
216
217
218
    boolean succeeded = job.waitForCompletion(true);
    Assert.assertTrue(succeeded);
    Assert.assertEquals(JobStatus.State.SUCCEEDED, job.getJobState());
    Counters counters = job.getCounters();
    Assert.assertEquals(2, counters.findCounter(JobCounter.TOTAL_LAUNCHED_MAPS)
            .getValue());
    Assert.assertEquals(2, counters.findCounter(JobCounter.TOTAL_LAUNCHED_REDUCES)
            .getValue());
    Assert.assertEquals(0, counters.findCounter(JobCounter.NUM_FAILED_MAPS)
            .getValue());
View Full Code Here
210
211
212
213
214
215
216
217
218
219
220
    Assert.assertTrue(succeeded);
    Assert.assertEquals(JobStatus.State.SUCCEEDED, job.getJobState());
    Counters counters = job.getCounters();
    Assert.assertEquals(2, counters.findCounter(JobCounter.TOTAL_LAUNCHED_MAPS)
            .getValue());
    Assert.assertEquals(2, counters.findCounter(JobCounter.TOTAL_LAUNCHED_REDUCES)
            .getValue());
    Assert.assertEquals(0, counters.findCounter(JobCounter.NUM_FAILED_MAPS)
            .getValue());

    /*----------------------------------------------------------------------
 
View Full Code Here
212
213
214
215
216
217
218
219
220
221
222
    Counters counters = job.getCounters();
    Assert.assertEquals(2, counters.findCounter(JobCounter.TOTAL_LAUNCHED_MAPS)
            .getValue());
    Assert.assertEquals(2, counters.findCounter(JobCounter.TOTAL_LAUNCHED_REDUCES)
            .getValue());
    Assert.assertEquals(0, counters.findCounter(JobCounter.NUM_FAILED_MAPS)
            .getValue());

    /*----------------------------------------------------------------------
     * Test that Mapper speculates if MAP_SPECULATIVE is true and
     * REDUCE_SPECULATIVE is false.
View Full Code Here
228
229
230
231
232
233
234
235
236
237
238
    Assert.assertTrue(succeeded);
    Assert.assertEquals(JobStatus.State.SUCCEEDED, job.getJobState());
    counters = job.getCounters();

    // The long-running map will be killed and a new one started.
    Assert.assertEquals(3, counters.findCounter(JobCounter.TOTAL_LAUNCHED_MAPS)
            .getValue());
    Assert.assertEquals(2, counters.findCounter(JobCounter.TOTAL_LAUNCHED_REDUCES)
            .getValue());
    Assert.assertEquals(1, counters.findCounter(JobCounter.NUM_FAILED_MAPS)
            .getValue());
View Full Code Here
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
      if (start != 0 && now - start <= Integer.MAX_VALUE) {
        splitsBlock.getProgressWallclockTime().extend(newProgress,
            (int) (now - start));
      }

      Counter cpuCounter = counters.findCounter(TaskCounter.CPU_MILLISECONDS);
      if (cpuCounter != null && cpuCounter.getValue() <= Integer.MAX_VALUE) {
        splitsBlock.getProgressCPUTime().extend(newProgress,
            (int) cpuCounter.getValue()); // long to int? TODO: FIX. Same below
      }
View Full Code Here
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
      if (cpuCounter != null && cpuCounter.getValue() <= Integer.MAX_VALUE) {
        splitsBlock.getProgressCPUTime().extend(newProgress,
            (int) cpuCounter.getValue()); // long to int? TODO: FIX. Same below
      }

      Counter virtualBytes = counters
        .findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES);
      if (virtualBytes != null) {
        splitsBlock.getProgressVirtualMemoryKbytes().extend(newProgress,
            (int) (virtualBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
      }
View Full Code Here
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
      if (virtualBytes != null) {
        splitsBlock.getProgressVirtualMemoryKbytes().extend(newProgress,
            (int) (virtualBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
      }

      Counter physicalBytes = counters
        .findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES);
      if (physicalBytes != null) {
        splitsBlock.getProgressPhysicalMemoryKbytes().extend(newProgress,
            (int) (physicalBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
      }
View Full Code Here
592
593
594
595
596
597
598
599
600
601
602
      //  get the resource usage metrics
      ResourceUsageMetrics metrics = attempt.getResourceUsageMetrics();
     
      //  check with the actual values
      testResourceUsageMetricViaDeepCompare(metrics,
          counters.findCounter(TaskCounter.CPU_MILLISECONDS).getValue(),
          counters.findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES).getValue(),
          counters.findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES).getValue(),
          counters.findCounter(TaskCounter.COMMITTED_HEAP_BYTES).getValue(),
          true);
    } finally {
View Full Code Here
593
594
595
596
597
598
599
600
601
602
603
      ResourceUsageMetrics metrics = attempt.getResourceUsageMetrics();
     
      //  check with the actual values
      testResourceUsageMetricViaDeepCompare(metrics,
          counters.findCounter(TaskCounter.CPU_MILLISECONDS).getValue(),
          counters.findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES).getValue(),
          counters.findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES).getValue(),
          counters.findCounter(TaskCounter.COMMITTED_HEAP_BYTES).getValue(),
          true);
    } finally {
      // stop the MR cluster
View Full Code Here
594
595
596
597
598
599
600
601
602
603
604
     
      //  check with the actual values
      testResourceUsageMetricViaDeepCompare(metrics,
          counters.findCounter(TaskCounter.CPU_MILLISECONDS).getValue(),
          counters.findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES).getValue(),
          counters.findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES).getValue(),
          counters.findCounter(TaskCounter.COMMITTED_HEAP_BYTES).getValue(),
          true);
    } finally {
      // stop the MR cluster
      mrCluster.shutdown();
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.