Examples of org.apache.hadoop.mapreduce.Counter

org.apache.hadoop.mapreduce.Counter
A named counter that tracks the progress of a map/reduce job.
Counters represent global counters, defined either by the Map-Reduce framework or applications. Each Counter is named by an {@link Enum} and has a long for the value.
Counters are bunched into Groups, each comprising of counters from a particular Enum class.

      if (start != 0 && now - start <= Integer.MAX_VALUE) {
        splitsBlock.getProgressWallclockTime().extend(newProgress,
            (int) (now - start));
      }


      Counter cpuCounter = counters.findCounter(TaskCounter.CPU_MILLISECONDS);
      if (cpuCounter != null && cpuCounter.getValue() <= Integer.MAX_VALUE) {
        splitsBlock.getProgressCPUTime().extend(newProgress,
            (int) cpuCounter.getValue()); // long to int? TODO: FIX. Same below
      }


      Counter virtualBytes = counters
        .findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES);
      if (virtualBytes != null) {
        splitsBlock.getProgressVirtualMemoryKbytes().extend(newProgress,
            (int) (virtualBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
      }


      Counter physicalBytes = counters
        .findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES);
      if (physicalBytes != null) {
        splitsBlock.getProgressPhysicalMemoryKbytes().extend(newProgress,
            (int) (physicalBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
      }
    }
  }

View Full Code Here

    }
  }


  private void setSummarySlotSeconds(JobSummary summary, Counters allCounters) {


    Counter slotMillisMapCounter = allCounters
      .findCounter(JobCounter.SLOTS_MILLIS_MAPS);
    if (slotMillisMapCounter != null) {
      summary.setMapSlotSeconds(slotMillisMapCounter.getValue());
    }


    Counter slotMillisReduceCounter = allCounters
      .findCounter(JobCounter.SLOTS_MILLIS_REDUCES);
    if (slotMillisReduceCounter != null) {
      summary.setMapSlotSeconds(slotMillisReduceCounter.getValue());
    }
  }

View Full Code Here

  }


  @Test
  public void skip() throws Exception {


    Counter skippedInstances = EasyMock.createMock(Counter.class);


    EasyMock.expect(ctx.getCounter(IndexInstancesMapper.Counter.SKIPPED_INSTANCES)).andReturn(skippedInstances);
    skippedInstances.increment(1);


    EasyMock.replay(ctx, skippedInstances);


    IndexInstancesMapper indexInstances = new IndexInstancesMapper();
    setField(indexInstances, "labelIndex", labelIndex);

View Full Code Here

      if (start != 0 && now - start <= Integer.MAX_VALUE) {
        splitsBlock.getProgressWallclockTime().extend(newProgress,
            (int) (now - start));
      }


      Counter cpuCounter = counters.findCounter(TaskCounter.CPU_MILLISECONDS);
      if (cpuCounter != null && cpuCounter.getValue() <= Integer.MAX_VALUE) {
        splitsBlock.getProgressCPUTime().extend(newProgress,
            (int) cpuCounter.getValue()); // long to int? TODO: FIX. Same below
      }


      Counter virtualBytes = counters
        .findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES);
      if (virtualBytes != null) {
        splitsBlock.getProgressVirtualMemoryKbytes().extend(newProgress,
            (int) (virtualBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
      }


      Counter physicalBytes = counters
        .findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES);
      if (physicalBytes != null) {
        splitsBlock.getProgressPhysicalMemoryKbytes().extend(newProgress,
            (int) (physicalBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
      }
    }
  }

View Full Code Here

     Format decimal = new DecimalFormat();
  
     boolean isFirst = true;
     Iterator<Counter> ctrItr = totalGroup.iterator();
     while(ctrItr.hasNext()) {
       Counter counter = ctrItr.next();
       String name = counter.getName();
       String mapValue = 
        decimal.format(mapGroup.findCounter(name).getValue());
       String reduceValue = 
        decimal.format(reduceGroup.findCounter(name).getValue());
       String totalValue = 
        decimal.format(counter.getValue());


      out.write("\n       <tr>\n");


       if (isFirst) {
         isFirst = false;


      out.write("\n         <td rowspan=\"");
      out.print(totalGroup.size());
      out.write("\">\n         ");
      out.print(HtmlQuoting.quoteHtmlChars(totalGroup.getDisplayName()));
      out.write("</td>\n");


       }


      out.write("\n       <td>");
      out.print(HtmlQuoting.quoteHtmlChars(counter.getDisplayName()));
      out.write("</td>\n       <td align=\"right\">");
      out.print(mapValue);
      out.write("</td>\n       <td align=\"right\">");
      out.print(reduceValue);
      out.write("</td>\n       <td align=\"right\">");

View Full Code Here

            Process proc = startJob(context,
                conf.get("user.name"),
                conf.get(OVERRIDE_CLASSPATH));


            String statusdir = conf.get(STATUSDIR_NAME);
            Counter cnt = context.getCounter(ControllerCounters.SIMPLE_COUNTER);


            ExecutorService pool = Executors.newCachedThreadPool();
            executeWatcher(pool, conf, context.getJobID(),
                proc.getInputStream(), statusdir, STDOUT_FNAME);
            executeWatcher(pool, conf, context.getJobID(),

View Full Code Here

      if (start != 0 && now - start <= Integer.MAX_VALUE) {
        splitsBlock.getProgressWallclockTime().extend(newProgress,
            (int) (now - start));
      }


      Counter cpuCounter = counters.findCounter(TaskCounter.CPU_MILLISECONDS);
      if (cpuCounter != null && cpuCounter.getValue() <= Integer.MAX_VALUE) {
        splitsBlock.getProgressCPUTime().extend(newProgress,
            (int) cpuCounter.getValue()); // long to int? TODO: FIX. Same below
      }


      Counter virtualBytes = counters
        .findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES);
      if (virtualBytes != null) {
        splitsBlock.getProgressVirtualMemoryKbytes().extend(newProgress,
            (int) (virtualBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
      }


      Counter physicalBytes = counters
        .findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES);
      if (physicalBytes != null) {
        splitsBlock.getProgressPhysicalMemoryKbytes().extend(newProgress,
            (int) (physicalBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
      }
    }
  }

View Full Code Here

    Configuration conf = opts.getConfiguration();
    args = opts.getRemainingArgs();
    Job job = RowCounter.createSubmittableJob(conf, args);
    job.waitForCompletion(true);
    assertTrue(job.isSuccessful());
    Counter counter = job.getCounters().findCounter(
        RowCounterMapper.Counters.ROWS);
    assertEquals(expectedCount, counter.getValue());
  }

View Full Code Here


    ScanMetrics scanMetrics = ProtobufUtil.toScanMetrics(serializedMetrics);


    try {
      for (Map.Entry<String, Long> entry:scanMetrics.getMetricsMap().entrySet()) {
        Counter ct = (Counter)this.getCounter.invoke(context,
            HBASE_COUNTER_GROUP_NAME, entry.getKey());


        ct.increment(entry.getValue());
      }


      ((Counter) this.getCounter.invoke(context, HBASE_COUNTER_GROUP_NAME,
          "NUM_SCANNER_RESTARTS")).increment(numRestarts);
    } catch (Exception e) {

View Full Code Here

        throw new IllegalStateException("You should call run() first");
      }


      Counters counters = job.getCounters();


      Counter referenced = counters.findCounter(Counts.REFERENCED);
      Counter unreferenced = counters.findCounter(Counts.UNREFERENCED);
      Counter undefined = counters.findCounter(Counts.UNDEFINED);
      Counter multiref = counters.findCounter(Counts.EXTRAREFERENCES);


      boolean success = true;
      //assert
      if (expectedReferenced != referenced.getValue()) {
        LOG.error("Expected referenced count does not match with actual referenced count. " +
            "expected referenced=" + expectedReferenced + " ,actual=" + referenced.getValue());
        success = false;
      }


      if (unreferenced.getValue() > 0) {
        boolean couldBeMultiRef = (multiref.getValue() == unreferenced.getValue());
        LOG.error("Unreferenced nodes were not expected. Unreferenced count=" + unreferenced.getValue()
            + (couldBeMultiRef ? "; could be due to duplicate random numbers" : ""));
        success = false;
      }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapreduce.Counter

co.nubetech.hiho.dedup.TestDedupKeyMapper

co.nubetech.hiho.dedup.TestDedupKeyReducer

co.nubetech.hiho.dedup.TestDedupValueMapper

co.nubetech.hiho.dedup.TestDedupValueReducer

co.nubetech.hiho.merge.TestMergeKeyMapper

co.nubetech.hiho.merge.TestMergeKeyReducer

co.nubetech.hiho.merge.TestMergeValueMapper

co.nubetech.hiho.merge.TestMergeValueReducer

com.asakusafw.runtime.compatibility.JobCompatibility$MockStatusReporter

com.ebay.erl.mobius.core.mapred.CounterUpdateThread

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.