Examples of org.apache.hadoop.mapreduce.Counter

org.apache.hadoop.mapreduce.Counter
A named counter that tracks the progress of a map/reduce job.
Counters represent global counters, defined either by the Map-Reduce framework or applications. Each Counter is named by an {@link Enum} and has a long for the value.
Counters are bunched into Groups, each comprising of counters from a particular Enum class.

   */
  @Test
  public void testToUserVectorReducerWithBooleanData() throws Exception {
    Reducer<VarLongWritable,VarLongWritable,VarLongWritable,VectorWritable>.Context context =
      EasyMock.createMock(Reducer.Context.class);
    Counter userCounters = EasyMock.createMock(Counter.class);


    EasyMock.expect(context.getCounter(ToUserVectorsReducer.Counters.USERS)).andReturn(userCounters);
    userCounters.increment(1);
    context.write(EasyMock.eq(new VarLongWritable(12L)), MathHelper.vectorMatches(
        MathHelper.elem(TasteHadoopUtils.idToIndex(34L), 1.0), MathHelper.elem(TasteHadoopUtils.idToIndex(56L), 1.0)));


    EasyMock.replay(context, userCounters);

View Full Code Here

    }
  }


  private void setSummarySlotSeconds(JobSummary summary, Counters allCounters) {


    Counter slotMillisMapCounter = allCounters
      .findCounter(JobCounter.SLOTS_MILLIS_MAPS);
    if (slotMillisMapCounter != null) {
      summary.setMapSlotSeconds(slotMillisMapCounter.getValue() / 1000);
    }


    Counter slotMillisReduceCounter = allCounters
      .findCounter(JobCounter.SLOTS_MILLIS_REDUCES);
    if (slotMillisReduceCounter != null) {
      summary.setReduceSlotSeconds(slotMillisReduceCounter.getValue() / 1000);
    }
  }

View Full Code Here

      if (start != 0 && now - start <= Integer.MAX_VALUE) {
        splitsBlock.getProgressWallclockTime().extend(newProgress,
            (int) (now - start));
      }


      Counter cpuCounter = counters.findCounter(TaskCounter.CPU_MILLISECONDS);
      if (cpuCounter != null && cpuCounter.getValue() <= Integer.MAX_VALUE) {
        splitsBlock.getProgressCPUTime().extend(newProgress,
            (int) cpuCounter.getValue()); // long to int? TODO: FIX. Same below
      }


      Counter virtualBytes = counters
        .findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES);
      if (virtualBytes != null) {
        splitsBlock.getProgressVirtualMemoryKbytes().extend(newProgress,
            (int) (virtualBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
      }


      Counter physicalBytes = counters
        .findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES);
      if (physicalBytes != null) {
        splitsBlock.getProgressPhysicalMemoryKbytes().extend(newProgress,
            (int) (physicalBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
      }
    }
  }

View Full Code Here

  @Override
  @SuppressWarnings("unchecked")
  public void write(DataOutput out) throws IOException {
    WritableUtils.writeVInt(out, size());
    for (int i = 0; i < counters.length; ++i) {
      Counter counter = (C) counters[i];
      if (counter != null) {
        WritableUtils.writeVInt(out, i);
        WritableUtils.writeVLong(out, counter.getValue());
      }
    }
  }

View Full Code Here

    clear();
    int len = WritableUtils.readVInt(in);
    T[] enums = enumClass.getEnumConstants();
    for (int i = 0; i < len; ++i) {
      int ord = WritableUtils.readVInt(in);
      Counter counter = newCounter(enums[ord]);
      counter.setValue(WritableUtils.readVLong(in));
      counters[ord] = counter;
    }
  }

View Full Code Here

              a(url(urlBase,urlId,g.getName(), 
                  counter.getName()), counter.getDisplayName()).
            _();
          }
        if (map != null) {
          Counter mc = mg == null ? null : mg.findCounter(counter.getName());
          Counter rc = rg == null ? null : rg.findCounter(counter.getName());
          groupRow.
            td(mc == null ? "0" : String.valueOf(mc.getValue())).
            td(rc == null ? "0" : String.valueOf(rc.getValue()));
        }
        groupRow.td(String.valueOf(counter.getValue()))._();
      }
      group._()._()._()._();
    }

View Full Code Here

    }
  }


  private void setSummarySlotSeconds(JobSummary summary, Counters allCounters) {


    Counter slotMillisMapCounter = allCounters
      .findCounter(JobCounter.SLOTS_MILLIS_MAPS);
    if (slotMillisMapCounter != null) {
      summary.setMapSlotSeconds(slotMillisMapCounter.getValue() / 1000);
    }


    Counter slotMillisReduceCounter = allCounters
      .findCounter(JobCounter.SLOTS_MILLIS_REDUCES);
    if (slotMillisReduceCounter != null) {
      summary.setReduceSlotSeconds(slotMillisReduceCounter.getValue() / 1000);
    }
  }

View Full Code Here


    ScanMetrics scanMetrics = ProtobufUtil.toScanMetrics(serializedMetrics);


    try {
      for (Map.Entry<String, Long> entry:scanMetrics.getMetricsMap().entrySet()) {
        Counter ct = (Counter)this.getCounter.invoke(context,
            HBASE_COUNTER_GROUP_NAME, entry.getKey());


        ct.increment(entry.getValue());
      }


      ((Counter) this.getCounter.invoke(context, HBASE_COUNTER_GROUP_NAME,
          "NUM_SCANNER_RESTARTS")).increment(numRestarts);
    } catch (Exception e) {

View Full Code Here

      if (start != 0 && now - start <= Integer.MAX_VALUE) {
        splitsBlock.getProgressWallclockTime().extend(newProgress,
            (int) (now - start));
      }


      Counter cpuCounter = counters.findCounter(TaskCounter.CPU_MILLISECONDS);
      if (cpuCounter != null && cpuCounter.getValue() <= Integer.MAX_VALUE) {
        splitsBlock.getProgressCPUTime().extend(newProgress,
            (int) cpuCounter.getValue()); // long to int? TODO: FIX. Same below
      }


      Counter virtualBytes = counters
        .findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES);
      if (virtualBytes != null) {
        splitsBlock.getProgressVirtualMemoryKbytes().extend(newProgress,
            (int) (virtualBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
      }


      Counter physicalBytes = counters
        .findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES);
      if (physicalBytes != null) {
        splitsBlock.getProgressPhysicalMemoryKbytes().extend(newProgress,
            (int) (physicalBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
      }
    }
  }

View Full Code Here

      return;
    }


    try {
      for (Map.Entry<String, Long> entry:scanMetrics.getMetricsMap().entrySet()) {
        Counter ct = (Counter)getCounter.invoke(context,
            HBASE_COUNTER_GROUP_NAME, entry.getKey());


        ct.increment(entry.getValue());
      }
      ((Counter) getCounter.invoke(context, HBASE_COUNTER_GROUP_NAME,
          "NUM_SCANNER_RESTARTS")).increment(numScannerRestarts);
      ((Counter) getCounter.invoke(context, HBASE_COUNTER_GROUP_NAME,
          "NUM_SCAN_RESULTS_STALE")).increment(numStale);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapreduce.Counter

co.nubetech.hiho.dedup.TestDedupKeyMapper

co.nubetech.hiho.dedup.TestDedupKeyReducer

co.nubetech.hiho.dedup.TestDedupValueMapper

co.nubetech.hiho.dedup.TestDedupValueReducer

co.nubetech.hiho.merge.TestMergeKeyMapper

co.nubetech.hiho.merge.TestMergeKeyReducer

co.nubetech.hiho.merge.TestMergeValueMapper

co.nubetech.hiho.merge.TestMergeValueReducer

com.asakusafw.runtime.compatibility.JobCompatibility$MockStatusReporter

com.ebay.erl.mobius.core.mapred.CounterUpdateThread

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.