Examples of org.apache.hadoop.mapreduce.Counter

org.apache.hadoop.mapreduce.Counter
A named counter that tracks the progress of a map/reduce job.
Counters represent global counters, defined either by the Map-Reduce framework or applications. Each Counter is named by an {@link Enum} and has a long for the value.
Counters are bunched into Groups, each comprising of counters from a particular Enum class.

        throw new IllegalStateException("You should call run() first");
      }


      Counters counters = job.getCounters();


      Counter referenced = counters.findCounter(Counts.REFERENCED);
      Counter unreferenced = counters.findCounter(Counts.UNREFERENCED);
      Counter undefined = counters.findCounter(Counts.UNDEFINED);
      Counter multiref = counters.findCounter(Counts.EXTRAREFERENCES);


      boolean success = true;
      //assert
      if (expectedReferenced != referenced.getValue()) {
        LOG.error("Expected referenced count does not match with actual referenced count. " +
            "expected referenced=" + expectedReferenced + " ,actual=" + referenced.getValue());
        success = false;
      }


      if (unreferenced.getValue() > 0) {
        boolean couldBeMultiRef = (multiref.getValue() == unreferenced.getValue());
        LOG.error("Unreferenced nodes were not expected. Unreferenced count=" + unreferenced.getValue()
            + (couldBeMultiRef ? "; could be due to duplicate random numbers" : ""));
        success = false;
      }

View Full Code Here


    @Override
    protected void map(ImmutableBytesWritable key, Result value, Context context)
        throws IOException, InterruptedException {
      byte[] row = value.getRow();
      Counter c = getCounter(row);
      c.increment(1);
    }

View Full Code Here

      Counter c = getCounter(row);
      c.increment(1);
    }


    private Counter getCounter(byte[] row) {
      Counter c = null;
      if (Bytes.indexOf(row, Bytes.toBytes(VISIBILITY_EXPS[0])) != -1) {
        c = rowsExp1;
      } else if (Bytes.indexOf(row, Bytes.toBytes(VISIBILITY_EXPS[1])) != -1) {
        c = rowsExp2;
      } else if (Bytes.indexOf(row, Bytes.toBytes(VISIBILITY_EXPS[2])) != -1) {

View Full Code Here

      throw new IllegalStateException("You should call run() first");
    }
    
    Counters counters = job.getCounters();
    
    Counter referenced = counters.findCounter(Counts.REFERENCED);
    Counter unreferenced = counters.findCounter(Counts.UNREFERENCED);
    Counter undefined = counters.findCounter(Counts.UNDEFINED);
    
    boolean success = true;
    //assert
    if (expectedReferenced != referenced.getValue()) {
      LOG.error("Expected referenced count does not match with actual referenced count. " +
          "expected referenced=" + expectedReferenced + " ,actual=" + referenced.getValue());
      success = false;
    }


    if (unreferenced.getValue() > 0) { 
      LOG.error("Unreferenced nodes were not expected. Unreferenced count=" + unreferenced.getValue());
      success = false;
    }
    
    if (undefined.getValue() > 0) { 
      LOG.error("Found an undefined node. Undefined count=" + undefined.getValue());
      success = false;
    }
    
    return success;
  }

View Full Code Here

        throw new IllegalStateException("You should call run() first");
      }


      Counters counters = job.getCounters();


      Counter referenced = counters.findCounter(Counts.REFERENCED);
      Counter unreferenced = counters.findCounter(Counts.UNREFERENCED);
      Counter undefined = counters.findCounter(Counts.UNDEFINED);
      Counter multiref = counters.findCounter(Counts.EXTRAREFERENCES);


      boolean success = true;
      //assert
      if (expectedReferenced != referenced.getValue()) {
        LOG.error("Expected referenced count does not match with actual referenced count. " +
            "expected referenced=" + expectedReferenced + " ,actual=" + referenced.getValue());
        success = false;
      }


      if (unreferenced.getValue() > 0) {
        boolean couldBeMultiRef = (multiref.getValue() == unreferenced.getValue());
        LOG.error("Unreferenced nodes were not expected. Unreferenced count=" + unreferenced.getValue()
            + (couldBeMultiRef ? "; could be due to duplicate random numbers" : ""));
        success = false;
      }

View Full Code Here

    public void incrementGcCounter() {
      if (null == counters) {
        return; // nothing to do.
      }


      Counter gcCounter = counters.findCounter(TaskCounter.GC_TIME_MILLIS);
      if (null != gcCounter) {
        gcCounter.increment(getElapsedGc());
      }
    }

View Full Code Here

    Counters result = new Counters();
    for (JhCounterGroup g : counters.groups) {
      CounterGroup group =
        new CounterGroup(g.name.toString(), g.displayName.toString());
      for (JhCounter c : g.counts) {
        group.addCounter(new Counter(c.name.toString(),
                                     c.displayName.toString(),
                                     c.value));
      }
      result.addGroup(group);
    }

View Full Code Here

    }
  }


  private void setSummarySlotSeconds(JobSummary summary, Counters allCounters) {


    Counter slotMillisMapCounter = allCounters
      .findCounter(JobCounter.SLOTS_MILLIS_MAPS);
    if (slotMillisMapCounter != null) {
      summary.setMapSlotSeconds(slotMillisMapCounter.getValue() / 1000);
    }


    Counter slotMillisReduceCounter = allCounters
      .findCounter(JobCounter.SLOTS_MILLIS_REDUCES);
    if (slotMillisReduceCounter != null) {
      summary.setReduceSlotSeconds(slotMillisReduceCounter.getValue() / 1000);
    }
  }

View Full Code Here

      if (start != 0 && now - start <= Integer.MAX_VALUE) {
        splitsBlock.getProgressWallclockTime().extend(newProgress,
            (int) (now - start));
      }


      Counter cpuCounter = counters.findCounter(TaskCounter.CPU_MILLISECONDS);
      if (cpuCounter != null && cpuCounter.getValue() <= Integer.MAX_VALUE) {
        splitsBlock.getProgressCPUTime().extend(newProgress,
            (int) cpuCounter.getValue()); // long to int? TODO: FIX. Same below
      }


      Counter virtualBytes = counters
        .findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES);
      if (virtualBytes != null) {
        splitsBlock.getProgressVirtualMemoryKbytes().extend(newProgress,
            (int) (virtualBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
      }


      Counter physicalBytes = counters
        .findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES);
      if (physicalBytes != null) {
        splitsBlock.getProgressPhysicalMemoryKbytes().extend(newProgress,
            (int) (physicalBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
      }
    }
  }

View Full Code Here

        throw new IllegalStateException("You should call run() first");
      }


      Counters counters = job.getCounters();


      Counter referenced = counters.findCounter(Counts.REFERENCED);
      Counter unreferenced = counters.findCounter(Counts.UNREFERENCED);
      Counter undefined = counters.findCounter(Counts.UNDEFINED);


      boolean success = true;
      //assert
      if (expectedReferenced != referenced.getValue()) {
        LOG.error("Expected referenced count does not match with actual referenced count. " +
            "expected referenced=" + expectedReferenced + " ,actual=" + referenced.getValue());
        success = false;
      }


      if (unreferenced.getValue() > 0) {
        LOG.error("Unreferenced nodes were not expected. Unreferenced count=" + unreferenced.getValue());
        success = false;
      }


      if (undefined.getValue() > 0) {
        LOG.error("Found an undefined node. Undefined count=" + undefined.getValue());
        success = false;
      }


      return success;
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapreduce.Counter

co.nubetech.hiho.dedup.TestDedupKeyMapper

co.nubetech.hiho.dedup.TestDedupKeyReducer

co.nubetech.hiho.dedup.TestDedupValueMapper

co.nubetech.hiho.dedup.TestDedupValueReducer

co.nubetech.hiho.merge.TestMergeKeyMapper

co.nubetech.hiho.merge.TestMergeKeyReducer

co.nubetech.hiho.merge.TestMergeValueMapper

co.nubetech.hiho.merge.TestMergeValueReducer

com.asakusafw.runtime.compatibility.JobCompatibility$MockStatusReporter

com.ebay.erl.mobius.core.mapred.CounterUpdateThread

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.