Examples of org.apache.hadoop.mapreduce.Counter

org.apache.hadoop.mapreduce.Counter
A named counter that tracks the progress of a map/reduce job.
Counters represent global counters, defined either by the Map-Reduce framework or applications. Each Counter is named by an {@link Enum} and has a long for the value.
Counters are bunched into Groups, each comprising of counters from a particular Enum class.

    try {
      for (String cgName : job.getCounters().getGroupNames()) {
        CounterGroup group = job.getCounters().getGroup(cgName);
        Iterator<Counter> ci = group.iterator();
        while (ci.hasNext()) {
          Counter c = ci.next();
          executionContext.put(group.getDisplayName().trim() + "::" + c.getDisplayName().trim(), c.getValue());
        }
      }
    } catch (Exception ignore) {}
  }

View Full Code Here

    // the system will produce individual mocks for each counter so we can watch what
    // happens with counters
    //


    private void incrementCounter(Context context,Enum <?> counterId,long amount) {
        Counter counter=context.getCounter(counterId);
        if(counter!=null) {
            counter.increment(amount);
        };
    };

View Full Code Here

    // the system will produce individual mocks for each counter so we can watch what
    // happens with counters
    //
    
    private void incrementCounter(Context context,Enum <?> counterId,long amount) {
        Counter counter=context.getCounter(counterId);
        if(counter!=null) {
            counter.increment(amount);
        };
    };

View Full Code Here

    // the system will produce individual mocks for each counter so we can watch what
    // happens with counters
    //


    private void incrementCounter(Context context,Enum <?> counterId,long amount) {
        Counter counter=context.getCounter(counterId);
        if(counter!=null) {
            counter.increment(amount);
        };
    };

View Full Code Here

            return getCounter(name.getDeclaringClass().getName(), name.name());
        }


        @Override
        public Counter getCounter(String group, String name) {
            return new Counter() {
                // empty
            };
        }

View Full Code Here

      if (start != 0 && now - start <= Integer.MAX_VALUE) {
        splitsBlock.getProgressWallclockTime().extend(newProgress,
            (int) (now - start));
      }


      Counter cpuCounter = counters.findCounter(TaskCounter.CPU_MILLISECONDS);
      if (cpuCounter != null && cpuCounter.getValue() <= Integer.MAX_VALUE) {
        splitsBlock.getProgressCPUTime().extend(newProgress,
            (int) cpuCounter.getValue()); // long to int? TODO: FIX. Same below
      }


      Counter virtualBytes = counters
        .findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES);
      if (virtualBytes != null) {
        splitsBlock.getProgressVirtualMemoryKbytes().extend(newProgress,
            (int) (virtualBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
      }


      Counter physicalBytes = counters
        .findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES);
      if (physicalBytes != null) {
        splitsBlock.getProgressPhysicalMemoryKbytes().extend(newProgress,
            (int) (physicalBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
      }
    }
  }

View Full Code Here

    MetricsTimeVaryingLong[] mlvs =
      scanMetrics.getMetricsTimeVaryingLongArray();


    try {
      for (MetricsTimeVaryingLong mlv : mlvs) {
        Counter ct = (Counter)this.getCounter.invoke(context,
          HBASE_COUNTER_GROUP_NAME, mlv.getName());
        ct.increment(mlv.getCurrentIntervalValue());
      }
      ((Counter) this.getCounter.invoke(context, HBASE_COUNTER_GROUP_NAME,
          "NUM_SCANNER_RESTARTS")).increment(numRestarts);
    } catch (Exception e) {
      LOG.debug("can't update counter." + StringUtils.stringifyException(e));

View Full Code Here

        throw new IllegalStateException("You should call run() first");
      }


      Counters counters = job.getCounters();


      Counter referenced = counters.findCounter(Counts.REFERENCED);
      Counter unreferenced = counters.findCounter(Counts.UNREFERENCED);
      Counter undefined = counters.findCounter(Counts.UNDEFINED);
      Counter multiref = counters.findCounter(Counts.EXTRAREFERENCES);


      boolean success = true;
      //assert
      if (expectedReferenced != referenced.getValue()) {
        LOG.error("Expected referenced count does not match with actual referenced count. " +
            "expected referenced=" + expectedReferenced + " ,actual=" + referenced.getValue());
        success = false;
      }


      if (unreferenced.getValue() > 0) {
        boolean couldBeMultiRef = (multiref.getValue() == unreferenced.getValue());
        LOG.error("Unreferenced nodes were not expected. Unreferenced count=" + unreferenced.getValue()
            + (couldBeMultiRef ? "; could be due to duplicate random numbers" : ""));
        success = false;
      }

View Full Code Here

        _writer = new IndexWriter(_finalDir, _conf.clone());
      }
    }


    private Counter emptyCounter() {
      return new Counter() {
      };
    }

View Full Code Here

      MetricsTimeVaryingLong[] mlvs =
        scanMetrics.getMetricsTimeVaryingLongArray();


      try {
        for (MetricsTimeVaryingLong mlv : mlvs) {
          Counter ct = (Counter)this.getCounter.invoke(context,
            HBASE_COUNTER_GROUP_NAME, mlv.getName());
          ct.increment(mlv.getCurrentIntervalValue());
        }
      } catch (Exception e) {
        LOG.debug("can't update counter." + StringUtils.stringifyException(e));
      }
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapreduce.Counter

co.nubetech.hiho.dedup.TestDedupKeyMapper

co.nubetech.hiho.dedup.TestDedupKeyReducer

co.nubetech.hiho.dedup.TestDedupValueMapper

co.nubetech.hiho.dedup.TestDedupValueReducer

co.nubetech.hiho.merge.TestMergeKeyMapper

co.nubetech.hiho.merge.TestMergeKeyReducer

co.nubetech.hiho.merge.TestMergeValueMapper

co.nubetech.hiho.merge.TestMergeValueReducer

com.asakusafw.runtime.compatibility.JobCompatibility$MockStatusReporter

com.ebay.erl.mobius.core.mapred.CounterUpdateThread

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.