Examples of org.apache.hadoop.mapreduce.Counter

org.apache.hadoop.mapreduce.Counter
A named counter that tracks the progress of a map/reduce job.
Counters represent global counters, defined either by the Map-Reduce framework or applications. Each Counter is named by an {@link Enum} and has a long for the value.
Counters are bunched into Groups, each comprising of counters from a particular Enum class.

        throw new IllegalStateException("You should call run() first");
      }


      Counters counters = job.getCounters();


      Counter referenced = counters.findCounter(Counts.REFERENCED);
      Counter unreferenced = counters.findCounter(Counts.UNREFERENCED);
      Counter undefined = counters.findCounter(Counts.UNDEFINED);
      Counter multiref = counters.findCounter(Counts.EXTRAREFERENCES);


      boolean success = true;
      //assert
      if (expectedReferenced != referenced.getValue()) {
        LOG.error("Expected referenced count does not match with actual referenced count. " +
            "expected referenced=" + expectedReferenced + " ,actual=" + referenced.getValue());
        success = false;
      }


      if (unreferenced.getValue() > 0) {
        boolean couldBeMultiRef = (multiref.getValue() == unreferenced.getValue());
        LOG.error("Unreferenced nodes were not expected. Unreferenced count=" + unreferenced.getValue()
            + (couldBeMultiRef ? "; could be due to duplicate random numbers" : ""));
        success = false;
      }

View Full Code Here


    @Override
    protected void map(ImmutableBytesWritable key, Result value, Context context)
        throws IOException, InterruptedException {
      byte[] row = value.getRow();
      Counter c = getCounter(row);
      c.increment(1);
    }

View Full Code Here

      Counter c = getCounter(row);
      c.increment(1);
    }


    private Counter getCounter(byte[] row) {
      Counter c = null;
      if (Bytes.indexOf(row, Bytes.toBytes(VISIBILITY_EXPS[0])) != -1) {
        c = rowsExp1;
      } else if (Bytes.indexOf(row, Bytes.toBytes(VISIBILITY_EXPS[1])) != -1) {
        c = rowsExp2;
      } else if (Bytes.indexOf(row, Bytes.toBytes(VISIBILITY_EXPS[2])) != -1) {

View Full Code Here

    @SuppressWarnings("unchecked")
    public void warn(Object o, String msg, Enum warningEnum) {
        String displayMessage = o.getClass().getName() + ": " + msg;
        if(aggregate) {
            if(taskIOContext != null) {
                Counter c = taskIOContext.getCounter(warningEnum);
                c.increment(1);
            } else {
                //TODO:
                //in local mode of execution if the PigHadoopLogger is used initially,
                //then aggregation cannot be performed as the reporter will be null. 
                //The reference to a reporter is given by Hadoop at run time.

View Full Code Here

        throw new IllegalStateException("You should call run() first");
      }


      Counters counters = job.getCounters();


      Counter referenced = counters.findCounter(Counts.REFERENCED);
      Counter unreferenced = counters.findCounter(Counts.UNREFERENCED);
      Counter undefined = counters.findCounter(Counts.UNDEFINED);


      boolean success = true;
      //assert
      if (expectedReferenced != referenced.getValue()) {
        LOG.error("Expected referenced count does not match with actual referenced count. " +
            "expected referenced=" + expectedReferenced + " ,actual=" + referenced.getValue());
        success = false;
      }


      if (unreferenced.getValue() > 0) {
        LOG.error("Unreferenced nodes were not expected. Unreferenced count=" + unreferenced.getValue());
        success = false;
      }


      if (undefined.getValue() > 0) {
        LOG.error("Found an undefined node. Undefined count=" + undefined.getValue());
        success = false;
      }


      return success;
    }

View Full Code Here

  }


  private static void accumulateCounters(
    Counters aggregate, Counters increment) {
    for (JobInProgress.Counter key : JobInProgress.Counter.values()) {
      Counter counter = increment.findCounter(key);
      if (counter != null) {
        aggregate.findCounter(key).increment(counter.getValue());
      }
    }
    for (Task.Counter key : Task.Counter.values()) {
      Counter counter = increment.findCounter(key);
      if (counter != null) {
        aggregate.findCounter(key).increment(counter.getValue());
      }
    }
    for (Counters.Counter counter :
      increment.getGroup(Task.FILESYSTEM_COUNTER_GROUP)) {
      aggregate.incrCounter(
        Task.FILESYSTEM_COUNTER_GROUP, counter.getName(), counter.getValue());
    }
  }

View Full Code Here

    // do nothing.
  }


  @Override
  public Counter getCounter(String group, String name) {
    Counter counter = null;
    if (counters != null) {
      counter = counters.findCounter(group, name);
    }


    return counter;

View Full Code Here

    return counter;
  }


  @Override
  public Counter getCounter(Enum key) {
    Counter counter = null;
    if (counters != null) {
      counter = counters.findCounter(key);
    }


    return counter;

View Full Code Here

    conf.setBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, true);
    TaskAttemptID taskid = new TaskAttemptID();


    RawKeyValueIterator input = new FakeRawKeyValueIterator();


    Counter counter = new GenericCounter();
    Counter inputValueCounter = new GenericCounter();
    LoadRecordWriter output = new LoadRecordWriter();


    OutputCommitter committer = new CustomOutputCommitter();


    StatusReporter reporter = new DummyReporter();
    RawComparator<GridmixKey> comparator = new FakeRawComparator();


    ReduceContext<GridmixKey, GridmixRecord, NullWritable, GridmixRecord> reduceContext = new ReduceContextImpl<GridmixKey, GridmixRecord, NullWritable, GridmixRecord>(
            conf, taskid, input, counter, inputValueCounter, output, committer,
            reporter, comparator, GridmixKey.class, GridmixRecord.class);
    // read for previous data
    reduceContext.nextKeyValue();
    org.apache.hadoop.mapreduce.Reducer<GridmixKey, GridmixRecord, NullWritable, GridmixRecord>.Context context = new WrappedReducer<GridmixKey, GridmixRecord, NullWritable, GridmixRecord>()
            .getReducerContext(reduceContext);


    // test.setup(context);
    test.run(context);
    // have been readed 9 records (-1 for previous)
    assertEquals(9, counter.getValue());
    assertEquals(10, inputValueCounter.getValue());
    assertEquals(1, output.getData().size());
    GridmixRecord record = output.getData().values().iterator()
            .next();


    assertEquals(1593, record.getSize());

View Full Code Here

    conf.setBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, true);
    TaskAttemptID taskId = new TaskAttemptID();


    RawKeyValueIterator input = new FakeRawKeyValueReducerIterator();


    Counter counter = new GenericCounter();
    Counter inputValueCounter = new GenericCounter();
    RecordWriter<NullWritable, NullWritable> output = new LoadRecordReduceWriter();


    OutputCommitter committer = new CustomOutputCommitter();


    StatusReporter reporter = new DummyReporter();

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapreduce.Counter

co.nubetech.hiho.dedup.TestDedupKeyMapper

co.nubetech.hiho.dedup.TestDedupKeyReducer

co.nubetech.hiho.dedup.TestDedupValueMapper

co.nubetech.hiho.dedup.TestDedupValueReducer

co.nubetech.hiho.merge.TestMergeKeyMapper

co.nubetech.hiho.merge.TestMergeKeyReducer

co.nubetech.hiho.merge.TestMergeValueMapper

co.nubetech.hiho.merge.TestMergeValueReducer

com.asakusafw.runtime.compatibility.JobCompatibility$MockStatusReporter

com.ebay.erl.mobius.core.mapred.CounterUpdateThread

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.