Package org.apache.hadoop.mapred.Counters

Examples of org.apache.hadoop.mapred.Counters.Group

1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
          if (counters == null) {
            System.out.println("Counters not available for retired job " +
                jobid);
            exitCode = -1;
          } else {
            Group group = counters.getGroup(counterGroupName);
            Counter counter = group.getCounterForName(counterName);
            System.out.println(counter.getCounter());
            exitCode = 0;
          }
        }
      } else if (killJob) {
View Full Code Here
300
301
302
303
304
305
306
307
308
309
310
311
312
    @SuppressWarnings("deprecation")
    private static void parseAndAddJobCounters(Map<String, String> job, String counters) {
        try {
            Counters counterGroups = Counters.fromEscapedCompactString(counters);
            for (Group otherGroup : counterGroups) {
                Group group = counterGroups.getGroup(otherGroup.getName());
                for (Counter otherCounter : otherGroup) {
                    Counter counter = group.getCounterForName(otherCounter.getName());
                    job.put(otherCounter.getName(), String.valueOf(counter.getValue()));
                }
            }
        } catch (ParseException e) {
           LOG.warn("Failed to parse job counters", e);
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
65
66
      outFile.delete();
      assertEquals(outputExpect, output);
     
      Counters counters = job.running_.getCounters();
      assertNotNull("Counters", counters);
      Group group = counters.getGroup("UserCounters");
      assertNotNull("Group", group);
      Counter counter = group.getCounterForName("InputLines");
      assertNotNull("Counter", counter);
      assertEquals(3, counter.getCounter());
    } finally {
      INPUT_FILE.delete();
      FileUtil.fullyDelete(OUTPUT_DIR.getAbsoluteFile());
View Full Code Here
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
        RunningJob job = getJob(JobID.forName(jobid));
        if (job == null) {
          System.out.println("Could not find job " + jobid);
        } else {
          Counters counters = job.getCounters();
          Group group = counters.getGroup(counterGroupName);
          Counter counter = group.getCounterForName(counterName);
          System.out.println(counter.getCounter());
          exitCode = 0;
        }
      } else if (killJob) {
        RunningJob job = getJob(JobID.forName(jobid));
View Full Code Here
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
          if (counters == null) {
            System.out.println("Counters not available for retired job " +
                jobid);
            exitCode = -1;
          } else {
            Group group = counters.getGroup(counterGroupName);
            Counter counter = group.getCounterForName(counterName);
            System.out.println(counter.getCounter());
            exitCode = 0;
          }
        }
      } else if (killJob) {
View Full Code Here
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
        RunningJob job = getJob(JobID.forName(jobid));
        if (job == null) {
          System.out.println("Could not find job " + jobid);
        } else {
          Counters counters = job.getCounters();
          Group group = counters.getGroup(counterGroupName);
          Counter counter = group.getCounterForName(counterName);
          System.out.println(counter.getCounter());
          exitCode = 0;
        }
      } else if (killJob) {
        RunningJob job = getJob(JobID.forName(jobid));
View Full Code Here
247
248
249
250
251
252
253
254
255
256
257
258
259
        job.setMapperClass(DBFilter.class);
        job.setReducerClass(DedupReducer.class);

        try {
            RunningJob rj = JobClient.runJob(job);
            Group g = rj.getCounters().getGroup("DeduplicationJobStatus");
            if (g != null){
              long dups = g.getCounter("Documents marked as duplicate");
              LOG.info("Deduplication: "+(int)dups+" documents marked as duplicates");
            }
        } catch (final Exception e) {
            LOG.error("DeduplicationJob: " + StringUtils.stringifyException(e));
            return -1;
View Full Code Here
607
608
609
610
611
612
613
614
615
616
617
    public Counters.Counter getCounter(String group, String name) {
      Counters.Counter counter = null;
      if (counters != null) {
        counter = counters.findCounter(group, name);
        if (counter == null) {
          Group grp = counters.addGroup(group, group);
          counter = grp.addCounter(name, name, 10);
        }
      }
      return counter;
    }
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
  public Collection<Pair<String, String>> findCounterValues() {
    final Collection<Pair<String, String>> counters = new LinkedList<Pair<String, String>>();
    final Iterable<String> groupNames = getGroupNames();
    if (mapred != null) {
      for (String groupName : groupNames) {
        final Group group = mapred.getGroup(groupName);
        collectCounters(counters, groupName, group.iterator());
      }
    } else {
      for (String groupName : groupNames) {
        final CounterGroup group = mapreduce.getGroup(groupName);
        collectCounters(counters, groupName, group.iterator());
      }
    }
    return counters;
  }
View Full Code Here
207
208
209
210
211
212
213
214
215
216
  @SuppressWarnings("deprecation")
  @Test
  public void testGroupIteratorConcurrency() {
    Counters counters = new Counters();
    counters.incrCounter("group1", "counter1", 1);
    Group group = counters.getGroup("group1");
    Iterator<Counter> iterator = group.iterator();
    counters.incrCounter("group1", "counter2", 1);
    iterator.next();
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.Counters.Group

  • com.atlantbh.jmeter.plugins.hadooputilities.jobstatistics.JobLayer
  • com.atlantbh.jmeter.plugins.hadooputilities.jobstatistics.TaskLayer
  • com.netflix.lipstick.warnings.JobWarnings
  • org.apache.hadoop.mapred.gridmix.test.system.GridmixJobVerification
  • org.apache.hadoop.mapred.GridMixRunner
  • org.apache.hadoop.mapred.HistoryViewer
  • org.apache.hadoop.mapred.JobClient
  • org.apache.hadoop.mapred.pipes.TestPipeApplication$TestTaskReporter
  • org.apache.hadoop.mapred.TestCounters
  • org.apache.hadoop.mrunit.internal.counters.CounterWrapper
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.