Package org.apache.hadoop.mapred.Counters

Examples of org.apache.hadoop.mapred.Counters.Counter


    for (Map.Entry<String, ? extends Number> entry : counterNameToValue.entrySet()) {

      String[] cNames = entry.getKey().split("::");
      String groupName = cNames[0];
      String counterName = cNames[1];
      Counter counter = counters.findCounter(groupName, counterName);
      counter.setValue(entry.getValue().longValue());
    }
    return CounterGroup.counterGroupInfoMap(counters);
  }
View Full Code Here


      Group mapGroup = mapCounters.getGroup(groupName);
      Group reduceGroup = reduceCounters.getGroup(groupName);
      Format decimal = new DecimalFormat();
      Iterator<Counter> ctrItr = totalGroup.iterator();
      while (ctrItr.hasNext()) {
        Counter counter = ctrItr.next();
        String name = counter.getDisplayName();
        String mapValue = decimal.format(mapGroup.getCounter(name));
        String reduceValue = decimal.format(reduceGroup.getCounter(name));
        String totalValue = decimal.format(counter.getValue());
        buff.append(
          String.format("\n|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s",
          totalGroup.getDisplayName(),
          counter.getDisplayName(),
          mapValue, reduceValue, totalValue));
      }
    }
  }
View Full Code Here

                  + jobid);
            }
            exitCode = -1;
          } else {
            Group group = counters.getGroup(counterGroupName);
            Counter counter = group.getCounterForName(counterName);
            System.out.println(counter.getCounter());
            exitCode = 0;
          }
        }
      } else if (killJob) {
        RunningJob job = getJob(JobID.forName(jobid));
View Full Code Here

                  + jobid);
            }
            exitCode = -1;
          } else {
            Group group = counters.getGroup(counterGroupName);
            Counter counter = group.getCounterForName(counterName);
            System.out.println(counter.getCounter());
            exitCode = 0;
          }
        }
      } else if (killJob) {
        RunningJob job = getJob(JobID.forName(jobid));
View Full Code Here

      // find out CPU msecs
      // In the case that we can't find out this number, we just skip the step to print
      // it out.
      if (ctrs != null) {
        Counter counterCpuMsec = ctrs.findCounter("org.apache.hadoop.mapred.Task$Counter",
            "CPU_MILLISECONDS");
        if (counterCpuMsec != null) {
          long newCpuMSec = counterCpuMsec.getValue();
          if (newCpuMSec > 0) {
            cpuMsec = newCpuMSec;
            report.append(", Cumulative CPU ").append((cpuMsec / 1000D)).append(" sec");
          }
        }
      }

      // write out serialized plan with counters to log file
      // LOG.info(queryPlan);
      String output = report.toString();
      SessionState ss = SessionState.get();
      if (ss != null) {
        ss.getHiveHistory().setTaskCounters(SessionState.get().getQueryId(), getId(), ctrs);
        ss.getHiveHistory().setTaskProperty(SessionState.get().getQueryId(), getId(),
            Keys.TASK_HADOOP_PROGRESS, output);
        if (ss.getConf().getBoolVar(HiveConf.ConfVars.HIVE_LOG_INCREMENTAL_PLAN_PROGRESS)) {
          ss.getHiveHistory().progressTask(SessionState.get().getQueryId(), this.task);
          this.callBackObj.logPlanProgress(ss);
        }
      }
      console.printInfo(output);
      reportTime = System.currentTimeMillis();
    }

    if (cpuMsec > 0) {
      console.printInfo("MapReduce Total cumulative CPU time: "
          + Utilities.formatMsecToStr(cpuMsec));
    }

    boolean success;

    Counters ctrs = th.getCounters();
    if (fatal) {
      success = false;
    } else {
      // check for fatal error again in case it occurred after
      // the last check before the job is completed
      if (checkFatalErrors(ctrs, errMsg)) {
        console.printError("[Fatal Error] " + errMsg.toString());
        success = false;
      } else {
        SessionState ss = SessionState.get();
        if (ss != null) {
          ss.getHiveHistory().setTaskCounters(SessionState.get().getQueryId(), getId(), ctrs);
        }
        success = rj.isSuccessful();
      }
    }

    if (ctrs != null) {
      Counter counterCpuMsec = ctrs.findCounter("org.apache.hadoop.mapred.Task$Counter",
          "CPU_MILLISECONDS");
      if (counterCpuMsec != null) {
        long newCpuMSec = counterCpuMsec.getValue();
        if (newCpuMSec > cpuMsec) {
          cpuMsec = newCpuMSec;
        }
      }
    }
View Full Code Here

    if (cpuMSec > 0) {
      sb.append(" Cumulative CPU: " + (cpuMSec / 1000D) + " sec  ");
    }

    if (counters != null) {
      Counter hdfsReadCntr = counters.findCounter("FileSystemCounters",
          "HDFS_BYTES_READ");
      long hdfsRead;
      if (hdfsReadCntr != null && (hdfsRead = hdfsReadCntr.getValue()) >= 0) {
        sb.append(" HDFS Read: " + hdfsRead);
      }

      Counter hdfsWrittenCntr = counters.findCounter("FileSystemCounters",
          "HDFS_BYTES_WRITTEN");
      long hdfsWritten;
      if (hdfsWrittenCntr != null && (hdfsWritten = hdfsWrittenCntr.getValue()) >= 0) {
        sb.append(" HDFS Write: " + hdfsWritten);
      }
    }

    sb.append(" " + (success ? "SUCCESS" : "FAIL"));
View Full Code Here

            String groupName = getGroupName(counters.getGroupNames());
            // In case that the counter group was not find, we need to find
            // out why. Only acceptable state is that the relation has been
            // empty.
            if (groupName == null) {
                Counter outputRecords =
                    counters.getGroup(MRPigStatsUtil.TASK_COUNTER_GROUP)
                    .getCounterForName(MRPigStatsUtil.MAP_OUTPUT_RECORDS);

                if(outputRecords.getCounter() == 0) {
                    globalCounters.put(operationID, new ArrayList<Pair<String, Long>>());
                    return;
                } else {
                  throw new RuntimeException("Did not found RANK counter group for operationId: " + operationID);
                }
            }
            groupCounters = counters.getGroup(groupName);

            Iterator<Counter> it = groupCounters.iterator();
            HashMap<Integer,Long> counterList = new HashMap<Integer, Long>();

            while(it.hasNext()) {
                try{
                    Counter c = it.next();
                    counterList.put(Integer.valueOf(c.getDisplayName()), c.getValue());
                } catch (Exception ex) {
                    ex.printStackTrace();
                }
            }
            counterSize = counterList.size();
View Full Code Here

      for (String counterName : stats.getCountersFor(groupName)) { //retreiving counters in that group
        Long counterValue = stats.getCounterValue(groupName, counterName);
        counterNameToValue.put(groupName+"::"+counterName, counterValue);

        //creating counter
        Counter counter = counters.findCounter(groupName, counterName);
        counter.setValue(counterValue);
      }
    }
    setCounterGroupMap(CounterGroup.counterGroupInfoMap(counters));
    return counterNameToValue;
  }
View Full Code Here

            activeSpillCountRecs = counters.findCounter(
                    PigCounters.PROACTIVE_SPILL_COUNT_RECS).getCounter();

            Iterator<Counter> iter = multistoregroup.iterator();
            while (iter.hasNext()) {
                Counter cter = iter.next();
                multiStoreCounters.put(cter.getName(), cter.getValue());
            }    
           
            Iterator<Counter> iter2 = multiloadgroup.iterator();
            while (iter2.hasNext()) {
                Counter cter = iter2.next();
                multiInputCounters.put(cter.getName(), cter.getValue());
            }
           
        }             
    }
View Full Code Here

            Iterator<Counter> it = groupCounters.iterator();
            HashMap<Integer,Long> counterList = new HashMap<Integer, Long>();

            while(it.hasNext()) {
                try{
                    Counter c = it.next();
                    counterList.put(Integer.valueOf(c.getDisplayName()), c.getValue());
                } catch (Exception ex) {
                    ex.printStackTrace();
                }
            }
            counterSize = counterList.size();
View Full Code Here

TOP

Related Classes of org.apache.hadoop.mapred.Counters.Counter

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.