Examples of org.apache.hadoop.mapred.ClusterStatus

org.apache.hadoop.mapred.ClusterStatus
Status information on the current state of the Map-Reduce cluster.
ClusterStatus provides clients with information such as:
1. Size of the cluster.
2. Name of the trackers.
3. Task capacity of the cluster.
4. The number of currently running map & reduce tasks.
5. State of the JobTracker.
Clients can query for the latest ClusterStatus, via {@link JobClient#getClusterStatus()}.
@see JobClient


    @Override
    public List<InputSplit> getSplits(JobContext jobCtxt) throws IOException {
      final JobClient client =
        new JobClient(new JobConf(jobCtxt.getConfiguration()));
      ClusterStatus stat = client.getClusterStatus(true);
      final long toGen =
        jobCtxt.getConfiguration().getLong(GRIDMIX_GEN_BYTES, -1);
      if (toGen < 0) {
        throw new IOException("Invalid/missing generation bytes: " + toGen);
      }
      final int nTrackers = stat.getTaskTrackers();
      final long bytesPerTracker = toGen / nTrackers;
      final ArrayList<InputSplit> splits = new ArrayList<InputSplit>(nTrackers);
      final Pattern trackerPattern = Pattern.compile("tracker_([^:]*):.*");
      final Matcher m = trackerPattern.matcher("");
      for (String tracker : stat.getActiveTrackerNames()) {
        m.reset(tracker);
        if (!m.find()) {
          System.err.println("Skipping node: " + tracker);
          continue;
        }

View Full Code Here

      HiveClusterStatus hcs;
      try {
        Driver drv = new Driver();
        drv.init();


        ClusterStatus cs = drv.getClusterStatus();
        JobTracker.State jbs = cs.getJobTrackerState();


        // Convert the ClusterStatus to its Thrift equivalent: HiveClusterStatus
        JobTrackerState state;
        switch (jbs) {
        case INITIALIZING:
          state = JobTrackerState.INITIALIZING;
          break;
        case RUNNING:
          state = JobTrackerState.RUNNING;
          break;
        default:
          String errorMsg = "Unrecognized JobTracker state: " + jbs.toString();
          throw new Exception(errorMsg);
        }


        hcs = new HiveClusterStatus(cs.getTaskTrackers(), cs.getMapTasks(), cs
            .getReduceTasks(), cs.getMaxMapTasks(), cs.getMaxReduceTasks(),
            state);
      } catch (Exception e) {
        LOG.error(e.toString());
        e.printStackTrace();
        HiveServerException ex = new HiveServerException();

View Full Code Here

      return printUsage();    
    }
    
    Configuration conf = getConf();
    JobClient client = new JobClient(conf);
    ClusterStatus cluster = client.getClusterStatus();
    int numMapsPerHost = conf.getInt(MAPS_PER_HOST, 10);
    long numBytesToWritePerMap = conf.getLong(BYTES_PER_MAP,
                                             1*1024*1024*1024);
    if (numBytesToWritePerMap == 0) {
      System.err.println("Cannot have " + BYTES_PER_MAP +" set to 0");
      return -2;
    }
    long totalBytesToWrite = conf.getLong(TOTAL_BYTES, 
         numMapsPerHost*numBytesToWritePerMap*cluster.getTaskTrackers());
    int numMaps = (int) (totalBytesToWrite / numBytesToWritePerMap);
    if (numMaps == 0 && totalBytesToWrite > 0) {
      numMaps = 1;
      conf.setLong(BYTES_PER_MAP, totalBytesToWrite);
    }

View Full Code Here


  /**
   * Return the status information about the Map-Reduce cluster
   */
  public ClusterStatus getClusterStatus() throws Exception {
    ClusterStatus cs;
    try {
      JobConf job = new JobConf(conf, ExecDriver.class);
      JobClient jc = new JobClient(job);
      cs = jc.getClusterStatus();
    } catch (Exception e) {
      e.printStackTrace();
      throw e;
    }
    LOG.info("Returning cluster status: " + cs.toString());
    return cs;
  }

View Full Code Here

  private static String getMRStats() throws Exception {
    Configuration conf = CachedConfiguration.getInstance();
    // No alternatives for hadoop 20
    JobClient jc = new JobClient(new org.apache.hadoop.mapred.JobConf(conf));
    
    ClusterStatus cs = jc.getClusterStatus(false);
    
    return "" + cs.getMapTasks() + " " + cs.getMaxMapTasks() + " " + cs.getReduceTasks() + " " + cs.getMaxReduceTasks() + " " + cs.getTaskTrackers() + " "
        + cs.getBlacklistedTrackers();
    
  }

View Full Code Here

    return conf;
  }
  
  private static void waitForTaskTrackers(JobClient client) throws IOException {
    while (true) {
      ClusterStatus clusterStatus = client.getClusterStatus();
      int taskTrackerCount = clusterStatus.getTaskTrackers();
      if (taskTrackerCount > 0) {
        break;
      }
      try {
        System.out.print(".");

View Full Code Here


  /**
   * Return the status information about the Map-Reduce cluster
   */
  public ClusterStatus getClusterStatus() throws Exception {
    ClusterStatus cs;
    try {
      JobConf job = new JobConf(conf);
      JobClient jc = new JobClient(job);
      cs = jc.getClusterStatus();
    } catch (Exception e) {
      e.printStackTrace();
      throw e;
    }
    LOG.info("Returning cluster status: " + cs.toString());
    return cs;
  }

View Full Code Here

    return conf;
  }
  
  private static void waitForTaskTrackers(JobClient client) throws IOException {
    while (true) {
      ClusterStatus clusterStatus = client.getClusterStatus();
      int taskTrackerCount = clusterStatus.getTaskTrackers();
      if (taskTrackerCount > 0) {
        break;
      }
      try {
        System.out.print(".");

View Full Code Here

      String port = httpAddress.split(":")[1];
      String href = "http://" + address.getHostName() + ":" + port;
      sb.append("<tr><th colspan='2'><a href='" + href + "'>JobTracker</a></th></tr>\n");
      boolean highlight = false;
      tableRow(sb, (highlight = !highlight), "Running&nbsp;Jobs", jc.jobsToComplete().length);
      ClusterStatus status = jc.getClusterStatus();
      tableRow(sb, (highlight = !highlight), "Map&nbsp;Tasks", status.getMapTasks() + "/" + status.getMaxMapTasks());
      tableRow(sb, (highlight = !highlight), "Reduce&nbsp;Tasks", status.getReduceTasks() + "/" + status.getMaxReduceTasks());
      tableRow(sb, (highlight = !highlight), "Trackers", status.getTaskTrackers());
      tableRow(sb, (highlight = !highlight), "Blacklisted", status.getBlacklistedTrackers());
    } catch (Exception ex) {
      sb.append("<tr><td colspan='2'><span class='error'>Job Tracker is Down</span></td></tr>\n");
    }
    sb.append("</table>\n");
  }

View Full Code Here

     * Return the status information about the Map-Reduce cluster
     */
    public HiveClusterStatus getClusterStatus() throws HiveServerException, TException {
      HiveClusterStatus hcs;
      try {
        ClusterStatus cs = driver.getClusterStatus();
        JobTracker.State jbs = cs.getJobTrackerState();


        // Convert the ClusterStatus to its Thrift equivalent: HiveClusterStatus
        int state;
        switch (jbs) {
          case INITIALIZING:
            state = JobTrackerState.INITIALIZING;
            break;
          case RUNNING:
            state = JobTrackerState.RUNNING;
            break;
          default:
            String errorMsg = "Unrecognized JobTracker state: " + jbs.toString();
            throw new Exception(errorMsg);
        }


        hcs = new HiveClusterStatus(
            cs.getTaskTrackers(),
            cs.getMapTasks(),
            cs.getReduceTasks(),
            cs.getMaxMapTasks(),
            cs.getMaxReduceTasks(),
            state);
      }
      catch (Exception e) {
        LOG.error(e.toString());
        e.printStackTrace();

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapred.ClusterStatus

com.google.HadoopMonitor$ProgressResult

com.nexr.rhive.hadoop.JobManager

org.apache.accumulo.server.monitor.servlets.DefaultServlet

org.apache.accumulo.server.test.continuous.ContinuousStatsCollector

org.apache.accumulo.test.continuous.ContinuousStatsCollector

org.apache.hadoop.examples.Join

org.apache.hadoop.examples.NNBench

org.apache.hadoop.examples.RandomTextWriter

org.apache.hadoop.examples.RandomWriter

org.apache.hadoop.examples.Sort

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.