Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.ClusterStatus


        //Fetch cluster data only if required.i.e .
        // only if there are clusterStats listener.
        if (clusterStatlisteners.size() > 0) {
          try {
            ClusterStatus clusterStatus = cluster.getClusterStatus();
            updateAndNotifyClusterStatsListeners(
              clusterStatus);
          } catch (IOException e) {
            LOG.error(
              "Statistics io exception while polling JT ", e);
View Full Code Here


  /**
   * Generates an XML-formatted block that summarizes the state of the JobTracker.
   */
  public void generateSummaryTable(JspWriter out,
                                   JobTracker tracker) throws IOException {
    ClusterStatus status = tracker.getClusterStatus();
    int maxMapTasks = status.getMaxMapTasks();
    int maxReduceTasks = status.getMaxReduceTasks();
    int numTaskTrackers = status.getTaskTrackers();
    String tasksPerNodeStr;
    if (numTaskTrackers > 0) {
      double tasksPerNodePct = (double) (maxMapTasks + maxReduceTasks) / (double) numTaskTrackers;
      tasksPerNodeStr = percentFormat.format(tasksPerNodePct);
    } else {
      tasksPerNodeStr = "-";
    }
    out.print("<maps>" + status.getMapTasks() + "</maps>\n" +
            "<reduces>" + status.getReduceTasks() + "</reduces>\n" +
            "<total_submissions>" + tracker.getTotalSubmissions() + "</total_submissions>\n" +
            "<nodes>" + status.getTaskTrackers() + "</nodes>\n" +
            "<map_task_capacity>" + status.getMaxMapTasks() + "</map_task_capacity>\n" +
            "<reduce_task_capacity>" + status.getMaxReduceTasks() + "</reduce_task_capacity>\n" +
            "<avg_tasks_per_node>" + tasksPerNodeStr + "</avg_tasks_per_node>\n");
  }
View Full Code Here

   */
  @Override
  public void update(Statistics.ClusterStats item) {
    lock.lock();
    try {
      ClusterStatus clusterMetrics = item.getStatus();
      try {
        checkLoadAndGetSlotsToBackfill(item,clusterMetrics);
      } catch (IOException e) {
        LOG.error("Couldn't get the new Status",e);
      }
View Full Code Here

      return printUsage();   
    }
   
    Configuration conf = getConf();
    JobClient client = new JobClient(conf);
    ClusterStatus cluster = client.getClusterStatus();
    int numMapsPerHost = conf.getInt(MAPS_PER_HOST, 10);
    long numBytesToWritePerMap = conf.getLong(BYTES_PER_MAP,
                                             1*1024*1024*1024);
    if (numBytesToWritePerMap == 0) {
      System.err.println("Cannot have " + BYTES_PER_MAP +" set to 0");
      return -2;
    }
    long totalBytesToWrite = conf.getLong(TOTAL_BYTES,
         numMapsPerHost*numBytesToWritePerMap*cluster.getTaskTrackers());
    int numMaps = (int) (totalBytesToWrite / numBytesToWritePerMap);
    if (numMaps == 0 && totalBytesToWrite > 0) {
      numMaps = 1;
      conf.setLong(BYTES_PER_MAP, totalBytesToWrite);
    }
View Full Code Here

      }
      if (totalBytes != -1 &&
          numMaps > (int)(totalBytes / MIN_BYTES_PER_MAP)) {
        numMaps = (int) (totalBytes / MIN_BYTES_PER_MAP);
      }
      ClusterStatus cluster = client.getClusterStatus();
      int tmpMaps = cluster.getTaskTrackers() * MAX_MAPS_PER_NODE;
      if (numMaps > tmpMaps) {
        numMaps = tmpMaps;
      }
      if (numMaps == 0) {
        numMaps = 1;
View Full Code Here

  /**
   * Return the status information about the Map-Reduce cluster
   */
  public ClusterStatus getClusterStatus() throws Exception {
    ClusterStatus cs;
    try {
      JobConf job = new JobConf(conf, ExecDriver.class);
      JobClient jc = new JobClient(job);
      cs = jc.getClusterStatus();
    } catch (Exception e) {
      e.printStackTrace();
      throw e;
    }
    LOG.info("Returning cluster status: " + cs.toString());
    return cs;
  }
View Full Code Here

     */
    public HiveClusterStatus getClusterStatus() throws HiveServerException,
        TException {
      HiveClusterStatus hcs;
      try {
        ClusterStatus cs = driver.getClusterStatus();
        JobTracker.State jbs = cs.getJobTrackerState();

        // Convert the ClusterStatus to its Thrift equivalent: HiveClusterStatus
        JobTrackerState state;
        switch (jbs) {
        case INITIALIZING:
          state = JobTrackerState.INITIALIZING;
          break;
        case RUNNING:
          state = JobTrackerState.RUNNING;
          break;
        default:
          String errorMsg = "Unrecognized JobTracker state: " + jbs.toString();
          throw new Exception(errorMsg);
        }

        hcs = new HiveClusterStatus(cs.getTaskTrackers(), cs.getMapTasks(), cs
            .getReduceTasks(), cs.getMaxMapTasks(), cs.getMaxReduceTasks(),
            state);
      } catch (Exception e) {
        LOG.error(e.toString());
        e.printStackTrace();
        HiveServerException ex = new HiveServerException();
View Full Code Here

      throws IOException {
    // from RandomWriter
    job.setInputFormat(RandomInputFormat.class);
    job.setMapperClass(RandomMapOutput.class);

    final ClusterStatus cluster = new JobClient(job).getClusterStatus();
    int numMapsPerHost = job.getInt("test.randomtextwrite.maps_per_host", 10);
    long numBytesToWritePerMap =
      job.getLong("test.randomtextwrite.bytes_per_map", 1*1024*1024*1024);
    if (numBytesToWritePerMap == 0) {
      throw new IOException(
          "Cannot have test.randomtextwrite.bytes_per_map set to 0");
    }
    long totalBytesToWrite = job.getLong("test.randomtextwrite.total_bytes",
         numMapsPerHost * numBytesToWritePerMap * cluster.getTaskTrackers());
    int numMaps = (int)(totalBytesToWrite / numBytesToWritePerMap);
    if (numMaps == 0 && totalBytesToWrite > 0) {
      numMaps = 1;
      job.setLong("test.randomtextwrite.bytes_per_map", totalBytesToWrite);
    }
View Full Code Here

  /**
   * Return the status information about the Map-Reduce cluster
   */
  public ClusterStatus getClusterStatus() throws Exception {
    ClusterStatus cs;
    try {
      JobConf job = new JobConf(conf);
      JobClient jc = new JobClient(job);
      cs = jc.getClusterStatus();
    } catch (Exception e) {
      e.printStackTrace();
      throw e;
    }
    LOG.info("Returning cluster status: " + cs.toString());
    return cs;
  }
View Full Code Here

        jobConf.getPartitionerClass(), jobConf);
    int num_reduces = part.getNumPartitions();
   
   
    JobClient client = new JobClient(jobConf);
    ClusterStatus cluster = client.getClusterStatus();
    int num_maps = cluster.getTaskTrackers()
        * jobConf.getInt("test.sort.maps_per_host", 10);
    List <String>otherArgs = new ArrayList<String>();
    for (int i = 0; i < args.length; ++i) {
      try {
        if ("-m".equals(args[i])) {
          num_maps = Integer.parseInt(args[++i]);
        } else {
          otherArgs.add(args[i]);
        }
      } catch (NumberFormatException except) {
        System.out.println("ERROR: Integer expected instead of "
            + args[i]);
        printUsage();
      } catch (ArrayIndexOutOfBoundsException except) {
        System.out.println("ERROR: Required parameter missing from "
            + args[i - 1]);
        printUsage(); // exits
      }
    }

    jobConf.setNumMapTasks(num_maps);
    jobConf.setNumReduceTasks(num_reduces);

    // Make sure there are exactly 2 parameters left.
    if (otherArgs.size() != 2) {
      System.out.println("ERROR: Wrong number of parameters: "
          + otherArgs.size() + " instead of 2.");
      printUsage();
    }
    jobConf.setInputPath(new Path((String) otherArgs.get(0)));
    jobConf.setOutputPath(new Path((String) otherArgs.get(1)));

    // Uncomment to run locally in a single process
    //job_conf.set("mapred.job.tracker", "local");

    System.out.println("Running on " + cluster.getTaskTrackers()
        + " nodes to sort from " + jobConf.getInputPaths()[0]
        + " into " + jobConf.getOutputPath() + " with " + num_reduces
        + " reduces.");
    Date startTime = new Date();
    System.out.println("Job started: " + startTime);
View Full Code Here

TOP

Related Classes of org.apache.hadoop.mapred.ClusterStatus

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.