Examples of org.apache.hadoop.mapreduce.Cluster

org.apache.hadoop.mapreduce.Cluster
Provides a way to access information about the map/reduce cluster.

   * @throws Exception - EXception if any
   */
  private Path createMetaFolderPath() throws Exception {
    Configuration configuration = getConf();
    Path stagingDir = JobSubmissionFiles.getStagingDir(
            new Cluster(configuration), configuration);
    Path metaFolderPath = new Path(stagingDir, PREFIX + String.valueOf(rand.nextInt()));
    if (LOG.isDebugEnabled())
      LOG.debug("Meta folder location: " + metaFolderPath);
    configuration.set(DistCpConstants.CONF_LABEL_META_FOLDER, metaFolderPath.toString());    
    return metaFolderPath;

View Full Code Here

    historyService.init(conf);
    historyService.start(conf);


    LOG.info("services started");


    Cluster cluster = new Cluster(conf);
    org.apache.hadoop.mapreduce.JobID jobID =
      new org.apache.hadoop.mapred.JobID("201103121733", 1);
    org.apache.hadoop.mapreduce.Counters counters =
        cluster.getJob(jobID).getCounters();
    validateCounters(counters);
    Assert.assertTrue(amContact);


    LOG.info("Sleeping for 5 seconds before stop for" +
    " the client socket to not get EOF immediately..");
    Thread.sleep(5000);


    //bring down the AM service
    amService.stop();


    LOG.info("Sleeping for 5 seconds after stop for" +
        " the server to exit cleanly..");
    Thread.sleep(5000);


    amRestarting = true;


    // Same client
    //results are returned from fake (not started job)
    counters = cluster.getJob(jobID).getCounters();
    Assert.assertEquals(0, counters.countCounters());
    Job job = cluster.getJob(jobID);
    org.apache.hadoop.mapreduce.TaskID taskId =
      new org.apache.hadoop.mapreduce.TaskID(jobID, TaskType.MAP, 0);
    TaskAttemptID tId = new TaskAttemptID(taskId, 0);


    //invoke all methods to check that no exception is thrown
    job.killJob();
    job.killTask(tId);
    job.failTask(tId);
    job.getTaskCompletionEvents(0, 100);
    job.getStatus();
    job.getTaskDiagnostics(tId);
    job.getTaskReports(TaskType.MAP);
    job.getTrackingURL();


    amRestarting = false;
    amService = new AMService();
    amService.init(conf);
    amService.start(conf);
    amContact = false; //reset


    counters = cluster.getJob(jobID).getCounters();
    validateCounters(counters);
    Assert.assertTrue(amContact);


    // Stop the AM. It is not even restarting. So it should be treated as
    // completed.
    amService.stop();


    // Same client
    counters = cluster.getJob(jobID).getCounters();
    validateCounters(counters);
    Assert.assertTrue(hsContact);


    rmService.stop();
    historyService.stop();

View Full Code Here

   * @throws Exception - EXception if any
   */
  private Path createMetaFolderPath() throws Exception {
    Configuration configuration = getConf();
    Path stagingDir = JobSubmissionFiles.getStagingDir(
            new Cluster(configuration), configuration);
    Path metaFolderPath = new Path(stagingDir, PREFIX + String.valueOf(rand.nextInt()));
    if (LOG.isDebugEnabled())
      LOG.debug("Meta folder location: " + metaFolderPath);
    configuration.set(DistCpConstants.CONF_LABEL_META_FOLDER, metaFolderPath.toString());    
    return metaFolderPath;

View Full Code Here


      DistCpOptions options = new DistCpOptions(sources, target);


      Configuration conf = getConf();
      Path stagingDir = JobSubmissionFiles.getStagingDir(
              new Cluster(conf), conf);
      stagingDir.getFileSystem(conf).mkdirs(stagingDir);


      try {
        new DistCp(conf, options).execute();
      } catch (Throwable t) {

View Full Code Here

    }
    return exitCode;
  }


  Cluster createCluster() throws IOException {
    return new Cluster(getConf());
  }

View Full Code Here

   * @param conf the job configuration.
   * @throws IOException
   */
  public void init(JobConf conf) throws IOException {
    setConf(conf);
    cluster = new Cluster(conf);
    clientUgi = UserGroupInformation.getCurrentUser();
  }

View Full Code Here

   * @param jobTrackAddr the job tracker to connect to.
   * @param conf configuration.
   */
  public JobClient(InetSocketAddress jobTrackAddr, 
                   Configuration conf) throws IOException {
    cluster = new Cluster(jobTrackAddr, conf);
    clientUgi = UserGroupInformation.getCurrentUser();
  }

View Full Code Here


    @SuppressWarnings("unchecked")
    @Override
    public long renew(Token<?> token, Configuration conf
                      ) throws IOException, InterruptedException {
      return new Cluster(conf).
        renewDelegationToken((Token<DelegationTokenIdentifier>) token);
    }

View Full Code Here


    @SuppressWarnings("unchecked")
    @Override
    public void cancel(Token<?> token, Configuration conf
                       ) throws IOException, InterruptedException {
      new Cluster(conf).
        cancelDelegationToken((Token<DelegationTokenIdentifier>) token);
    }

View Full Code Here


        String temp1 = "_tmp/bang.number.stage1";
        String temp2 = "_tmp/bang.number.stage2";


        Configuration conf = new Configuration();
        Cluster cluster = new Cluster(conf);
        FileSystem fs = FileSystem.get(conf);
        fs.delete(new Path(temp1), true);


        //System.err.println("Stage 1...");
        {

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapreduce.Cluster

NewMissingTemperatureFields

nise.Number

org.apache.hadoop.examples.pi.DistSum$MixMachine

org.apache.hadoop.examples.terasort.TeraChecksum

org.apache.hadoop.examples.terasort.TeraGen

org.apache.hadoop.examples.terasort.TeraSort

org.apache.hadoop.examples.terasort.TeraValidate

org.apache.hadoop.fs.Path

org.apache.hadoop.hbase.mapreduce.JobUtil

org.apache.hadoop.io.Text

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.