Package org.apache.hadoop.mapreduce

Examples of org.apache.hadoop.mapreduce.JobID


  public static org.apache.hadoop.mapred.JobID getOldJobId(Job job) {
    if (job == null) {
      return null;
    }
    JobID id = getJobId(job);
    if (id != null) {
      return org.apache.hadoop.mapred.JobID.downgrade(id);
    }
    return null;
  }
View Full Code Here


  /**
   * removing failed DT
   * @param jobId
   */
  private static void removeFailedDelegationToken(DelegationTokenToRenew t) {
    JobID jobId = t.jobId;
    if (LOG.isDebugEnabled())
      LOG.debug("removing failed delegation token for jobid=" + jobId +
          ";t=" + t.token.getService());
    delegationTokens.remove(t);
    // cancel the timer
View Full Code Here

      throws IOException {
    if (jobs.size() > 0) {
      for (JobInProgress job : jobs) {
        JobProfile profile = job.getProfile();
        JobStatus status = job.getStatus();
        JobID jobid = profile.getJobID();

        int desiredMaps = job.desiredMaps();
        int desiredReduces = job.desiredReduces();
        int completedMaps = job.finishedMaps();
        int completedReduces = job.finishedReduces();
View Full Code Here

   * @param attemptId
   *          raw {@link TaskAttemptID} read from trace
   * @return masked {@link TaskAttemptID} with empty {@link JobID}.
   */
  private TaskAttemptID maskAttemptID(TaskAttemptID attemptId) {
    JobID jobId = new JobID();
    TaskID taskId = attemptId.getTaskID();
    return new TaskAttemptID(jobId.getJtIdentifier(), jobId.getId(),
        attemptId.isMap(), taskId.getId(), attemptId.getId());
  }
View Full Code Here

          : State.FAILED;
    }
  }

  private TaskID getMaskedTaskID(TaskType taskType, int taskNumber) {
    return new TaskID(new JobID(), TaskType.MAP == taskType, taskNumber);
  }
View Full Code Here

   * @throws Exception
   */
  public RunningJob submitAndVerifyJob(Configuration conf) throws Exception {
    JobConf jconf = new JobConf(conf);
    RunningJob rJob = getClient().submitJob(jconf);
    JobID jobId = rJob.getID();
    verifyRunningJob(jobId);
    verifyCompletedJob(jobId);
    return rJob;
  }
View Full Code Here

    JobConf jconf = new JobConf(job.getConfiguration(), JobKillCommitter.class);
    jconf.setOutputCommitter(theClass);
    if(!isUserKill)
    { 
      RunningJob rJob = cluster.getJTClient().getClient().submitJob(jconf);
      JobID id = rJob.getID();
      JobInfo jInfo = wovenClient.getJobInfo(id);
      Assert.assertTrue("Job is not in PREP state",
          jInfo.getStatus().getRunState() == JobStatus.PREP);
    }
    else
    {
      //user kill job
      RunningJob rJob = cluster.getJTClient().getClient().submitJob(jconf);
      JobInfo info = wovenClient.getJobInfo(rJob.getID());
      Assert.assertNotNull("Job Info is null",info);
      JobID id = rJob.getID();
      while (info.runningMaps() != 1) {
        Thread.sleep(1000);
        info = wovenClient.getJobInfo(id);
      }
      rJob.killJob();
View Full Code Here

    conf = job.setupJobConf(1, 1, 100, 100, 100, 100);
    JobClient client = cluster.getJTClient().getClient();

    RunningJob rJob = client.submitJob(new JobConf(conf));
    JobID id = rJob.getID();

    JobInfo jInfo = wovenClient.getJobInfo(id);

    while (jInfo.getStatus().getRunState() != JobStatus.RUNNING) {
      Thread.sleep(1000);
View Full Code Here

   * @param taskId
   *          raw {@link TaskID} read from trace
   * @return masked {@link TaskID} with empty {@link JobID}.
   */
  private TaskID maskTaskID(TaskID taskId) {
    JobID jobId = new JobID();
    return new TaskID(jobId, taskId.isMap(), taskId.getId());
  }
View Full Code Here

    ts.addToken(new Text(nn2), token2);
    ts.addToken(new Text(nn3), token3);
   
    // register the tokens for renewal
    DelegationTokenRenewal.registerDelegationTokensForRenewal(
        new JobID("job1", 1), ts, conf);
    // first 3 initial renewals + 1 real
    int numberOfExpectedRenewals = 3+1;
   
    int attempts = 10;
    while(attempts-- > 0) {
      try {
        Thread.sleep(3*1000); // sleep 3 seconds, so it has time to renew
      } catch (InterruptedException e) {}
     
      // since we cannot guarantee timely execution - let's give few chances
      if(dfs.getCounter()==numberOfExpectedRenewals)
        break;
    }
   
    System.out.println("Counter = " + dfs.getCounter() + ";t="+
        dfs.getToken());
    assertEquals("renew wasn't called as many times as expected(4):",
        numberOfExpectedRenewals, dfs.getCounter());
    assertEquals("most recently renewed token mismatch", dfs.getToken(),
        token1);
   
    // Test 2.
    // add another token ( that expires in 2 secs). Then remove it, before
    // time is up.
    // Wait for 3 secs , and make sure no renews were called
    ts = new Credentials();
    MyToken token4 = dfs.getDelegationToken(new Text("user4"));
   
    //to cause this one to be set for renew in 2 secs
    dfs.setTokenToRenewIn2Sec(token4);
    System.out.println("token="+token4+" should be renewed for 2 secs");
   
    String nn4 = DelegationTokenRenewal.SCHEME + "://host4:0";
    ts.addToken(new Text(nn4), token4);
   

    JobID jid2 = new JobID("job2",1);
    DelegationTokenRenewal.registerDelegationTokensForRenewal(jid2, ts, conf);
    DelegationTokenRenewal.removeDelegationTokenRenewalForJob(jid2);
    numberOfExpectedRenewals = dfs.getCounter(); // number of renewals so far
    try {
      Thread.sleep(6*1000); // sleep 6 seconds, so it has time to renew
View Full Code Here

TOP

Related Classes of org.apache.hadoop.mapreduce.JobID

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.