Examples of JTProtocol


Examples of org.apache.hadoop.mapreduce.test.system.JTProtocol

      bListedTT.add(client);
      blackListTT(client);
    }
    //Now run the high ram job
    JobClient jobClient = cluster.getJTClient().getClient();
    JTProtocol remoteJTClient = cluster.getJTClient().getProxy();
    Configuration conf = remoteJTClient.getDaemonConf();   
    hRamHelper.runHighRamJob(conf, jobClient, remoteJTClient,
        "Job did not succeed");
    //put the task tracker back in healthy state
    for( int i =0; i < bListedTT.size() ; ++i) {
      unBlackListTT(bListedTT.get(i));
View Full Code Here

Examples of org.apache.hadoop.mapreduce.test.system.JTProtocol

    FileInputFormat.setInputPaths(jobConf, inputDir);
    FileOutputFormat.setOutputPath(jobConf, outputDir);
   
    JTClient jtClient = cluster.getJTClient();
    JobClient client = jtClient.getClient();
    JTProtocol wovenClient = cluster.getJTClient().getProxy();
    RunningJob runJob = client.submitJob(jobConf);
    JobID id = runJob.getID();
    JobInfo jInfo = wovenClient.getJobInfo(id);
    Assert.assertNotNull("Job information is null",jInfo);

    Assert.assertTrue("Job has not been started for 1 min.",
        jtClient.isJobStarted(id));

    TaskInfo[] taskInfos = wovenClient.getTaskInfo(id);
    for (TaskInfo taskinfo : taskInfos) {
      if (!taskinfo.isSetupOrCleanup()) {
        taskInfo = taskinfo;
        break;
      }
    }

    Assert.assertTrue("Task has not been started for 1 min.",
        jtClient.isTaskStarted(taskInfo));

    tID = TaskID.downgrade(taskInfo.getTaskID());
    TaskAttemptID tAttID = new TaskAttemptID(tID,0);
    FinishTaskControlAction action = new FinishTaskControlAction(tID);

    Collection<TTClient> ttClients = cluster.getTTClients();
    for (TTClient ttClient : ttClients) {
      TTProtocol tt = ttClient.getProxy();
      tt.sendAction(action);
      ttTaskinfo = tt.getTasks();
      for (TTTaskInfo tttInfo : ttTaskinfo) {
        if (!tttInfo.isTaskCleanupTask()) {
          pid = tttInfo.getPid();
          ttClientIns = ttClient;
          ttIns = tt;
          break;
        }
      }
      if (ttClientIns != null) {
        break;
      }
    }


    Assert.assertTrue("Map process tree is not alive before task kills.",
        ttIns.isProcessTreeAlive(pid));

    String args[] = new String[] { "-kill-task", tAttID.toString() };
    int exitCode = runTool(jobConf, client, args);
    Assert.assertEquals("Exit Code:", 0, exitCode);
   
    LOG.info("Waiting till the task is killed...");
    counter = 0;
    while (counter < 30) {
      if (taskInfo.getTaskStatus().length > 0) {
        if (taskInfo.getTaskStatus()[0].getRunState() ==
            TaskStatus.State.KILLED) {
          break;
        }
      }
      UtilsForTests.waitFor(1000);
      taskInfo = wovenClient.getTaskInfo(taskInfo.getTaskID());
      counter ++;
    }

    runJob.killJob();
    LOG.info("Waiting till the job is completed...");
    counter = 0;
    while (counter < 60) {
      if (jInfo.getStatus().isJobComplete()) {
        break;
      }
      UtilsForTests.waitFor(1000);
      jInfo = wovenClient.getJobInfo(id);
      counter ++;
    }
    Assert.assertTrue("Job has not  been completed for 1 min.",
        counter != 60 );
    ttIns = ttClientIns.getProxy();
View Full Code Here

Examples of org.apache.hadoop.mapreduce.test.system.JTProtocol

    FileInputFormat.setInputPaths(jobConf, inputDir);
    FileOutputFormat.setOutputPath(jobConf, outputDir);
    JTClient jtClient = cluster.getJTClient();
    JobClient client = jtClient.getClient();
    JTProtocol wovenClient = cluster.getJTClient().getProxy();
    RunningJob runJob = client.submitJob(jobConf);
    JobID id = runJob.getID();
    JobInfo jInfo = wovenClient.getJobInfo(id);   
    Assert.assertNotNull("Job information is null", jInfo);

    Assert.assertTrue("Job has not been started for 1 min.",
        jtClient.isJobStarted(id));

    TaskInfo[] taskInfos = wovenClient.getTaskInfo(id);
    for (TaskInfo taskinfo : taskInfos) {
      if (!taskinfo.isSetupOrCleanup()) {
        taskInfo = taskinfo;
        break;
      }
    }

    Assert.assertTrue("Task has not been started for 1 min.",
        jtClient.isTaskStarted(taskInfo));

    tID = TaskID.downgrade(taskInfo.getTaskID());
    taskAttID = new TaskAttemptID(tID,0);
    FinishTaskControlAction action = new FinishTaskControlAction(tID);
    Collection<TTClient> ttClients = cluster.getTTClients();
    for (TTClient ttClient : ttClients) {
      TTProtocol tt = ttClient.getProxy();
      tt.sendAction(action);
      ttTaskinfo = tt.getTasks();
      for (TTTaskInfo tttInfo : ttTaskinfo) {
        if (!tttInfo.isTaskCleanupTask()) {
          pid = tttInfo.getPid();
          ttClientIns = ttClient;
          ttIns = tt;
          break;
        }
      }
      if (ttClientIns != null) {
        break;
      }
    }


    Assert.assertTrue("Map process is not alive before task kills.",
        ttIns.isProcessTreeAlive(pid));

    runJob.killTask(taskAttID, false);

    LOG.info("Waiting till the task is killed...");
    taskInfo = wovenClient.getTaskInfo(taskInfo.getTaskID());
    counter = 0;
    while (counter < 30) {
      if (taskInfo.getTaskStatus()[0].getRunState() ==
              TaskStatus.State.KILLED) {
        break;
      }
      UtilsForTests.waitFor(1000);
      taskInfo = wovenClient.getTaskInfo(taskInfo.getTaskID());
      counter ++;
    }
    runJob.killJob();
    LOG.info("Waiting till the job is completed...");
    counter = 0;
    while (counter < 60) {
      if (jInfo.getStatus().isJobComplete()) {
        break;
      }
      UtilsForTests.waitFor(1000);
      jInfo = wovenClient.getJobInfo(id);
      counter ++;
    }
    Assert.assertTrue("Job has not been completed for 1 min.",
        counter != 60);
    UtilsForTests.waitFor(2000);
View Full Code Here

Examples of org.apache.hadoop.mapreduce.test.system.JTProtocol

    FileInputFormat.setInputPaths(jobConf, inputDir);
    FileOutputFormat.setOutputPath(jobConf, outputDir);

    JTClient jtClient = cluster.getJTClient();
    JobClient client = jtClient.getClient();
    JTProtocol wovenClient = cluster.getJTClient().getProxy();
    RunningJob runJob = client.submitJob(jobConf);
    JobID id = runJob.getID();
    JobInfo jInfo = wovenClient.getJobInfo(id);
    Assert.assertNotNull("Job information is null", jInfo);

    Assert.assertTrue("Job has not been started for 1 min.",
        jtClient.isJobStarted(id));

    TaskInfo[] taskInfos = wovenClient.getTaskInfo(id);
    for (TaskInfo taskinfo : taskInfos) {
      if (!taskinfo.isSetupOrCleanup()) {
        taskInfo = taskinfo;
        break;
      }
    }

    Assert.assertTrue("Task has not been started for 1 min.",
        jtClient.isTaskStarted(taskInfo));

    tID = TaskID.downgrade(taskInfo.getTaskID());   
    FinishTaskControlAction action = new FinishTaskControlAction(tID);
   
    Collection<TTClient> ttClients = cluster.getTTClients();
    for (TTClient ttClient : ttClients) {
      TTProtocol tt = ttClient.getProxy();
      tt.sendAction(action);
      ttTaskinfo = tt.getTasks();
      for (TTTaskInfo tttInfo : ttTaskinfo) {
        if (!tttInfo.isTaskCleanupTask()) {
          pid = tttInfo.getPid();
          ttClientIns = ttClient;
          ttIns = tt;
          break;
        }
      }
      if (ttClientIns != null) {
        break;
      }
    }

    Assert.assertTrue("Map process is not alive before task fails.",
            ttIns.isProcessTreeAlive(pid));

    LOG.info("Waiting till the task is failed...");
    counter = 0;
    while (counter < 60) {
      if (taskInfo.getTaskStatus().length > 0) {
        if (taskInfo.getTaskStatus()[0].getRunState() ==
            TaskStatus.State.FAILED) {
          break;
        }
      }
      UtilsForTests.waitFor(1000);
      taskInfo = wovenClient.getTaskInfo(taskInfo.getTaskID());
      counter++;
    }

    LOG.info("Waiting till the job is completed...");
    counter = 0;
    while (counter < 60) {
      if (jInfo.getStatus().isJobComplete()) {
        break;
      }
      UtilsForTests.waitFor(1000);
      jInfo = wovenClient.getJobInfo(id);
      counter ++;
    }
    Assert.assertTrue("Job has not been completed for 1 min.",
        counter != 60);
    ttIns = ttClientIns.getProxy();
View Full Code Here

Examples of org.apache.hadoop.mapreduce.test.system.JTProtocol

    FileInputFormat.setInputPaths(jobConf, inputDir);
    FileOutputFormat.setOutputPath(jobConf, outputDir);

    JTClient jtClient = cluster.getJTClient();
    JobClient client = jtClient.getClient();
    JTProtocol wovenClient = cluster.getJTClient().getProxy();
    RunningJob runJob = client.submitJob(jobConf);
    JobID id = runJob.getID();
    JobInfo jInfo = wovenClient.getJobInfo(id);
    Assert.assertNotNull("Job information is null", jInfo);

    Assert.assertTrue("Job has not been started for 1 min.",
        jtClient.isJobStarted(id));

    TaskInfo[] taskInfos = wovenClient.getTaskInfo(id);
    for (TaskInfo taskinfo : taskInfos) {
      if (!taskinfo.isSetupOrCleanup()) {
        taskInfo = taskinfo;
        break;
      }
    }

    Assert.assertTrue("Task has not been started for 1 min.",
        jtClient.isTaskStarted(taskInfo));

    tID = TaskID.downgrade(taskInfo.getTaskID());
    TaskAttemptID tAttID = new TaskAttemptID(tID,0);
    FinishTaskControlAction action = new FinishTaskControlAction(tID);

    Collection<TTClient> ttClients = cluster.getTTClients();
    for (TTClient ttClient : ttClients) {
      TTProtocol tt = ttClient.getProxy();
      tt.sendAction(action);
      ttTaskinfo = tt.getTasks();
      for (TTTaskInfo tttInfo : ttTaskinfo) {
        if (!tttInfo.isTaskCleanupTask()) {
          pid = tttInfo.getPid();
          ttClientIns = ttClient;
          ttIns = tt;
          break;
        }
      }
      if (ttClientIns != null) {
        break;
      }
    }


    Assert.assertTrue("Map process is not alive before task fails.",
        ttIns.isProcessTreeAlive(pid));

    String args[] = new String[] { "-fail-task", tAttID.toString() };
    int exitCode = runTool(jobConf, client, args);
    Assert.assertEquals("Exit Code:", 0, exitCode);

    LOG.info("Waiting till the task is failed...");
    taskInfo = wovenClient.getTaskInfo(taskInfo.getTaskID());
    counter = 0;
    while (counter < 60) {
      if (taskInfo.getTaskStatus().length > 0) {
        if (taskInfo.getTaskStatus()[0].getRunState() ==
            TaskStatus.State.FAILED) {
          break;
        }
      }
      UtilsForTests.waitFor(1000);
      taskInfo = wovenClient.getTaskInfo(taskInfo.getTaskID());
      counter ++;
    }
    counter = 0;
    LOG.info("Waiting till the job is completed...");
    while (counter < 60) {
      if (jInfo.getStatus().isJobComplete()) {
        break;
      }
      UtilsForTests.waitFor(1000);
      jInfo = wovenClient.getJobInfo(id);
      counter ++;
    }

    Assert.assertTrue("Job has not been completed for 1 min",
        counter != 60);
View Full Code Here

Examples of org.apache.hadoop.mapreduce.test.system.JTProtocol

   * @return void
   */
  public void testRetiredJobsSuccessful() throws Exception {
    Configuration conf = new Configuration(cluster.getConf());
    conf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", false);
    JTProtocol remoteJTClient = cluster.getJTClient().getProxy();
    int testLoopCount = 0;

    //First run a successful job and verify if JobInProgress
    //object is removed by checking the getJobInfo. In the
    //second iteration, verify if a killed job JobInProgress
    //is removed.
    do {
      testLoopCount++;
      SleepJob job = new SleepJob();
      job.setConf(conf);
      conf = job.setupJobConf(5, 1, 100, 100, 100, 100);
      //Get the value of mapred.jobtracker.retirejob.check. If not
      //found then use 60000 milliseconds, which is the application default.
      retiredJobInterval =
        conf.getInt("mapred.jobtracker.retirejob.check", 60000);
      //Assert if retiredJobInterval is 0
      if ( retiredJobInterval == 0 ) {
        Assert.fail("mapred.jobtracker.retirejob.check is 0");
      }

      JobConf jconf = new JobConf(conf);
      //Controls the job till all verification is done
      FinishTaskControlAction.configureControlActionForJob(conf);
      //Submitting the job
      RunningJob rJob = cluster.getJTClient().getClient().submitJob(jconf);

      JobID jobID = rJob.getID();

      JobInfo jInfo = remoteJTClient.getJobInfo(jobID);
      LOG.info("jInfo is :" + jInfo);

      boolean jobStarted = cluster.getJTClient().isJobStarted(jobID);

      Assert.assertTrue("Job has not started even after a minute",
          jobStarted );

      LOG.info("job id is :" + jobID.toString());

      TaskInfo[] taskInfos = cluster.getJTClient().getProxy()
          .getTaskInfo(jobID);

      // getJobInfo method should
      // return a JobInProgress object when running,
      JobInfo jobInfo = cluster.getJTClient().getProxy()
          .getJobInfo(jobID);

      Assert.assertNotNull("The Job information is not present ", jobInfo);

      //Allow the job to continue through MR control job.
      for (TaskInfo taskInfoRemaining : taskInfos) {
        FinishTaskControlAction action = new FinishTaskControlAction(TaskID
            .downgrade(taskInfoRemaining.getTaskID()));
        Collection<TTClient> tts = cluster.getTTClients();
        for (TTClient cli : tts) {
          cli.getProxy().sendAction(action);
        }
      }

      //Killing this job will happen only in the second iteration.
      if (testLoopCount == 2) {
        //Killing the job because all the verification needed
        //for this testcase is completed.
        rJob.killJob();
      }

      //Making sure that the job is complete.
      int count = 0;
      while (jInfo != null && !jInfo.getStatus().isJobComplete()) {
        UtilsForTests.waitFor(10000);
        count++;
        jInfo = remoteJTClient.getJobInfo(rJob.getID());
        //If the count goes more than 100 seconds, then fail; This is to
        //avoid infinite loop
        if (count > 10) {
          Assert.fail("Since the job has not completed even after" +
              " 100 seconds, failing at this point");
View Full Code Here

Examples of org.apache.hadoop.mapreduce.test.system.JTProtocol

   * @throws Exception
   */
  @Test
  public void testTaskDetails() throws Exception {
    Configuration conf = new Configuration(cluster.getConf());
    JTProtocol wovenClient = cluster.getJTClient().getProxy();
    FinishTaskControlAction.configureControlActionForJob(conf);
    SleepJob job = new SleepJob();
    job.setConf(conf);

    conf = job.setupJobConf(1, 1, 100, 100, 100, 100);
    JobClient client = cluster.getJTClient().getClient();

    RunningJob rJob = client.submitJob(new JobConf(conf));
    JobID id = rJob.getID();

    JobInfo jInfo = wovenClient.getJobInfo(id);

    while (jInfo.getStatus().getRunState() != JobStatus.RUNNING) {
      Thread.sleep(1000);
      jInfo = wovenClient.getJobInfo(id);
    }

    LOG.info("Waiting till job starts running one map");

    TaskInfo[] myTaskInfos = wovenClient.getTaskInfo(id);
    boolean isOneTaskStored = false;
    String sometaskpid = null;
    org.apache.hadoop.mapreduce.TaskAttemptID sometaskId = null;
    TTClient myCli = null;
    for(TaskInfo info : myTaskInfos) {
      if(!info.isSetupOrCleanup()) {
        String[] taskTrackers = info.getTaskTrackers();
        for(String taskTracker : taskTrackers) {
          TTInfo ttInfo = wovenClient.getTTInfo(taskTracker);
          TTClient ttCli =  cluster.getTTClient(ttInfo.getStatus().getHost());
          TaskID taskId = info.getTaskID();
          TTTaskInfo ttTaskInfo = ttCli.getProxy().getTask(taskId);
          Assert.assertNotNull(ttTaskInfo);
          Assert.assertNotNull(ttTaskInfo.getConf());
View Full Code Here

Examples of org.apache.hadoop.mapreduce.test.system.JTProtocol

    jobConf.setMemoryForMapTask(PER_TASK_LIMIT);
    jobConf.setMemoryForReduceTask(PER_TASK_LIMIT);
   
    JTClient jtClient = cluster.getJTClient();
    JobClient client = jtClient.getClient();
    JTProtocol wovenClient = cluster.getJTClient().getProxy();
    RunningJob runJob = client.submitJob(jobConf);
    JobID id = runJob.getID();
    JobInfo jInfo = wovenClient.getJobInfo(id);
    Assert.assertNotNull("Job information is null",jInfo);

    Assert.assertTrue("Job has not been started for 1 min.",
        jtClient.isJobStarted(id));

    TaskInfo[] taskInfos = wovenClient.getTaskInfo(id);
    for (TaskInfo taskinfo : taskInfos) {
      if (!taskinfo.isSetupOrCleanup()) {
        taskInfo = taskinfo;
        break;
      }
    }

    Assert.assertTrue("Task has not been started for 1 min.",
        jtClient.isTaskStarted(taskInfo));

    tID = TaskID.downgrade(taskInfo.getTaskID());
    TaskAttemptID tAttID = new TaskAttemptID(tID,0);
    FinishTaskControlAction action = new FinishTaskControlAction(tID);
    Collection<TTClient> ttClients = cluster.getTTClients();
    for (TTClient ttClient : ttClients) {
      TTProtocol tt = ttClient.getProxy();
      tt.sendAction(action);
      ttTaskinfo = tt.getTasks();
      for (TTTaskInfo tttInfo : ttTaskinfo) {
        if (!tttInfo.isTaskCleanupTask()) {
          pid = tttInfo.getPid();
          ttClientIns = ttClient;
          ttIns = tt;
          break;
        }
      }
      if (ttClientIns != null) {
        break;
      }
    }
    Assert.assertTrue("Map process is not alive before task fails.",
        ttIns.isProcessTreeAlive(pid));
   
    while (ttIns.getTask(tID).getTaskStatus().getRunState()
        == TaskStatus.State.RUNNING) {
      UtilsForTests.waitFor(1000);
      ttIns = ttClientIns.getProxy();
    }

    String[] taskDiagnostics = runJob.getTaskDiagnostics(tAttID);
    Assert.assertNotNull("Task diagnostics is null", taskDiagnostics);

    for (String strVal : taskDiagnostics) {
      mat = taskOverLimitPattern.matcher(strVal);
      Assert.assertTrue("Taskover limit error message is not matched.",
          mat.find());
    }

    runJob.killJob();

    LOG.info("Waiting till the job is completed...");
    counter = 0;
    while (counter < 60) {
      if (jInfo.getStatus().isJobComplete()) {
        break;
      }
      UtilsForTests.waitFor(1000);
      jInfo = wovenClient.getJobInfo(id);
      counter ++;
    }
    Assert.assertTrue("Job has not been completed...", counter != 60);
    UtilsForTests.waitFor(1000);
    ttIns = ttClientIns.getProxy();
View Full Code Here

Examples of org.apache.hadoop.mapreduce.test.system.JTProtocol

    jobConf.setMemoryForMapTask(PER_TASK_LIMIT);
    jobConf.setMemoryForReduceTask(PER_TASK_LIMIT);

    JTClient jtClient = cluster.getJTClient();
    JobClient client = jtClient.getClient();
    JTProtocol wovenClient = cluster.getJTClient().getProxy();
    RunningJob runJob = client.submitJob(jobConf);
    JobID id = runJob.getID();
    JobInfo jInfo = wovenClient.getJobInfo(id);
    Assert.assertNotNull("Job information is null", jInfo);

    Assert.assertTrue("Job has not been started for 1 min.",
        jtClient.isJobStarted(id));

    TaskInfo[] taskInfos = wovenClient.getTaskInfo(id);
    for (TaskInfo taskinfo : taskInfos) {
      if (!taskinfo.isSetupOrCleanup()) {
        taskInfo = taskinfo;
        break;
      }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.test.system.JTProtocol

   * @param none
   * @return void
   */
  public void testDistributedCache() throws Exception {
    Configuration conf = new Configuration(cluster.getConf());
    JTProtocol wovenClient = cluster.getJTClient().getProxy();

    //This counter will check for count of a loop,
    //which might become infinite.
    int count = 0;
    //This boolean will decide whether to run job again
    boolean continueLoop = true;
    //counter for job Loop
    int countLoop = 0;
    //This counter incerases with all the tasktrackers in which tasks ran
    int taskTrackerCounter = 0;
    //This will store all the tasktrackers in which tasks ran
    ArrayList<String> taskTrackerCollection = new ArrayList<String>();

    do {
      SleepJob job = new SleepJob();
      job.setConf(conf);
      conf = job.setupJobConf(5, 1, 1000, 1000, 100, 100);
      conf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", false);
      DistributedCache.createSymlink(conf);
      URI uri = URI.create(uriPath);
      DistributedCache.addCacheFile(uri, conf);
      JobConf jconf = new JobConf(conf);
      //Controls the job till all verification is done
      FinishTaskControlAction.configureControlActionForJob(conf);

      //Submitting the job
      RunningJob rJob = cluster.getJTClient().getClient().submitJob(jconf);

      //counter for job Loop
      countLoop++;

      TTClient tClient = null;
      JobInfo jInfo = wovenClient.getJobInfo(rJob.getID());
      LOG.info("jInfo is :" + jInfo);

      //Assert if jobInfo is null
      Assert.assertNotNull("jobInfo is null", jInfo);

      //Wait for the job to start running.
      count = 0;
      while (jInfo.getStatus().getRunState() != JobStatus.RUNNING) {
        UtilsForTests.waitFor(10000);
        count++;
        jInfo = wovenClient.getJobInfo(rJob.getID());
        //If the count goes beyond a point, then break; This is to avoid
        //infinite loop under unforeseen circumstances. Testcase will anyway
        //fail later.
        if (count > 10) {
          Assert.fail("job has not reached running state for more than" +
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.