Package org.apache.hadoop.mapred.FakeObjectUtilities

Examples of org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobInProgress


            trackers[1]),0,new ArrayList<TaskStatus>(), 0, 2, 2);
    tt1.setStatus(status1);
    tt2.setStatus(status2);
    tt3.setStatus(status3);
   
    FakeJobInProgress fjob = new FakeJobInProgress(conf, jobTracker);
    fjob.setClusterSize(3);
    fjob.initTasks();
   
    tt1.reserveSlots(TaskType.MAP, fjob, 2);
    tt1.reserveSlots(TaskType.REDUCE, fjob, 2);
    tt3.reserveSlots(TaskType.MAP, fjob, 2);
    tt3.reserveSlots(TaskType.REDUCE, fjob, 2);
   
    assertEquals("Trackers not reserved for the job : maps",
        2, fjob.getNumReservedTaskTrackersForMaps());
    assertEquals("Trackers not reserved for the job : reduces",
        2, fjob.getNumReservedTaskTrackersForReduces());
    ClusterMetrics metrics = jobTracker.getClusterMetrics();
    assertEquals("reserved map slots do not match",
          4, metrics.getReservedMapSlots());
    assertEquals("reserved reduce slots do not match",
          4, metrics.getReservedReduceSlots());
   
    TaskAttemptID mTid = fjob.findMapTask(trackers[1]);
    TaskAttemptID rTid = fjob.findReduceTask(trackers[1]);

    fjob.finishTask(mTid);
    fjob.finishTask(rTid);
   
    assertEquals("Job didnt complete successfully complete", fjob.getStatus()
        .getRunState(), JobStatus.SUCCEEDED);
   
    assertEquals("Reservation for the job not released: Maps",
        0, fjob.getNumReservedTaskTrackersForMaps());
    assertEquals("Reservation for the job not released : Reduces",
        0, fjob.getNumReservedTaskTrackersForReduces());
    metrics = jobTracker.getClusterMetrics();
    assertEquals("reserved map slots do not match",
        0, metrics.getReservedMapSlots());
    assertEquals("reserved reduce slots do not match",
        0, metrics.getReservedReduceSlots());
View Full Code Here


   *
   * @throws Exception
   */
 
  public void testTrackerReservationWithJobBlackListedTracker() throws Exception {
    FakeJobInProgress job = TestTaskTrackerBlacklisting.runBlackListingJob(
        jobTracker, trackers);
    assertEquals("Job has no blacklisted trackers", 1, job
        .getBlackListedTrackers().size());
    assertTrue("Tracker 1 not blacklisted for the job", job
        .getBlackListedTrackers().contains(
            JobInProgress.convertTrackerNameToHostName(trackers[0])));
    assertEquals("Job didnt complete successfully complete", job.getStatus()
        .getRunState(), JobStatus.SUCCEEDED);
    assertEquals("Reservation for the job not released: Maps",
        0, job.getNumReservedTaskTrackersForMaps());
    assertEquals("Reservation for the job not released : Reduces",
        0, job.getNumReservedTaskTrackersForReduces());
    ClusterMetrics metrics = jobTracker.getClusterMetrics();
    assertEquals("reserved map slots do not match",
        0, metrics.getReservedMapSlots());
    assertEquals("reserved reduce slots do not match",
        0, metrics.getReservedReduceSlots());
View Full Code Here

    conf.setNumReduceTasks(2);
    conf.set(JobContext.REDUCE_FAILURES_MAXPERCENT, ".70");
    conf.set(JobContext.MAP_FAILURES_MAX_PERCENT, ".70");
    conf.setBoolean(JobContext.SETUP_CLEANUP_NEEDED, false);
    conf.setMaxTaskFailuresPerTracker(1);
    FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker);
    job.setClusterSize(trackers.length);
    job.initTasks();
   
    TaskTracker tt1 = jobTracker.getTaskTracker(trackers[0]);
    TaskTracker tt2 = jobTracker.getTaskTracker(trackers[1]);
    TaskTracker tt3 = jobTracker.getTaskTracker(trackers[2]);
    TaskTrackerStatus status1 = new TaskTrackerStatus(
        trackers[0],JobInProgress.convertTrackerNameToHostName(
            trackers[0]),0,new ArrayList<TaskStatus>(), 0, 2, 2);
    TaskTrackerStatus status2 = new TaskTrackerStatus(
        trackers[1],JobInProgress.convertTrackerNameToHostName(
            trackers[1]),0,new ArrayList<TaskStatus>(), 0, 2, 2);
    TaskTrackerStatus status3 = new TaskTrackerStatus(
        trackers[1],JobInProgress.convertTrackerNameToHostName(
            trackers[1]),0,new ArrayList<TaskStatus>(), 0, 2, 2);
    tt1.setStatus(status1);
    tt2.setStatus(status2);
    tt3.setStatus(status3);
   
    tt1.reserveSlots(TaskType.MAP, job, 2);
    tt1.reserveSlots(TaskType.REDUCE, job, 2);
    tt3.reserveSlots(TaskType.MAP, job, 2);
    tt3.reserveSlots(TaskType.REDUCE, job, 2);
   
    assertEquals("Trackers not reserved for the job : maps",
        2, job.getNumReservedTaskTrackersForMaps());
    assertEquals("Trackers not reserved for the job : reduces",
        2, job.getNumReservedTaskTrackersForReduces());
    ClusterMetrics metrics = jobTracker.getClusterMetrics();
    assertEquals("reserved map slots do not match",
        4, metrics.getReservedMapSlots());
    assertEquals("reserved reduce slots do not match",
        4, metrics.getReservedReduceSlots());
 
    /*
     * FakeJobInProgress.findMapTask does not handle
     * task failures. So working around it by failing
     * reduce and blacklisting tracker.
     * Then finish the map task later.
     */
    TaskAttemptID mTid = job.findMapTask(trackers[0]);
    TaskAttemptID rTid = job.findReduceTask(trackers[0]);
    //Task should blacklist the tasktracker.
    job.failTask(rTid);
   
    assertEquals("Tracker 0 not blacklisted for the job", 1,
        job.getBlackListedTrackers().size());
    assertEquals("Extra Trackers reserved for the job : maps",
        1, job.getNumReservedTaskTrackersForMaps());
    assertEquals("Extra Trackers reserved for the job : reduces",
        1, job.getNumReservedTaskTrackersForReduces());
    metrics = jobTracker.getClusterMetrics();
    assertEquals("reserved map slots do not match",
        2, metrics.getReservedMapSlots());
    assertEquals("reserved reduce slots do not match",
        2, metrics.getReservedReduceSlots());

    //Finish the map task on the tracker 1. Finishing it here to work
    //around bug in the FakeJobInProgress object
    job.finishTask(mTid);
    mTid = job.findMapTask(trackers[1]);
    rTid = job.findReduceTask(trackers[1]);
    job.finishTask(mTid);
    job.finishTask(rTid);
    rTid = job.findReduceTask(trackers[1]);
    job.finishTask(rTid);
    assertEquals("Job didnt complete successfully complete", job.getStatus()
        .getRunState(), JobStatus.SUCCEEDED);
    assertEquals("Trackers not unreserved for the job : maps",
        0, job.getNumReservedTaskTrackersForMaps());
    assertEquals("Trackers not unreserved for the job : reduces",
        0, job.getNumReservedTaskTrackersForReduces());
    metrics = jobTracker.getClusterMetrics();
    assertEquals("reserved map slots do not match",
        0, metrics.getReservedMapSlots());
    assertEquals("reserved reduce slots do not match",
        0, metrics.getReservedReduceSlots());
View Full Code Here

    TaskAttemptID[] taskAttemptID = new TaskAttemptID[20];
    JobConf conf = new JobConf();
    conf.setSpeculativeExecution(true);
    conf.setNumMapTasks(10);
    conf.setNumReduceTasks(0);
    FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker);   
    job.initTasks();
    //schedule some tasks
    taskAttemptID[0] = job.findMapTask(trackers[0]);
    taskAttemptID[1] = job.findMapTask(trackers[0]);
    taskAttemptID[2] = job.findMapTask(trackers[0]);
    taskAttemptID[3] = job.findMapTask(trackers[1]);
    taskAttemptID[4] = job.findMapTask(trackers[1]);
    taskAttemptID[5] = job.findMapTask(trackers[1]);
    taskAttemptID[6] = job.findMapTask(trackers[2]);
    taskAttemptID[7] = job.findMapTask(trackers[2]);
    taskAttemptID[8] = job.findMapTask(trackers[2]);
    clock.advance(1000);
    //Some tasks finish in 1 second (on trackers[0])
    job.finishTask(taskAttemptID[0]);
    job.finishTask(taskAttemptID[1]);
    job.finishTask(taskAttemptID[2]);
    clock.advance(1000);
    //Some tasks finish in 2 second (on trackers[1])
    job.finishTask(taskAttemptID[3]);
    job.finishTask(taskAttemptID[4]);
    job.finishTask(taskAttemptID[5]);
    assertEquals("Tracker "+ trackers[0] + " expected to be not slow ",
        job.isSlowTracker(trackers[0]), false);
    clock.advance(100000);
    //After a long time, some tasks finished on trackers[2]
    job.finishTask(taskAttemptID[6]);
    job.finishTask(taskAttemptID[7]);
    job.finishTask(taskAttemptID[8]);
    assertEquals("Tracker "+ trackers[2] + " expected to be slow ",
        job.isSlowTracker(trackers[2]), true);
    // Verify total speculative tasks by jobtracker instrumentation
    assertEquals("Total speculative maps", 1, fakeInst.numSpeculativeMaps);
    assertEquals("Total speculative reduces", 1,
                 fakeInst.numSpeculativeReduces);
    LOG.info("Total speculative maps = " + fakeInst.numSpeculativeMaps);
View Full Code Here

    JobConf conf = new JobConf();
    conf.setSpeculativeExecution(true);
    conf.setNumMapTasks(5);
    conf.setNumReduceTasks(5);
    conf.setFloat(JobContext.SPECULATIVE_SLOWTASK_THRESHOLD, 0.5f);
    FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker);   
    job.initTasks();
    //schedule maps
    taskAttemptID[0] = job.findReduceTask(trackers[0]);
    taskAttemptID[1] = job.findReduceTask(trackers[1]);
    taskAttemptID[2] = job.findReduceTask(trackers[2]);
    taskAttemptID[3] = job.findReduceTask(trackers[3]);
    taskAttemptID[4] = job.findReduceTask(trackers[3]);
    clock.advance(5000);
    job.finishTask(taskAttemptID[0]);
    clock.advance(1000);
    job.finishTask(taskAttemptID[1]);
    clock.advance(20000);
    clock.advanceBySpeculativeLag();
    //we should get a speculative task now
    taskAttemptID[5] = job.findReduceTask(trackers[4]);
    assertEquals(taskAttemptID[5].getTaskID().getId(),2);
    clock.advance(5000);
    job.finishTask(taskAttemptID[5]);
   
    taskAttemptID[5] = job.findReduceTask(trackers[4]);
    assertEquals(taskAttemptID[5].getTaskID().getId(),3);
   
    // Verify total speculative tasks by jobtracker instrumentation
    assertEquals("Total speculative maps", 1, fakeInst.numSpeculativeMaps);
    assertEquals("Total speculative reduces", 3,
View Full Code Here

    JobConf conf = new JobConf();
    conf.setSpeculativeExecution(true);
    conf.setNumMapTasks(5);
    conf.setNumReduceTasks(0);
    conf.setFloat(JobContext.SPECULATIVE_SLOWTASK_THRESHOLD, 0.5f);
    FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker);
    job.initTasks();

    taskAttemptID[0] = job.findMapTask(trackers[0]);
    taskAttemptID[1] = job.findMapTask(trackers[1]);
    taskAttemptID[2] = job.findMapTask(trackers[2]);
    taskAttemptID[3] = job.findMapTask(trackers[3]);
    clock.advance(2000);
    job.finishTask(taskAttemptID[0]);
    job.finishTask(taskAttemptID[1]);
    job.finishTask(taskAttemptID[2]);
    clock.advance(250000);
    taskAttemptID[4] = job.findMapTask(trackers[3]);
    clock.advanceBySpeculativeLag();
    //by doing the above clock adjustments, we bring the progress rate of
    //taskID 3 lower than 4. For taskID 3, the rate is 85/317000
    //and for taskID 4, the rate is 20/65000. But when we ask for a spec task
    //now, we should get back taskID 4 (since that is expected to complete
    //later than taskID 3).
    job.progressMade(taskAttemptID[3], 0.85f);
    job.progressMade(taskAttemptID[4], 0.20f);
    taskAttemptID[5] = job.findMapTask(trackers[4]);
    assertEquals(taskAttemptID[5].getTaskID().getId(),4);
    // Verify total speculative tasks by jobtracker instrumentation
    assertEquals("Total speculative maps", 2, fakeInst.numSpeculativeMaps);
    assertEquals("Total speculative reduces", 3,
                 fakeInst.numSpeculativeReduces);
View Full Code Here

    JobConf conf = new JobConf();
    conf.setSpeculativeExecution(true);
    conf.setNumMapTasks(totalTasks);
    conf.setNumReduceTasks(0);
    jobTracker.setNumSlots(slots);
    FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker);
    job.initTasks();
    int i;
    for (i = 0; i < totalTasks; i++) {
      taskAttemptID[i] = job.findMapTask(trackers[0]);
    }
    clock.advance(5000);
    for (i = 0; i < numEarlyComplete; i++) {
      job.finishTask(taskAttemptID[i]);
    }

    clock.advanceBySpeculativeLag();

    for (i = numEarlyComplete; i < totalTasks; i++) {
      job.progressMade(taskAttemptID[i], 0.85f);
    }
    clock.advance(50000);
    for (i = 0; i < (totalTasks - numEarlyComplete); i++) {
      taskAttemptID[i] = job.findMapTask(trackers[1]);
      clock.advance(2000);
      if (taskAttemptID[i] != null) {
        //add some good progress constantly for the different
        //task-attempts so that
        //the tasktracker doesn't get into the slow trackers category
        job.progressMade(taskAttemptID[i], 0.99f);
      } else {
        break;
      }
    }
    return i;
View Full Code Here

    JobConf conf = new JobConf();
    conf.setSpeculativeExecution(true);
    conf.setNumMapTasks(3);
    conf.setNumReduceTasks(3);
    conf.setFloat(JobContext.SPECULATIVE_SLOWTASK_THRESHOLD, 0.5f);
    FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker);
    job.initTasks();

    //Check for runningMap counts first
    //schedule maps
    taskAttemptID[0] = job.findMapTask(trackers[0]);
    taskAttemptID[1] = job.findMapTask(trackers[1]);
    taskAttemptID[2] = job.findMapTask(trackers[2]);

    clock.advance(5000);
    job.finishTask(taskAttemptID[0]);
    clock.advance(1000);
    job.finishTask(taskAttemptID[1]);
    clock.advanceBySpeculativeLag();

    //we should get a speculative task now
    taskAttemptID[3] = job.findMapTask(trackers[3]);
    int oldRunningMap = job.runningMaps();
    LOG.info("No of running maps before fail was " + oldRunningMap);
    job.failTask(taskAttemptID[2]);
    assertEquals(
      "Running maps count should be updated from " + oldRunningMap + " to " +
        (oldRunningMap - 1), job.runningMaps(), oldRunningMap - 1);
    LOG.info(" Job running maps after fail " + job.runningMaps());

    clock.advance(5000);
    job.finishTask(taskAttemptID[3]);

    //check for runningReduce count.
    taskAttemptID[4] = job.findReduceTask(trackers[0]);
    taskAttemptID[5] = job.findReduceTask(trackers[1]);
    taskAttemptID[6] = job.findReduceTask(trackers[2]);

    clock.advance(5000);
    job.finishTask(taskAttemptID[4]);
    clock.advance(1000);
    job.finishTask(taskAttemptID[5]);

    clock.advanceBySpeculativeLag();
    taskAttemptID[7] = job.findReduceTask(trackers[4]);

    int oldRunningReduces = job.runningReduces();
    job.failTask(taskAttemptID[6]);
    LOG.info(
      " No of running Reduces before fail " + oldRunningReduces);
    LOG.info(
      " No of runing reduces after fail " + job.runningReduces());
    assertEquals(
      "Running reduces count should be updated from " + oldRunningReduces +
        " to " + (oldRunningReduces - 1), job.runningReduces(),
      oldRunningReduces - 1);
    // Verify total speculative tasks by jobtracker instrumentation
    assertEquals("Total speculative maps", 1, fakeInst.numSpeculativeMaps);
    assertEquals("Total speculative reduces", 1,
                 fakeInst.numSpeculativeReduces);
    LOG.info("Total speculative maps = " + fakeInst.numSpeculativeMaps);
    LOG.info("Total speculative reduces = " + fakeInst.numSpeculativeReduces);
   
    job.finishTask(taskAttemptID[7]);
  }
View Full Code Here

    conf.setSpeculativeExecution(true);
    conf.setNumMapTasks(10);
    conf.setNumReduceTasks(0);
    conf.setFloat(JobInProgress.SPECULATIVE_MAP_UNFINISHED_THRESHOLD_KEY, 0);
    conf.setFloat(JobInProgress.SPECULATIVE_REDUCE_UNFINISHED_THRESHOLD_KEY, 0);
    FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker);
    job.initTasks();
    //schedule some tasks
    taskAttemptID[0] = job.findMapTask(trackers[0]);
    taskAttemptID[1] = job.findMapTask(trackers[0]);
    taskAttemptID[2] = job.findMapTask(trackers[0]);
    taskAttemptID[3] = job.findMapTask(trackers[1]);
    taskAttemptID[4] = job.findMapTask(trackers[1]);
    taskAttemptID[5] = job.findMapTask(trackers[1]);
    taskAttemptID[6] = job.findMapTask(trackers[2]);
    taskAttemptID[7] = job.findMapTask(trackers[2]);
    taskAttemptID[8] = job.findMapTask(trackers[2]);
    clock.advance(1000);
    //Some tasks finish in 1 second (on trackers[0])
    job.finishTask(taskAttemptID[0]);
    job.finishTask(taskAttemptID[1]);
    job.finishTask(taskAttemptID[2]);
    clock.advance(1000);
    //Some tasks finish in 2 second (on trackers[1])
    job.finishTask(taskAttemptID[3]);
    job.finishTask(taskAttemptID[4]);
    job.finishTask(taskAttemptID[5]);
    assertEquals("Tracker "+ trackers[0] + " expected to be not slow ",
        job.isSlowTracker(trackers[0]), false);
    clock.advance(100000);
    //After a long time, some tasks finished on trackers[2]
    job.finishTask(taskAttemptID[6]);
    job.finishTask(taskAttemptID[7]);
    job.finishTask(taskAttemptID[8]);
    job.refresh(clock.getTime());
    assertEquals("Tracker "+ trackers[2] + " expected to be slow ",
        job.isSlowTracker(trackers[2]), true);
  }
View Full Code Here

    conf.setNumReduceTasks(5);
    conf.setFloat(JobInProgress.SPECULATIVE_SLOWNODE_THRESHOLD, 100f);
    conf.setFloat(JobInProgress.SPECULATIVE_SLOWTASK_THRESHOLD, 0.5f);
    conf.setFloat(JobInProgress.SPECULATIVE_MAP_UNFINISHED_THRESHOLD_KEY, 0);
    conf.setFloat(JobInProgress.SPECULATIVE_REDUCE_UNFINISHED_THRESHOLD_KEY, 0);
    FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker);
    job.initTasks();
    //schedule maps
    taskAttemptID[0] = job.findReduceTask(trackers[0]);
    taskAttemptID[1] = job.findReduceTask(trackers[1]);
    taskAttemptID[2] = job.findReduceTask(trackers[2]);
    taskAttemptID[3] = job.findReduceTask(trackers[3]);
    taskAttemptID[4] = job.findReduceTask(trackers[3]);
    clock.advance(5000);
    job.finishTask(taskAttemptID[0]);
    clock.advance(1000);
    job.finishTask(taskAttemptID[1]);
    clock.advance(20000);
    clock.advanceBySpeculativeLag();
    job.refresh(clock.getTime());
    //we should get a speculative task now
    taskAttemptID[5] = job.findReduceTask(trackers[4]);
    assertEquals(taskAttemptID[5].getTaskID().getId(),2);
    clock.advance(5000);
    job.finishTask(taskAttemptID[5]);

    job.refresh(clock.getTime());
    taskAttemptID[5] = job.findReduceTask(trackers[4]);
    assertEquals(taskAttemptID[5].getTaskID().getId(),3);
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobInProgress

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.