* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

  static final Log LOG = LogFactory.getLog(CoronaJobInProgress.class);

  Clock clock;

  @SuppressWarnings("deprecation")

  JobID jobId;

  SessionPriority priority;

  JobProfile profile;

  JobStatus status;

  private final JobStats jobStats = new JobStats();

  long startTime;

  long launchTime;

  long finishTime;

  long deadline;

  String user;

  @SuppressWarnings("deprecation")

  JobConf jobConf;

  Path jobFile;  // non-local

  Path localJobFile;

  // XXX: Do not limit number of tasks ourselves. Let Cluster Manager handle it.

  int maxTasks = 0;

  int numMapTasks;

  int numReduceTasks;

  long memoryPerMap;

  long memoryPerReduce;

  volatile int numSlotsPerMap = 1;

  volatile int numSlotsPerReduce = 1;

  List<TaskCompletionEvent> taskCompletionEvents;

  private int taskCompletionEventCounter = 0;

  static final float DEFAULT_COMPLETED_MAPS_PERCENT_FOR_REDUCE_SLOWSTART = 0.05f;

  public static final String SPECULATIVE_MAP_UNFINISHED_THRESHOLD_KEY =

      "mapred.map.tasks.speculation.unfinished.threshold";

  public static final String SPECULATIVE_REDUCE_UNFINISHED_THRESHOLD_KEY =

      "mapred.reduce.tasks.speculation.unfinished.threshold";

  static final int NUM_SLOTS_PER_MAP = 1;

  static final int NUM_SLOTS_PER_REDUCE = 1;

  private final boolean jobSetupCleanupNeeded;

  private final boolean jobFinishWhenReducesDone;

  private final boolean taskCleanupNeeded;

  private volatile boolean launchedSetup = false;

  private volatile boolean tasksInited = false;

  private final AtomicBoolean reduceResourcesRequested = new AtomicBoolean(false);

  private volatile boolean launchedCleanup = false;

  private volatile boolean jobKilled = false;

  private volatile boolean jobFailed = false;

  String[][] mapLocations;  // Has an array of locations for each map.

  List<TaskInProgress> nonRunningMaps;

  List<TaskInProgress> nonRunningReduces = new LinkedList<TaskInProgress>();

  int runningMapTasks = 0;

  int runningReduceTasks = 0;

  int failedMapTasks = 0;  // includes killed

  int failedReduceTasks = 0;  // includes killed

  int killedMapTasks = 0;

  int killedReduceTasks = 0;

  int failedMapTIPs = 0;

  int failedReduceTIPs = 0;

  int finishedMapTasks = 0;

  int finishedReduceTasks = 0;

  int completedMapsForReduceSlowstart = 0;

  int rushReduceReduces = 5;

  int rushReduceMaps = 5;

  float speculativeMapUnfininshedThreshold = 0.001F;

  float speculativeReduceUnfininshedThreshold = 0.001F;

  float speculativeMapLogRateThreshold = 0.001F;

  int speculativeMapLogNumThreshold = 3;

  float speculativeReduceLogRateThreshold = 0.001F;

  int speculativeReduceLogNumThreshold = 3;

  Set<TaskInProgress> runningMaps = new LinkedHashSet<TaskInProgress>();

  Set<TaskInProgress> runningReduces = new LinkedHashSet<TaskInProgress>();

  @SuppressWarnings("deprecation")

  List<TaskAttemptID> mapCleanupTasks = new LinkedList<TaskAttemptID>();

  @SuppressWarnings("deprecation")

  List<TaskAttemptID> reduceCleanupTasks = new LinkedList<TaskAttemptID>();

  int maxLevel;

  int anyCacheLevel;

  private final LocalityStats localityStats;

  private final Thread localityStatsThread;

  // runningMapTasks include speculative tasks, so we need to capture

  // speculative tasks separately

  int speculativeMapTasks = 0;

  int speculativeReduceTasks = 0;

  int mapFailuresPercent = 0;

  int reduceFailuresPercent = 0;

  @SuppressWarnings("deprecation")

  Counters jobCounters = new Counters();

  TaskErrorCollector taskErrorCollector;

  // Maximum no. of fetch-failure notifications after which

  // the map task is killed

  private static final int MAX_FETCH_FAILURES_NOTIFICATIONS = 3;

  private static final int MAX_FETCH_FAILURES_PER_MAP_DEFAULT = 50;

  private static final String MAX_FETCH_FAILURES_PER_MAP_KEY =

     "mapred.job.per.map.maxfetchfailures";

  private int maxFetchFailuresPerMapper;

  // The key for the property holding the job deadline value

  private static final String JOB_DEADLINE_KEY = "mapred.job.deadline";

  // The default value of the job deadline property

  private static final long JOB_DEADLINE_DEFAULT_VALUE = 0L;

  // The key for the property holding the job priority

  private static final String SESSION_PRIORITY_KEY = "mapred.job.priority";

  // The default value of the job priority

  private static final String SESSION_PRIORITY_DEFAULT = "NORMAL";

  // The maximum percentage of fetch failures allowed for a map

  private static final double MAX_ALLOWED_FETCH_FAILURES_PERCENT = 0.5;

  // Map of mapTaskId -> no. of fetch failures

  private final Map<TaskAttemptID, Integer> mapTaskIdToFetchFailuresMap =

    new TreeMap<TaskAttemptID, Integer>();

  // Don't lower speculativeCap below one TT's worth (for small clusters)

  private static final int MIN_SPEC_CAP = 10;

  public static final String SPECULATIVE_SLOWTASK_THRESHOLD =

    "mapreduce.job.speculative.slowtaskthreshold";

  public static final String RUSH_REDUCER_MAP_THRESHOLD =

    "mapred.job.rushreduce.map.threshold";

  public static final String RUSH_REDUCER_REDUCE_THRESHOLD =

    "mapred.job.rushreduce.reduce.threshold";

  public static final String SPECULATIVECAP =

    "mapreduce.job.speculative.speculativecap";

  public static final String SPECULATIVE_SLOWNODE_THRESHOLD =

    "mapreduce.job.speculative.slownodethreshold";

  public static final String SPECULATIVE_REFRESH_TIMEOUT =

    "mapreduce.job.speculative.refresh.timeout";

  public static final String LOG_CANNOTSPECULATIVE_INTERVAL =

    "mapreduce.job.log.cannotspeculative.interval";

  public static final String SPECULATIVE_STDDEVMEANRATIO_MAX =

    "mapreduce.job.speculative.stddevmeanratio.max";

  // thresholds for speculative execution

  float slowTaskThreshold;

  float speculativeCap;

  float slowNodeThreshold;

  private long lastSpeculativeMapRefresh, lastSpeculativeReduceRefresh;

  private long lastTimeCannotspeculativeMapLog, lastTimeCannotspeculativeReduceLog;

  private final long speculativeRefreshTimeout;

  private final long logCannotspeculativeInterval;

  private final float speculativeStddevMeanRatioMax;

  volatile private boolean hasSpeculativeMaps;

  volatile private boolean hasSpeculativeReduces;

  private List<TaskInProgress> candidateSpeculativeMaps, candidateSpeculativeReduces;

  //Statistics are maintained for a couple of things

  //mapTaskStats is used for maintaining statistics about

  //the completion time of map tasks on the trackers. On a per

  //tracker basis, the mean time for task completion is maintained

  private final DataStatistics mapTaskStats = new DataStatistics();

  //reduceTaskStats is used for maintaining statistics about

  //the completion time of reduce tasks on the trackers. On a per

  //tracker basis, the mean time for task completion is maintained

  private final DataStatistics reduceTaskStats = new DataStatistics();

  //trackerMapStats used to maintain a mapping from the tracker to the

  //the statistics about completion time of map tasks

  private final Map<String,DataStatistics> trackerMapStats =

    new HashMap<String,DataStatistics>();

  //trackerReduceStats used to maintain a mapping from the tracker to the

  //the statistics about completion time of reduce tasks

  private final Map<String,DataStatistics> trackerReduceStats =

    new HashMap<String,DataStatistics>();

  //runningMapStats used to maintain the RUNNING map tasks' statistics

  private final DataStatistics runningMapTaskStats = new DataStatistics();

  //runningReduceStats used to maintain the RUNNING reduce tasks' statistics

  private final DataStatistics runningReduceTaskStats = new DataStatistics();

  private static final String JOB_KILLED_REASON = "Job killed";

  private static final String EMPTY_TRACKER_NAME =

    "tracker_:localhost.localdomain/127.0.0.1:";

  protected CoronaJobTracker.TaskLookupTable taskLookupTable;

  private final TaskStateChangeListener taskStateChangeListener;

  private final Object lockObject;

  private final CoronaJobHistory jobHistory;

 

  private String jobTrackerId;

  private int terminated = -1;

  @SuppressWarnings("deprecation")

  public CoronaJobInProgress(

    Object lockObject,

    JobID jobId, Path systemDir, JobConf jobConf,

    CoronaJobTracker.TaskLookupTable taskLookupTable,

    TaskStateChangeListener taskStateChangeListener,

    TopologyCache topologyCache,

    CoronaJobHistory jobHistory, String url, String jobTrackerId)

    throws IOException {

    this.lockObject = lockObject;

    this.clock = JobTracker.getClock();

    this.jobId = jobId;

    this.jobConf = jobConf;

    this.taskLookupTable = taskLookupTable;

    this.taskStateChangeListener = taskStateChangeListener;

    this.jobHistory = jobHistory;

    this.jobTrackerId = jobTrackerId;

    // Status.

    this.startTime = clock.getTime();

    this.status = new JobStatus(jobId, 0.0f, 0.0f, JobStatus.PREP);

    status.setStartTime(startTime);

    // Job file.

    this.jobFile = getJobFile(systemDir, jobId);

    this.user = jobConf.getUser();

    this.profile = new JobProfile(

      user, jobId, jobFile.toString(), url, jobConf.getJobName(),

      jobConf.getQueueName());

    this.numMapTasks = jobConf.getNumMapTasks();

    this.numReduceTasks = jobConf.getNumReduceTasks();

    this.memoryPerMap = jobConf.getMemoryForMapTask();

    this.memoryPerReduce = jobConf.getMemoryForReduceTask();

    this.taskCompletionEvents = new ArrayList<TaskCompletionEvent>

       (numMapTasks + numReduceTasks + 10);

    this.jobSetupCleanupNeeded = jobConf.getJobSetupCleanupNeeded();

    this.jobFinishWhenReducesDone = jobConf.getJobFinishWhenReducesDone();

    this.taskCleanupNeeded = jobConf.getTaskCleanupNeeded();

    this.mapFailuresPercent = jobConf.getMaxMapTaskFailuresPercent();

    this.reduceFailuresPercent = jobConf.getMaxReduceTaskFailuresPercent();

    this.maxLevel = jobConf.getInt("mapred.task.cache.levels",

        NetworkTopology.DEFAULT_HOST_LEVEL);

    this.anyCacheLevel = this.maxLevel+1;

    this.localityStats = new LocalityStats(

      jobConf, maxLevel, jobCounters, jobStats, topologyCache);

    localityStatsThread = new Thread(localityStats);

    localityStatsThread.setName("Locality Stats");

    localityStatsThread.setDaemon(true);

    localityStatsThread.start();

    this.taskErrorCollector = new TaskErrorCollector(

      jobConf, Integer.MAX_VALUE, 1);

    this.slowTaskThreshold = Math.max(0.0f,

        jobConf.getFloat(CoronaJobInProgress.SPECULATIVE_SLOWTASK_THRESHOLD,1.0f));

    this.speculativeCap = jobConf.getFloat(

        CoronaJobInProgress.SPECULATIVECAP,0.1f);

    this.slowNodeThreshold = jobConf.getFloat(

        CoronaJobInProgress.SPECULATIVE_SLOWNODE_THRESHOLD,1.0f);

    this.speculativeRefreshTimeout = jobConf.getLong(

        CoronaJobInProgress.SPECULATIVE_REFRESH_TIMEOUT, 5000L);

    this.logCannotspeculativeInterval = jobConf.getLong(

        CoronaJobInProgress.LOG_CANNOTSPECULATIVE_INTERVAL, 150000L);

    this.speculativeStddevMeanRatioMax = jobConf.getFloat(

        CoronaJobInProgress.SPECULATIVE_STDDEVMEANRATIO_MAX, 0.33f);

    this.speculativeMapUnfininshedThreshold = jobConf.getFloat(

        CoronaJobInProgress.SPECULATIVE_MAP_UNFINISHED_THRESHOLD_KEY,

        speculativeMapUnfininshedThreshold);

    this.speculativeReduceUnfininshedThreshold = jobConf.getFloat(

        CoronaJobInProgress.SPECULATIVE_REDUCE_UNFINISHED_THRESHOLD_KEY,

        speculativeReduceUnfininshedThreshold);

    this.deadline = jobConf.getLong(JOB_DEADLINE_KEY,

        JOB_DEADLINE_DEFAULT_VALUE);

    this.priority = SessionPriority.valueOf(

        jobConf.get(SESSION_PRIORITY_KEY, SESSION_PRIORITY_DEFAULT));

    hasSpeculativeMaps = jobConf.getMapSpeculativeExecution();

    hasSpeculativeReduces = jobConf.getReduceSpeculativeExecution();

    LOG.info(jobId + ": hasSpeculativeMaps = " + hasSpeculativeMaps +

             ", hasSpeculativeReduces = " + hasSpeculativeReduces);

  }

  public JobStats getJobStats() {

    return jobStats;

  }

  @Override

  public String getUser() { return user; }

  public JobProfile getProfile() {  return profile; }

  @Override

  public JobStatus getStatus() { return status; }

  public boolean isSetupCleanupRequired() { return jobSetupCleanupNeeded; }

  public SessionPriority getPriority() {

    return this.priority;

  }

 

  public void setPriority(SessionPriority priority) {

    this.priority = priority;

  }

  @Override

  DataStatistics getRunningTaskStatistics(boolean isMap) {

    if (isMap) {

      return runningMapTaskStats;

    } else {

      return runningReduceTaskStats;

    }

  }

  @Override

  public float getSlowTaskThreshold() { return slowTaskThreshold; }

  @Override

  public float getStddevMeanRatioMax() { return speculativeStddevMeanRatioMax; }

  @Override

  public int getNumRestarts() { return 0; }

  public long getLaunchTime() { return launchTime; }

  public long getStartTime() { return startTime; }

  public long getFinishTime() { return finishTime; }

  public long getJobDeadline() { return deadline; }

  public int getNumMapTasks() { return numMapTasks; }

  public int getNumReduceTasks() { return numReduceTasks; }

  @SuppressWarnings("deprecation")

  public Counters getJobCounters() { return jobCounters; }

  /**

   *  Returns map phase counters by summing over all map tasks in progress.

   */

  @SuppressWarnings("deprecation")

  public Counters getMapCounters() {

    synchronized (lockObject) {

      return incrementTaskCountersUnprotected(new Counters(), maps);

    }

  }

  /**

   *  Returns map phase counters by summing over all map tasks in progress.

   */

  @SuppressWarnings("deprecation")

  public Counters getReduceCounters() {

    synchronized (lockObject) {

      return incrementTaskCountersUnprotected(new Counters(), reduces);

    }

  }

  /**

   * Get all the tasks of the desired type in this job.

   */

  TaskInProgress[] getTasks(TaskType type) {

    TaskInProgress[] tasks = null;

    switch (type) {

    case MAP:

      tasks = maps;

      break;

    case REDUCE:

      tasks = reduces;

      break;

    case JOB_SETUP:

      tasks = setup;

      break;

    case JOB_CLEANUP:

      tasks = cleanup;

      break;

    default:

      tasks = new TaskInProgress[0];

      break;

    }

    return tasks;

  }

  public TaskCompletionEvent[] getTaskCompletionEvents(

        int fromEventId, int maxEvents) {

    TaskCompletionEvent[] events = TaskCompletionEvent.EMPTY_ARRAY;

    synchronized (lockObject) {

      if (!tasksInited) {

        return events;

      }

      if (taskCompletionEvents.size() > fromEventId) {

        int actualMax = Math.min(maxEvents,

                                 (taskCompletionEvents.size() - fromEventId));

        events = taskCompletionEvents.subList(fromEventId,

          actualMax + fromEventId).toArray(events);

      }

      return events;

    }

  }

  public int getTaskCompletionEventsSize() {

    synchronized (lockObject) {

      return taskCompletionEvents.size();

    }

  }

  boolean fetchFailureNotification(

        TaskAttemptID reportingAttempt,

        TaskInProgress tip, TaskAttemptID mapAttemptId, String trackerName) {

    jobStats.incNumMapFetchFailures();

    synchronized (lockObject) {

      Integer fetchFailures = mapTaskIdToFetchFailuresMap.get(mapAttemptId);

      fetchFailures = (fetchFailures == null) ? 1 : (fetchFailures + 1);

      mapTaskIdToFetchFailuresMap.put(mapAttemptId, fetchFailures);

      LOG.info("Failed fetch notification #" + fetchFailures + " by " + reportingAttempt +

          " for task " + mapAttemptId + " tracker " + trackerName);

      float failureRate = (float)fetchFailures / runningReduceTasks;

      // declare faulty if fetch-failures >= max-allowed-failures

      final boolean isMapFaulty = (failureRate >= MAX_ALLOWED_FETCH_FAILURES_PERCENT) ||

          fetchFailures > maxFetchFailuresPerMapper;

      if (fetchFailures >= MAX_FETCH_FAILURES_NOTIFICATIONS && isMapFaulty) {

        String reason = "Too many fetch-failures (" + fetchFailures + ") at " +

          new Date();

        LOG.info(reason + " for " + mapAttemptId + " ... killing it");

        final boolean isFailed = true;

        TaskTrackerInfo ttStatus = null;

        jobStats.incNumMapTasksFailedByFetchFailures();

        failedTask(tip, mapAttemptId, reason,

            (tip.isMapTask() ? TaskStatus.Phase.MAP : TaskStatus.Phase.REDUCE),

            isFailed, trackerName, ttStatus);

        mapTaskIdToFetchFailuresMap.remove(mapAttemptId);

        return true;

      }

    }

    return false;

  }

  @SuppressWarnings("deprecation")

  public static Path getJobFile(Path systemDir, JobID jobId) {

    Path systemDirForJob = new Path(systemDir, jobId.toString());

    return new Path(systemDirForJob, "job.xml");

  }

  public int getMaxTasksPerJob() {

    return 0; // No maximum.

  }

  public void close() {

    localityStats.stop();

    localityStatsThread.interrupt();

    // The thread may be stuck in DNS lookups. This thread is a daemon,

    // so it will not prevent process exit.

    try {

      localityStatsThread.join(1000);

    } catch (InterruptedException e) {

      LOG.warn("localityStatsThread.join interrupted");

    }

  }

  /**

   * Read input splits and create a map per split.

   */

  public void initTasks() throws IOException {

    // log job info

    jobHistory.logSubmitted(jobFile.toString(), this.startTime, this.jobTrackerId);

    // log the job priority

    JobClient.RawSplit[] splits = null;

    splits = JobClient.getAndRemoveCachedSplits(jobId);

    if (splits == null) {

      FileSystem fs = jobFile.getFileSystem(jobConf);

      Path splitFile = new Path(jobFile.getParent(), "job.split");

      LOG.info("Reading splits from " + splitFile);

      DataInputStream splitFileIn = fs.open(splitFile);

      try {

        splits = JobClient.readSplitFile(splitFileIn);

      } finally {

        splitFileIn.close();

      }

    }

    initTasksFromSplits(splits);

    jobHistory.logInited(this.launchTime,

                         numMapTasks, numReduceTasks);

  }

  /**

   * Used by test code.

   */

  void initTasksFromSplits(JobClient.RawSplit[] splits) throws IOException {

    synchronized (lockObject) {

      initTasksFromSplitsUnprotected(splits);

    }

  }

  private void initTasksFromSplitsUnprotected(JobClient.RawSplit[] splits)

      throws IOException {

    String jobFile = profile.getJobFile();

    numMapTasks = splits.length;

    if (maxTasks > 0 && numMapTasks + numReduceTasks > maxTasks) {

      throw new IOException(

                "The number of tasks for this job " +

                (numMapTasks + numReduceTasks) +

                " exceeds the configured limit " + maxTasks);

    }

    long inputLength = 0;

    maps = new TaskInProgress[numMapTasks];

    mapLocations = new String[numMapTasks][];

    nonRunningMaps = new ArrayList<TaskInProgress>(numMapTasks);

    for(int i = 0; i < numMapTasks; i++) {

      inputLength += splits[i].getDataLength();

      maps[i] = new TaskInProgress(jobId, jobFile, splits[i],

                                 jobConf, this, i, 1); // numSlotsPerMap = 1

      nonRunningMaps.add(maps[i]);

      mapLocations[i] = splits[i].getLocations();

    }

    LOG.info("Input size for job " + jobId + " = " + inputLength

        + ". Number of splits = " + splits.length);

    this.launchTime = clock.getTime();

    LOG.info("Number of splits for job " + jobId + " = " + splits.length);

    // Create reduce tasks

    this.reduces = new TaskInProgress[numReduceTasks];

    for (int i = 0; i < numReduceTasks; i++) {

      reduces[i] = new TaskInProgress(jobId, jobFile, numMapTasks, i,

                                  jobConf, this, 1); // numSlotsPerReduce = 1

      nonRunningReduces.add(reduces[i]);

    }

    LOG.info("Number of reduces for job " + jobId + " = " + reduces.length);

    // Calculate the minimum number of maps to be complete before

    // we should start scheduling reduces

    completedMapsForReduceSlowstart =

      (int)Math.ceil(

          (jobConf.getFloat("mapred.reduce.slowstart.completed.maps",

                         DEFAULT_COMPLETED_MAPS_PERCENT_FOR_REDUCE_SLOWSTART) *

           numMapTasks));

    // The thresholds of total maps and reduces for scheduling reducers

    // immediately.

    rushReduceMaps =

      jobConf.getInt(RUSH_REDUCER_MAP_THRESHOLD, rushReduceMaps);

    rushReduceReduces =

      jobConf.getInt(RUSH_REDUCER_REDUCE_THRESHOLD, rushReduceReduces);

    maxFetchFailuresPerMapper = jobConf.getInt(MAX_FETCH_FAILURES_PER_MAP_KEY,

      MAX_FETCH_FAILURES_PER_MAP_DEFAULT);

    // Proceed to Setup/Cleanup.

    if (jobSetupCleanupNeeded) {

      // create cleanup two cleanup tips, one map and one reduce.

      cleanup = new TaskInProgress[2];

      // cleanup map tip. This map doesn't use any splits. Just assign an empty

      // split.

      JobClient.RawSplit emptySplit = new JobClient.RawSplit();

      cleanup[0] = new TaskInProgress(jobId, jobFile, emptySplit,

              jobConf, this, numMapTasks, 1);

      cleanup[0].setJobCleanupTask();

      // cleanup reduce tip.

      cleanup[1] = new TaskInProgress(jobId, jobFile, numMapTasks,

                         numReduceTasks, jobConf, this, 1);

      cleanup[1].setJobCleanupTask();

      // create two setup tips, one map and one reduce.

      setup = new TaskInProgress[2];

      // setup map tip. This map doesn't use any split. Just assign an empty

      // split.

      setup[0] = new TaskInProgress(jobId, jobFile, emptySplit,

              jobConf, this, numMapTasks + 1, 1);

      setup[0].setJobSetupTask();

      // setup reduce tip.

      setup[1] = new TaskInProgress(jobId, jobFile, numMapTasks,

                         numReduceTasks + 1, jobConf, this, 1);

      setup[1].setJobSetupTask();

    }

    tasksInited = true;

  }

  boolean scheduleReducesUnprotected() {

    // Start scheduling reducers if we have enough maps finished or

    // if the job has very few mappers or reducers.

    return numMapTasks <= rushReduceMaps ||

           numReduceTasks <= rushReduceReduces ||

           finishedMapTasks >= completedMapsForReduceSlowstart;

  }

  /**

   * Signals that the reduce resources are being requested

   * as soon as one process starts this nobody else should be

   * trying to request reduce resources, so get current and set to true

   *

   * @return false if the resources have not been requested yet,

   * true if they have

   */

  boolean initializeReducers() {

    return this.reduceResourcesRequested.getAndSet(true);

  }

  boolean areReducersInitialized() {

    return this.reduceResourcesRequested.get();

  }

  public Task obtainNewMapTaskForTip(

      String taskTrackerName, String hostName, TaskInProgress intendedTip) {

    synchronized (lockObject) {

      Task result = obtainTaskCleanupTask(taskTrackerName, intendedTip);

      if (result != null) {

        return result;

      }

      if (status.getRunState() != JobStatus.RUNNING) {

        return null;

      }

      TaskInProgress tip = removeMatchingTipUnprotected(nonRunningMaps, hostName, intendedTip);

      if (tip != null) {

        LOG.info("Running task " + tip.getTIPId() + " on " + taskTrackerName +

            "(" + hostName + ")");

      }

      if (tip == null && hasSpeculativeMaps) {

        SpeculationStatus speculationStatus = confirmSpeculativeTaskUnprotected(

            candidateSpeculativeMaps, intendedTip,

            taskTrackerName, hostName, TaskType.MAP);

        if (speculationStatus == SpeculationStatus.CAN_BE_SPECULATED) {

          tip = intendedTip;

          LOG.info("Speculating task " + tip.getTIPId() + " on " + taskTrackerName +

              "(" + hostName + ")");

        } else {

          LOG.warn("Cant speculate for given resource " +

            taskTrackerName + " because " + speculationStatus);

        }

      }

      if (tip == null) {

        return null;

      } else  if (tip != intendedTip) {

        throw new RuntimeException("Logic error:" + tip.getTIPId() +

            " was chosen instead of " + intendedTip.getTIPId());

      }

      scheduleMapUnprotected(tip);

      result = tip.getTaskToRun(taskTrackerName);

      if (result != null) {

        addRunningTaskToTIPUnprotected(tip, result.getTaskID(),

            taskTrackerName, hostName, true);

      }

      return result;

    }

  }

 

  /**

   * Registers new task attempt for given task

   * @param taskTrackerName name of destination tracker

   * @param hostName hostname of tracker

   * @param forcedTip task in progress to run

   * @return task to run

   */

  public Task forceNewMapTaskForTip(String taskTrackerName, String hostName,

      TaskInProgress forcedTip) {

    synchronized (lockObject) {

      Task result = obtainTaskCleanupTask(taskTrackerName, forcedTip);

      if (result != null) {