Examples of org.apache.hadoop.mapred.TaskAttemptID

org.apache.hadoop.mapred.TaskAttemptID
TaskAttemptID represents the immutable and unique identifier for a task attempt. Each task attempt is one particular instance of a Map or Reduce Task identified by its TaskID. TaskAttemptID consists of 2 parts. First part is the {@link TaskID}, that this TaskAttemptID belongs to. Second part is the task attempt number.
An example TaskAttemptID is : attempt_200707121733_0003_m_000005_0 , which represents the zeroth task attempt for the fifth map task in the third job running at the jobtracker started at 200707121733.
Applications should never construct or parse TaskAttemptID strings , but rather use appropriate constructors or {@link #forName(String)} method. @see JobID @see TaskID

    HiveApiOutputFormat outputFormat = new HiveApiOutputFormat();
    outputFormat.setMyProfileId(PROFILE_ID);


    JobConf jobConf = new JobConf(conf);
    TaskAttemptContext taskContext =
        new HackTaskAttemptContext(jobConf, new TaskAttemptID());
    JobContext jobContext = new HackJobContext(jobConf, taskContext.getJobID());
    outputFormat.checkOutputSpecs(jobContext);
    fail();
  }

View Full Code Here

    outputConf.writeOutputDescription(outputDesc);
    HiveApiOutputFormat outputFormat = new HiveApiOutputFormat();
    outputFormat.setMyProfileId(PROFILE_ID);
    JobConf jobConf = new JobConf(conf);
    TaskAttemptContext taskContext =
        new HackTaskAttemptContext(jobConf, new TaskAttemptID());
    JobContext jobContext = new HackJobContext(jobConf, taskContext.getJobID());


    outputFormat.checkOutputSpecs(jobContext);


    fail();

View Full Code Here

    outputConf.writeOutputDescription(outputDesc);
    HiveApiOutputFormat outputFormat = new HiveApiOutputFormat();
    outputFormat.setMyProfileId(PROFILE_ID);
    JobConf jobConf = new JobConf(conf);
    TaskAttemptContext taskContext =
        new HackTaskAttemptContext(jobConf, new TaskAttemptID());
    JobContext jobContext = new HackJobContext(jobConf, taskContext.getJobID());


    HiveMetastores.setTestClient(metastore);


    outputFormat.checkOutputSpecs(jobContext);

View Full Code Here

    }
    cmd.add(executable);
    // wrap the command in a stdout/stderr capture
    // we are starting map/reduce task of the pipes job. this is not a cleanup
    // attempt. 
    TaskAttemptID taskid = 
      TaskAttemptID.forName(conf.get(MRJobConfig.TASK_ATTEMPT_ID));
    File stdout = TaskLog.getTaskLogFile(taskid, false, TaskLog.LogName.STDOUT);
    File stderr = TaskLog.getTaskLogFile(taskid, false, TaskLog.LogName.STDERR);
    long logLength = TaskLog.getTaskLogLength(conf);
    cmd = TaskLog.captureOutAndError(null, cmd, stdout, stderr, logLength,

View Full Code Here

    throws IOException, InterruptedException {
    ImmutableClassesGiraphConfiguration iconf = new ImmutableClassesGiraphConfiguration(conf);
    WrappedVertexOutputFormat outputFormat = iconf.createWrappedVertexOutputFormat();
    JobConf jobConf = new JobConf(conf);
    TaskAttemptContext
        taskContext = new HackTaskAttemptContext(jobConf, new TaskAttemptID());
    OutputCommitter outputCommitter = outputFormat.getOutputCommitter(
        taskContext);
    JobContext jobContext = new HackJobContext(jobConf, taskContext.getJobID());
    outputCommitter.commitJob(jobContext);
  }

View Full Code Here

  }


  public static JobContext makeJobContext(Configuration conf) {
    JobConf jobConf = new JobConf(conf);
    TaskAttemptContext
        taskContext = new HackTaskAttemptContext(jobConf, new TaskAttemptID());
    return new HackJobContext(jobConf, taskContext.getJobID());
  }

View Full Code Here

  public void open(int taskNumber, int numTasks) throws IOException {
    if (Integer.toString(taskNumber + 1).length() > 6) {
      throw new IOException("Task id too large.");
    }
    
    TaskAttemptID taskAttemptID = TaskAttemptID.forName("attempt__0000_r_" 
        + String.format("%" + (6 - Integer.toString(taskNumber + 1).length()) + "s"," ").replace(" ", "0") 
        + Integer.toString(taskNumber + 1) 
        + "_0");
    
    this.jobConf.set("mapred.task.id", taskAttemptID.toString());
    this.jobConf.setInt("mapred.task.partition", taskNumber + 1);
    // for hadoop 2.2
    this.jobConf.set("mapreduce.task.attempt.id", taskAttemptID.toString());
    this.jobConf.setInt("mapreduce.task.partition", taskNumber + 1);
    
    try {
      this.context = HadoopUtils.instantiateTaskAttemptContext(this.jobConf, taskAttemptID);
    } catch (Exception e) {

View Full Code Here


        File pearFile = new File(urlPEAR.getPath());


        // should check whether a different mapper has already unpacked it
        // but for now we just unpack in a different location for every mapper
        TaskAttemptID attempt = TaskAttemptID.forName(conf
                .get("mapred.task.id"));
        installDir = new File(pearFile.getParentFile(), attempt.toString());
        PackageBrowser instPear = PackageInstaller.installPackage(installDir,
                pearFile, true);


        // get the resources required for the AnalysisEngine
        org.apache.uima.resource.ResourceManager rsrcMgr = UIMAFramework

View Full Code Here

      public void addFetchFailedMap(TaskAttemptID mapTaskId) {
      }
    };
    Progress progress = new Progress();


    TaskAttemptID reduceId = new TaskAttemptID("314159", 0, TaskType.REDUCE,
        0, 0);
    ShuffleSchedulerImpl scheduler = new ShuffleSchedulerImpl(job, status,
        reduceId, null, progress, null, null, null);


    JobID jobId = new JobID();

View Full Code Here


  @Test
  public void testConsecutiveFetch()
      throws IOException, InterruptedException {
    final int MAX_EVENTS_TO_FETCH = 100;
    TaskAttemptID tid = new TaskAttemptID("12345", 1, TaskType.REDUCE, 1, 1);


    TaskUmbilicalProtocol umbilical = mock(TaskUmbilicalProtocol.class);
    when(umbilical.getMapCompletionEvents(any(JobID.class),
        anyInt(), anyInt(), any(TaskAttemptID.class)))
      .thenReturn(getMockedCompletionEventsUpdate(0, 0));

View Full Code Here

0 1 2 3 4 5 6

TOP

Related Classes of org.apache.hadoop.mapred.TaskAttemptID

com.digitalpebble.behemoth.uima.UIMAMapper

com.facebook.hiveio.output.CheckOutputSpecsTest

com.facebook.hiveio.output.FaultyCheckOutputSpecsTest

com.facebook.hiveio.output.HiveOutput

com.facebook.hiveio.output.PerThread

org.apache.avro.mapred.tether.TetheredProcess

org.apache.blur.mapred.AbstractOutputCommitter

org.apache.blur.mapreduce.lib.BlurOutputCommitter

org.apache.flink.hadoopcompatibility.mapred.HadoopOutputFormat

org.apache.giraph.hive.Helpers

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.