Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.TaskAttemptID

118
119
120
121
122
123
124
125
126
127
128
      // In case of DefaultTaskController, set permissions here.
      FileUtil.chmod(executable, "u+x");
    }
    cmd.add(executable);
    // wrap the command in a stdout/stderr capture
    TaskAttemptID taskid = TaskAttemptID.forName(conf.get("mapred.task.id"));
    // we are starting map/reduce task of the pipes job. this is not a cleanup
    // attempt.
    File stdout = TaskLog.getTaskLogFile(taskid, false, TaskLog.LogName.STDOUT);
    File stderr = TaskLog.getTaskLogFile(taskid, false, TaskLog.LogName.STDERR);
    long logLength = TaskLog.getTaskLogLength(conf);
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
87
      Path path = fileStatus.getPath();
      String name = path.getName();
      boolean taskComplete = name.endsWith(".task_complete");
      if (fileStatus.isDir()) {
        String taskAttemptName = getTaskAttemptName(name);
        TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskAttemptName);
        if (taskAttemptID.getJobID().equals(jobContext.getJobID())) {
          if (commit) {
            if (taskComplete) {
              fileSystem.rename(path, new Path(shardPath, taskAttemptName + ".commit"));
              LOG.info("Committing [{0}] in path [{1}]", taskAttemptID, path);
            } else {
View Full Code Here
148
149
150
151
152
153
154
155
156
157
158
    if (System.getProperty("hadoop.log.dir") == null
        && System.getenv("HADOOP_LOG_DIR") != null)
      System.setProperty("hadoop.log.dir", System.getenv("HADOOP_LOG_DIR"));

    // wrap the command in a stdout/stderr capture
    TaskAttemptID taskid = TaskAttemptID.forName(job.get("mapred.task.id"));
    File stdout = TaskLog.getTaskLogFile(taskid, false, TaskLog.LogName.STDOUT);
    File stderr = TaskLog.getTaskLogFile(taskid, false, TaskLog.LogName.STDERR);
    long logLength = TaskLog.getTaskLogLength(job);
    command = TaskLog.captureOutAndError(null, command, stdout, stderr, logLength, false);
    stdout.getParentFile().mkdirs();
View Full Code Here
458
459
460
461
462
463
464
465
466
467
468
  /**
   * clean previous std error and outs
   */

  private void initStdOut(JobConf configuration) {
    TaskAttemptID taskId = TaskAttemptID.forName(configuration
            .get(MRJobConfig.TASK_ATTEMPT_ID));
    File stdOut = TaskLog.getTaskLogFile(taskId, false, TaskLog.LogName.STDOUT);
    File stdErr = TaskLog.getTaskLogFile(taskId, false, TaskLog.LogName.STDERR);
    // prepare folder
    if (!stdOut.getParentFile().exists()) {
View Full Code Here
472
473
474
475
476
477
478
479
480
481
482
      stdErr.deleteOnExit();
    }
  }

  private String readStdOut(JobConf conf) throws Exception {
    TaskAttemptID taskId = TaskAttemptID.forName(conf
            .get(MRJobConfig.TASK_ATTEMPT_ID));
    File stdOut = TaskLog.getTaskLogFile(taskId, false, TaskLog.LogName.STDOUT);

    return readFile(stdOut);
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
100
        String taskAttemptName = getTaskAttemptName(name);
        if (taskAttemptName == null) {
          LOG.info("Dir name [{0}] not task attempt", name);
          continue;
        }
        TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskAttemptName);
        if (taskAttemptID.getJobID().equals(jobContext.getJobID())) {
          if (commit) {
            if (taskComplete) {
              fileSystem.rename(path, new Path(shardPath, taskAttemptName + ".commit"));
              LOG.info("Committing [{0}] in path [{1}]", taskAttemptID, path);
            } else {
View Full Code Here
41
42
43
44
45
46
47
48
  private TableDescriptor _tableDescriptor;

  @Override
  public boolean needsTaskCommit(TaskAttemptContext context) throws IOException {
    int numReduceTasks = context.getNumReduceTasks();
    TaskAttemptID taskAttemptID = context.getTaskAttemptID();
    return taskAttemptID.isMap() && numReduceTasks != 0 ? false : true;
  }
View Full Code Here
148
149
150
151
152
153
154
155
156
157
158
    if (System.getProperty("hadoop.log.dir") == null
        && System.getenv("HADOOP_LOG_DIR") != null)
      System.setProperty("hadoop.log.dir", System.getenv("HADOOP_LOG_DIR"));

    // wrap the command in a stdout/stderr capture
    TaskAttemptID taskid = TaskAttemptID.forName(job.get("mapred.task.id"));
    File stdout = TaskLog.getTaskLogFile(taskid, false, TaskLog.LogName.STDOUT);
    File stderr = TaskLog.getTaskLogFile(taskid, false, TaskLog.LogName.STDERR);
    long logLength = TaskLog.getTaskLogLength(job);
    command = TaskLog.captureOutAndError(command, stdout, stderr, logLength);
    stdout.getParentFile().mkdirs();
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
    String executable = DistributedCache.getLocalCacheFiles(conf)[0].toString();
    FileUtil.chmod(executable, "a+x");
    cmd.add(executable);
    // wrap the command in a stdout/stderr capture
    TaskAttemptID taskid = TaskAttemptID.forName(conf.get("mapred.task.id"));
    File stdout = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDOUT);
    File stderr = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDERR);
    long logLength = TaskLog.getTaskLogLength(conf);
    cmd = TaskLog.captureOutAndError(cmd, stdout, stderr, logLength);
View Full Code Here
49
50
51
52
53
54
55
56
57
58
    return org.apache.hadoop.mapreduce.v2.api.records.Phase.valueOf(phase
        .name());
  }

  public static TaskAttemptId toYarn(TezTaskAttemptID taskAttemptId) {
    TaskAttemptID mrTaskAttemptId = IDConverter
        .toMRTaskAttemptId(taskAttemptId);
    TaskAttemptId mrv2TaskAttemptId = TypeConverter.toYarn(mrTaskAttemptId);
    return mrv2TaskAttemptId;
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.TaskAttemptID

  • com.digitalpebble.behemoth.uima.UIMAMapper
  • com.facebook.hiveio.output.CheckOutputSpecsTest
  • com.facebook.hiveio.output.FaultyCheckOutputSpecsTest
  • com.facebook.hiveio.output.HiveOutput
  • com.facebook.hiveio.output.PerThread
  • org.apache.avro.mapred.tether.TetheredProcess
  • org.apache.blur.mapred.AbstractOutputCommitter
  • org.apache.blur.mapreduce.lib.BlurOutputCommitter
  • org.apache.flink.hadoopcompatibility.mapred.HadoopOutputFormat
  • org.apache.giraph.hive.Helpers
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.