Examples of TezTaskOutput

org.apache.tez.runtime.library.common.task.local.output.TezTaskOutput
Manipulate the working area for the transient store for maps and reduces. This class is used by map and reduce tasks to identify the directories that they need to write to/read from for intermediate files. The callers of these methods are from child space and see mapreduce.cluster.local.dir as taskTracker/jobCache/jobId/attemptId This class should not be used from TaskTracker space.

Examples of org.apache.tez.runtime.library.common.task.local.output.TezTaskOutput

    task.initialize();
    task.run();
    task.close();
    
    InputContext inputContext = task.getInputContexts().iterator().next();
    TezTaskOutput mapOutputs = new TezLocalTaskOutputFiles(jobConf, inputContext.getUniqueIdentifier());
    
    
    // TODO NEWTEZ FIXME OutputCommitter verification
//    MRTask mrTask = (MRTask)t.getProcessor();
//    Assert.assertEquals(TezNullOutputCommitter.class.getName(), mrTask
//        .getCommitter().getClass().getName());
//    t.close();


    Path mapOutputFile = mapOutputs.getInputFile(new InputAttemptIdentifier(0, 0));
    LOG.info("mapOutputFile = " + mapOutputFile);
    IFile.Reader reader =
        new IFile.Reader(localFs, mapOutputFile, null, null, null, false, 0, -1);
    LongWritable key = new LongWritable();
    Text value = new Text();

View Full Code Here

Examples of org.apache.tez.runtime.library.common.task.local.output.TezTaskOutput

  @Override
  public List<Event> close() throws IOException {
    LOG.debug("Closing LocalOnFileSorterOutput");
    super.close();


    TezTaskOutput mapOutputFile = sorter.getMapOutput();
    FileSystem localFs = FileSystem.getLocal(conf);


    Path src = mapOutputFile.getOutputFile();
    Path dst =
        mapOutputFile.getInputFileForWrite(
            getContext().getTaskIndex(),
            localFs.getFileStatus(src).getLen());


    LOG.info("Renaming src = " + src + ", dst = " + dst);
    if (LOG.isDebugEnabled()) {

View Full Code Here

Examples of org.apache.tez.runtime.library.common.task.local.output.TezTaskOutput

    Class<?> clazz = conf.getClass(Constants.TEZ_RUNTIME_TASK_OUTPUT_MANAGER,
        TezTaskOutputFiles.class);
    try {
      Constructor<?> ctor = clazz.getConstructor(Configuration.class, String.class);
      ctor.setAccessible(true);
      TezTaskOutput instance = (TezTaskOutput) ctor.newInstance(conf, outputContext.getUniqueIdentifier());
      return instance;
    } catch (Exception e) {
      throw new TezUncheckedException(
          "Unable to instantiate configured TezOutputFileManager: "
              + conf.get(Constants.TEZ_RUNTIME_TASK_OUTPUT_MANAGER,

View Full Code Here

Examples of org.apache.tez.runtime.library.common.task.local.output.TezTaskOutput

    assertEquals(SHUFFLE_PORT, eventProto.getPort());
    assertEquals(uniqueId, eventProto.getPathComponent());


    // Verify the data
    // Verify the actual data
    TezTaskOutput taskOutput = new TezTaskOutputFiles(conf, uniqueId);
    Path outputFilePath = null;
    Path spillFilePath = null;
    try {
      outputFilePath = taskOutput.getOutputFile();
    } catch (DiskErrorException e) {
      if (numRecordsWritten > 0) {
        fail();
      } else {
        // Record checking not required.
        return;
      }
    }
    try {
      spillFilePath = taskOutput.getOutputIndexFile();
    } catch (DiskErrorException e) {
      if (numRecordsWritten > 0) {
        fail();
      } else {
        // Record checking not required.

View Full Code Here

Examples of org.apache.tez.runtime.library.common.task.local.output.TezTaskOutput

      assertFalse(eventProto.hasPort());
      assertFalse(eventProto.hasPathComponent());
    }


    // Verify the actual data
    TezTaskOutput taskOutput = new TezTaskOutputFiles(conf, uniqueId);
    Path outputFilePath = null;
    Path spillFilePath = null;
    try {
      outputFilePath = taskOutput.getOutputFile();
    } catch (DiskErrorException e) {
      if (numRecordsWritten > 0) {
        fail();
      } else {
        // Record checking not required.
        return;
      }
    }
    try {
      spillFilePath = taskOutput.getOutputIndexFile();
    } catch (DiskErrorException e) {
      if (numRecordsWritten > 0) {
        fail();
      } else {
        // Record checking not required.

View Full Code Here

Examples of org.apache.tez.runtime.library.common.task.local.output.TezTaskOutput

    task.initialize();
    task.run();
    task.close();
    
    TezInputContext inputContext = task.getInputContexts().iterator().next();
    TezTaskOutput mapOutputs = new TezLocalTaskOutputFiles(jobConf, inputContext.getUniqueIdentifier());
    
    
    // TODO NEWTEZ FIXME OutputCommitter verification
//    MRTask mrTask = (MRTask)t.getProcessor();
//    Assert.assertEquals(TezNullOutputCommitter.class.getName(), mrTask
//        .getCommitter().getClass().getName());
//    t.close();


    Path mapOutputFile = mapOutputs.getInputFile(new InputAttemptIdentifier(0, 0));
    LOG.info("mapOutputFile = " + mapOutputFile);
    IFile.Reader reader =
        new IFile.Reader(localFs, mapOutputFile, null, null, false, 0, -1);
    LongWritable key = new LongWritable();
    Text value = new Text();

View Full Code Here

Examples of org.apache.tez.runtime.library.common.task.local.output.TezTaskOutput

  @Override
  public List<Event> close() throws IOException {
    LOG.debug("Closing LocalOnFileSorterOutput");
    super.close();


    TezTaskOutput mapOutputFile = sorter.getMapOutput();
    FileSystem localFs = FileSystem.getLocal(conf);


    Path src = mapOutputFile.getOutputFile();
    Path dst =
        mapOutputFile.getInputFileForWrite(
            outputContext.getTaskIndex(),
            localFs.getFileStatus(src).getLen());


    LOG.info("Renaming src = " + src + ", dst = " + dst);
    if (LOG.isDebugEnabled()) {

View Full Code Here

Examples of org.apache.tez.runtime.library.common.task.local.output.TezTaskOutput

    task.initialize();
    task.run();
    task.close();
    
    TezInputContext inputContext = task.getInputContexts().iterator().next();
    TezTaskOutput mapOutputs = new TezLocalTaskOutputFiles(jobConf, inputContext.getUniqueIdentifier());
    
    
    // TODO NEWTEZ FIXME OutputCommitter verification
//    MRTask mrTask = (MRTask)t.getProcessor();
//    Assert.assertEquals(TezNullOutputCommitter.class.getName(), mrTask
//        .getCommitter().getClass().getName());
//    t.close();


    Path mapOutputFile = mapOutputs.getInputFile(new InputAttemptIdentifier(0, 0));
    LOG.info("mapOutputFile = " + mapOutputFile);
    IFile.Reader reader =
        new IFile.Reader(localFs, mapOutputFile, null, null, null, false, 0, -1);
    LongWritable key = new LongWritable();
    Text value = new Text();

View Full Code Here

Examples of org.apache.tez.runtime.library.common.task.local.output.TezTaskOutput

  @Override
  public List<Event> close() throws IOException {
    LOG.debug("Closing LocalOnFileSorterOutput");
    super.close();


    TezTaskOutput mapOutputFile = sorter.getMapOutput();
    FileSystem localFs = FileSystem.getLocal(conf);


    Path src = mapOutputFile.getOutputFile();
    Path dst =
        mapOutputFile.getInputFileForWrite(
            outputContext.getTaskIndex(),
            localFs.getFileStatus(src).getLen());


    LOG.info("Renaming src = " + src + ", dst = " + dst);
    if (LOG.isDebugEnabled()) {

View Full Code Here

Examples of org.apache.tez.runtime.library.common.task.local.output.TezTaskOutput

    Class<?> clazz = conf.getClass(Constants.TEZ_RUNTIME_TASK_OUTPUT_MANAGER,
        TezTaskOutputFiles.class);
    try {
      Constructor<?> ctor = clazz.getConstructor(Configuration.class, String.class);
      ctor.setAccessible(true);
      TezTaskOutput instance = (TezTaskOutput) ctor.newInstance(conf, outputContext.getUniqueIdentifier());
      return instance;
    } catch (Exception e) {
      throw new TezUncheckedException(
          "Unable to instantiate configured TezOutputFileManager: "
              + conf.get(Constants.TEZ_RUNTIME_TASK_OUTPUT_MANAGER,

View Full Code Here

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.