Examples of TezTaskOutputFiles

org.apache.tez.runtime.library.common.task.local.output.TezTaskOutputFiles
Manipulate the working area for the transient store for maps and reduces. This class is used by map and reduce tasks to identify the directories that they need to write to/read from for intermediate files. The callers of these methods are from child space and see mapreduce.cluster.local.dir as taskTracker/jobCache/jobId/attemptId This class should not be used from TaskTracker space.

Examples of org.apache.tez.runtime.library.common.task.local.output.TezTaskOutputFiles

    this.combiner = combiner;


    this.reduceCombineInputCounter = reduceCombineInputCounter;
    this.spilledRecordsCounter = spilledRecordsCounter;
    this.mergedMapOutputsCounter = mergedMapOutputsCounter;
    this.mapOutputFile = new TezTaskOutputFiles(conf, inputContext.getUniqueIdentifier());
    
    this.localFS = localFS;
    this.rfs = ((LocalFileSystem)localFS).getRaw();
    
    this.numDiskToDiskMerges = inputContext.getCounters().findCounter(TaskCounter.NUM_DISK_TO_DISK_MERGES);

View Full Code Here

Examples of org.apache.tez.runtime.library.common.task.local.output.TezTaskOutputFiles

      long maxTaskAvailableMemory, long memoryAvailable) {
    this.conf = conf;    
    this.maxAvailableTaskMemory = maxTaskAvailableMemory;
    this.initialMemoryAvailable = memoryAvailable;
    
    this.fileNameAllocator = new TezTaskOutputFiles(conf,
        uniqueIdentifier);
    this.localDirAllocator = new LocalDirAllocator(TezRuntimeFrameworkConfigs.LOCAL_DIRS);
    
    // Setup configuration
    final float maxInMemCopyUse = conf.getFloat(

View Full Code Here

Examples of org.apache.tez.runtime.library.common.task.local.output.TezTaskOutputFiles

    assertEquals(SHUFFLE_PORT, eventProto.getPort());
    assertEquals(uniqueId, eventProto.getPathComponent());


    // Verify the data
    // Verify the actual data
    TezTaskOutput taskOutput = new TezTaskOutputFiles(conf, uniqueId);
    Path outputFilePath = null;
    Path spillFilePath = null;
    try {
      outputFilePath = taskOutput.getOutputFile();
    } catch (DiskErrorException e) {
      if (numRecordsWritten > 0) {
        fail();
      } else {
        // Record checking not required.
        return;
      }
    }
    try {
      spillFilePath = taskOutput.getOutputIndexFile();
    } catch (DiskErrorException e) {
      if (numRecordsWritten > 0) {
        fail();
      } else {
        // Record checking not required.

View Full Code Here

Examples of org.apache.tez.runtime.library.common.task.local.output.TezTaskOutputFiles

      assertFalse(eventProto.hasPort());
      assertFalse(eventProto.hasPathComponent());
    }


    // Verify the actual data
    TezTaskOutput taskOutput = new TezTaskOutputFiles(conf, uniqueId);
    Path outputFilePath = null;
    Path spillFilePath = null;
    try {
      outputFilePath = taskOutput.getOutputFile();
    } catch (DiskErrorException e) {
      if (numRecordsWritten > 0) {
        fail();
      } else {
        // Record checking not required.
        return;
      }
    }
    try {
      spillFilePath = taskOutput.getOutputIndexFile();
    } catch (DiskErrorException e) {
      if (numRecordsWritten > 0) {
        fail();
      } else {
        // Record checking not required.

View Full Code Here

Examples of org.apache.tez.runtime.library.common.task.local.output.TezTaskOutputFiles

    this.combiner = combiner;


    this.reduceCombineInputCounter = reduceCombineInputCounter;
    this.spilledRecordsCounter = spilledRecordsCounter;
    this.mergedMapOutputsCounter = mergedMapOutputsCounter;
    this.mapOutputFile = new TezTaskOutputFiles(conf, inputContext.getUniqueIdentifier());
    
    this.localFS = localFS;
    this.rfs = ((LocalFileSystem)localFS).getRaw();


  }

View Full Code Here

Examples of org.apache.tez.runtime.library.common.task.local.output.TezTaskOutputFiles


  @Private
  void configureAndStart() {
    Preconditions.checkState(initialMemoryAvailable != -1,
        "Initial memory must be configured before starting");
    this.fileNameAllocator = new TezTaskOutputFiles(conf,
        uniqueIdentifier);
    this.localDirAllocator = new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);


    // Setup configuration
    final float maxInMemCopyUse = conf.getFloat(

View Full Code Here

Examples of org.apache.tez.runtime.library.common.task.local.output.TezTaskOutputFiles

  private volatile long usedMemory = 0;


  public BroadcastInputManager(String uniqueIdentifier, Configuration conf) {
    this.conf = conf;


    this.fileNameAllocator = new TezTaskOutputFiles(conf,
        uniqueIdentifier);
    this.localDirAllocator = new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);


    // Setup configuration
    final float maxInMemCopyUse = conf.getFloat(

View Full Code Here

Examples of org.apache.tez.runtime.library.common.task.local.output.TezTaskOutputFiles

    this.combiner = combiner;


    this.reduceCombineInputCounter = reduceCombineInputCounter;
    this.spilledRecordsCounter = spilledRecordsCounter;
    this.mergedMapOutputsCounter = mergedMapOutputsCounter;
    this.mapOutputFile = new TezTaskOutputFiles(conf, inputContext.getUniqueIdentifier());
    
    this.localFS = localFS;
    this.rfs = ((LocalFileSystem)localFS).getRaw();


    if (ConfigUtils.isIntermediateInputCompressed(conf)) {

View Full Code Here

Examples of org.apache.tez.runtime.library.common.task.local.output.TezTaskOutputFiles

    this.combiner = combiner;


    this.reduceCombineInputCounter = reduceCombineInputCounter;
    this.spilledRecordsCounter = spilledRecordsCounter;
    this.mergedMapOutputsCounter = mergedMapOutputsCounter;
    this.mapOutputFile = new TezTaskOutputFiles(conf, inputContext.getUniqueIdentifier());
    
    this.localFS = localFS;
    this.rfs = ((LocalFileSystem)localFS).getRaw();
    
    this.numDiskToDiskMerges = inputContext.getCounters().findCounter(TaskCounter.NUM_DISK_TO_DISK_MERGES);

View Full Code Here

Examples of org.apache.tez.runtime.library.common.task.local.output.TezTaskOutputFiles

      long maxTaskAvailableMemory, long memoryAvailable) {
    this.conf = conf;    
    this.maxAvailableTaskMemory = maxTaskAvailableMemory;
    this.initialMemoryAvailable = memoryAvailable;
    
    this.fileNameAllocator = new TezTaskOutputFiles(conf,
        uniqueIdentifier);
    this.localDirAllocator = new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);
    
    // Setup configuration
    final float maxInMemCopyUse = conf.getFloat(

View Full Code Here

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.