Examples of TezTaskAttemptID

org.apache.tez.dag.records.TezTaskAttemptID
TezTaskAttemptID represents the immutable and unique identifier for a task attempt. Each task attempt is one particular instance of a Tez Task identified by its TezTaskID. TezTaskAttemptID consists of 2 parts. First part is the {@link TezTaskID}, that this TaskAttemptID belongs to. Second part is the task attempt number.

Applications should never construct or parse TaskAttemptID strings , but rather use appropriate constructors or {@link Class#forName(String)}method. @see TezTaskID

Examples of org.apache.tez.dag.records.TezTaskAttemptID

    mapFinishTime = in.readLong();
    shuffleFinishTime = in.readLong();
    
    int numFailedDependencies = in.readInt();
    for (int i = 0 ; i < numFailedDependencies ; i++) {
      TezTaskAttemptID taskAttemptId = TezTaskAttemptID.readTezTaskAttemptID(in);
      failedTaskDependencies.add(taskAttemptId);
    }
    
  }

View Full Code Here

Examples of org.apache.tez.dag.records.TezTaskAttemptID

    return mrv2TaskAttemptId;
  }


  public static TezTaskAttemptID toTez(TaskAttemptId taskAttemptId) {
    TaskAttemptID mrTaskAttemptId = TypeConverter.fromYarn(taskAttemptId);
    TezTaskAttemptID tezTaskAttemptId = IDConverter
        .fromMRTaskAttemptId(mrTaskAttemptId);
    return tezTaskAttemptId;
  }

View Full Code Here

Examples of org.apache.tez.dag.records.TezTaskAttemptID

    String[] localDirs = new String[] { testDir.toString() };


    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
    TezVertexID vertexId = TezVertexID.getInstance(dagId, 1);
    TezTaskID taskId = TezTaskID.getInstance(vertexId, 1);
    TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 1);
    ProcessorDescriptor processorDescriptor = ProcessorDescriptor.create(processorClass)
        .setUserPayload(UserPayload.create(ByteBuffer.wrap(processorConf)));
    TaskSpec taskSpec = new TaskSpec(taskAttemptId, "dagName", "vertexName", -1, processorDescriptor,
        new ArrayList<InputSpec>(), new ArrayList<OutputSpec>(), null);

View Full Code Here

Examples of org.apache.tez.dag.records.TezTaskAttemptID

    // result of past getTaskAttemptTezEvents calls.
    List<TezEvent> fetchedList;
    TezTaskID taskId = getNewTaskID();
    scheduleTaskAttempt(taskId);
    sendTezEventsToTask(taskId, 2);
    TezTaskAttemptID attemptID = mockTask.getAttemptList().iterator().next()
        .getID();
    fetchedList = mockTask.getTaskAttemptTezEvents(attemptID, 0, 100);
    assertEquals(2, fetchedList.size());


    // Add events, make sure underlying list is the same, and no exceptions are

View Full Code Here

Examples of org.apache.tez.dag.records.TezTaskAttemptID

    String [] host1 = {"host1"};
    String [] host2 = {"host2"};


    String [] defaultRack = {"/default-rack"};


    TezTaskAttemptID taID11 = TezTaskAttemptID.getInstance(
      TezTaskID.getInstance(vertexID, 1), 1);
    TezTaskAttemptID taID21 = TezTaskAttemptID.getInstance(
      TezTaskID.getInstance(vertexID, 2), 1);
    TezTaskAttemptID taID31 = TezTaskAttemptID.getInstance(
      TezTaskID.getInstance(vertexID, 3), 1);
    TaskAttempt ta11 = mock(TaskAttempt.class);
    TaskAttempt ta21 = mock(TaskAttempt.class);
    TaskAttempt ta31 = mock(TaskAttempt.class);

View Full Code Here

Examples of org.apache.tez.dag.records.TezTaskAttemptID

    String [] host1 = {"host1"};
    String [] host2 = {"host2"};


    String [] defaultRack = {"/default-rack"};


    TezTaskAttemptID taID11 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID, 1), 1);
    TezTaskAttemptID taID21 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID, 2), 1);
    TezTaskAttemptID taID31 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID, 3), 1);
    TaskAttempt ta11 = mock(TaskAttempt.class);
    TaskAttempt ta21 = mock(TaskAttempt.class);
    TaskAttempt ta31 = mock(TaskAttempt.class);


    AMSchedulerEventTALaunchRequest lrTa11 = createLaunchRequestEvent(

View Full Code Here

Examples of org.apache.tez.dag.records.TezTaskAttemptID

    Priority priority1 = Priority.newInstance(1);


    TezVertexID vertexID1 = TezVertexID.getInstance(dagID, 1);


    //Vertex 1, Task 1, Attempt 1, host1
    TezTaskAttemptID taID11 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID1, 1), 1);
    TaskAttempt ta11 = mock(TaskAttempt.class);
    AMSchedulerEventTALaunchRequest lrEvent1 = createLaunchRequestEvent(taID11, ta11, resource1, host1, racks, priority1);


    //Vertex 1, Task 2, Attempt 1, host1
    TezTaskAttemptID taID12 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID1, 2), 1);
    TaskAttempt ta12 = mock(TaskAttempt.class);
    AMSchedulerEventTALaunchRequest lrEvent2 = createLaunchRequestEvent(taID12, ta12, resource1, host1, racks, priority1);


    //Vertex 1, Task 3, Attempt 1, host2
    TezTaskAttemptID taID13 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID1, 3), 1);
    TaskAttempt ta13 = mock(TaskAttempt.class);
    AMSchedulerEventTALaunchRequest lrEvent3 = createLaunchRequestEvent(taID13, ta13, resource1, host2, racks, priority1);


    //Vertex 1, Task 4, Attempt 1, host2
    TezTaskAttemptID taID14 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID1, 4), 1);
    TaskAttempt ta14 = mock(TaskAttempt.class);
    AMSchedulerEventTALaunchRequest lrEvent4 = createLaunchRequestEvent(taID14, ta14, resource1, host2, racks, priority1);


    taskSchedulerEventHandler.handleEvent(lrEvent1);
    taskSchedulerEventHandler.handleEvent(lrEvent2);

View Full Code Here

Examples of org.apache.tez.dag.records.TezTaskAttemptID

    /**
     * Schedule 2 tasks (1 with additional launch-cmd option and another in normal mode).
     * Container should not be reused in this case.
     */
    //Vertex 1, Task 1, Attempt 1, host1
    TezTaskAttemptID taID11 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID1, 1), 1);
    TaskAttempt ta11 = mock(TaskAttempt.class);
    AMSchedulerEventTALaunchRequest lrEvent1 =
        createLaunchRequestEvent(taID11, ta11, resource1, host1, racks,
          priority1, localResources, tsLaunchCmdOpts);


    //Vertex 1, Task 2, Attempt 1, host1
    TezTaskAttemptID taID12 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID1, 2), 1);
    TaskAttempt ta12 = mock(TaskAttempt.class);
    AMSchedulerEventTALaunchRequest lrEvent2 =
        createLaunchRequestEvent(taID12, ta12, resource1, host1, racks, priority1);


    taskSchedulerEventHandler.handleEvent(lrEvent1);
    taskSchedulerEventHandler.handleEvent(lrEvent2);


    Container container1 = createContainer(1, "host1", resource1, priority1);


    // One container allocated.
    drainNotifier.set(false);
    taskScheduler.onContainersAllocated(Collections.singletonList(container1));
    TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
    drainableAppCallback.drain();
    verify(taskSchedulerEventHandler).taskAllocated(eq(ta11), any(Object.class), eq(container1));


    // First task had profiling on. This container can not be reused further.
    taskSchedulerEventHandler.handleEvent(
        new AMSchedulerEventTAEnded(ta11, container1.getId(), TaskAttemptState.SUCCEEDED));
    drainableAppCallback.drain();
    verify(taskScheduler).deallocateTask(eq(ta11), eq(true));
    verify(taskSchedulerEventHandler, times(0)).taskAllocated(eq(ta12), any(Object.class),
        eq(container1));
    verify(rmClient, times(1)).releaseAssignedContainer(eq(container1.getId()));
    eventHandler.verifyInvocation(AMContainerEventStopRequest.class);
    eventHandler.reset();


    /**
     * Schedule 2 tasks (both having different task specific JVM option).
     * Container should not be reused.
     */
    //Vertex 1, Task 3, Attempt 1, host2
    tsLaunchCmdOpts = taskSpecificLaunchCmdOption.getTaskSpecificOption("", "v1", 3);
    TezTaskAttemptID taID13 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID1, 3), 1);
    TaskAttempt ta13 = mock(TaskAttempt.class);
    AMSchedulerEventTALaunchRequest lrEvent3 =
        createLaunchRequestEvent(taID13, ta13, resource1, host2, racks,
            priority1, localResources, tsLaunchCmdOpts);


    //Vertex 1, Task 4, Attempt 1, host2
    tsLaunchCmdOpts = taskSpecificLaunchCmdOption.getTaskSpecificOption("", "v1", 4);
    TezTaskAttemptID taID14 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID1, 4), 1);
    TaskAttempt ta14 = mock(TaskAttempt.class);
    AMSchedulerEventTALaunchRequest lrEvent4 =
        createLaunchRequestEvent(taID14, ta14, resource1, host2, racks,
            priority1, localResources, tsLaunchCmdOpts);


    Container container2 = createContainer(2, "host2", resource1, priority1);
    taskSchedulerEventHandler.handleEvent(lrEvent3);
    taskSchedulerEventHandler.handleEvent(lrEvent4);


    // Container started
    drainNotifier.set(false);
    taskScheduler.onContainersAllocated(Collections.singletonList(container2));
    TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
    drainableAppCallback.drain();
    verify(taskSchedulerEventHandler).taskAllocated(eq(ta13), any(Object.class), eq(container2));


    // Verify that the container can not be reused when profiling option is turned on
    // Even for 2 tasks having same profiling option can have container reusability.
    taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta13, container2.getId(), TaskAttemptState.SUCCEEDED));
    drainableAppCallback.drain();
    verify(taskScheduler).deallocateTask(eq(ta13), eq(true));
    verify(taskSchedulerEventHandler, times(0)).taskAllocated(eq(ta14), any(Object.class),
        eq(container2));
    verify(rmClient, times(1)).releaseAssignedContainer(eq(container2.getId()));
    eventHandler.verifyInvocation(AMContainerEventStopRequest.class);
    eventHandler.reset();


    /**
     * Schedule 2 tasks with same jvm profiling option.
     * Container should be reused.
     */
    tezConf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST, "v1[1,2,3,5,6]");
    tezConf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS, "dummyOpts");
    taskSpecificLaunchCmdOption =  new TaskSpecificLaunchCmdOption(tezConf);


    //Vertex 1, Task 5, Attempt 1, host3
    TezTaskAttemptID taID15 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID1, 3), 1);
    TaskAttempt ta15 = mock(TaskAttempt.class);
    AMSchedulerEventTALaunchRequest lrEvent5 =
        createLaunchRequestEvent(taID15, ta15, resource1, host3, racks,
            priority1, localResources,
            taskSpecificLaunchCmdOption.getTaskSpecificOption("", "v1", 5));


    //Vertex 1, Task 6, Attempt 1, host3
    tsLaunchCmdOpts = taskSpecificLaunchCmdOption.getTaskSpecificOption("", "v1", 4);
    TezTaskAttemptID taID16 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID1, 4), 1);
    TaskAttempt ta16 = mock(TaskAttempt.class);
    AMSchedulerEventTALaunchRequest lrEvent6 =
        createLaunchRequestEvent(taID16, ta16, resource1, host3, racks,
          priority1, localResources, taskSpecificLaunchCmdOption.getTaskSpecificOption("", "v1", 6));

View Full Code Here

Examples of org.apache.tez.dag.records.TezTaskAttemptID

    Priority priority = Priority.newInstance(3);


    TezVertexID vertexID = TezVertexID.getInstance(dagID, 1);


    //Vertex 1, Task 1, Attempt 1, no locality information.
    TezTaskAttemptID taID11 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID, 1), 1);
    TaskAttempt ta11 = mock(TaskAttempt.class);
    doReturn(vertexID).when(ta11).getVertexID();
    AMSchedulerEventTALaunchRequest lrEvent11 = createLaunchRequestEvent(
      taID11, ta11, resource1, emptyHosts, racks, priority);


    //Vertex1, Task2, Attempt 1,  no locality information.
    TezTaskAttemptID taID12 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID, 2), 1);
    TaskAttempt ta12 = mock(TaskAttempt.class);
    doReturn(vertexID).when(ta12).getVertexID();
    AMSchedulerEventTALaunchRequest lrEvent12 = createLaunchRequestEvent(
      taID12, ta12, resource1, emptyHosts, racks, priority);

View Full Code Here

Examples of org.apache.tez.dag.records.TezTaskAttemptID


    TezVertexID vertexID1 = TezVertexID.getInstance(dagID, 1);
    TezVertexID vertexID2 = TezVertexID.getInstance(dagID, 2);


    //Vertex 1, Task 1, Attempt 1, host1
    TezTaskAttemptID taID11 = TezTaskAttemptID.getInstance(
      TezTaskID.getInstance(vertexID1, 1), 1);
    TaskAttempt ta11 = mock(TaskAttempt.class);
    doReturn(vertexID1).when(ta11).getVertexID();
    AMSchedulerEventTALaunchRequest lrEvent11 = createLaunchRequestEvent(
      taID11, ta11, resource1, host1, racks, priority1);


    //Vertex2, Task1, Attempt 1, host1
    TezTaskAttemptID taID21 = TezTaskAttemptID.getInstance(
      TezTaskID.getInstance(vertexID2, 1), 1);
    TaskAttempt ta21 = mock(TaskAttempt.class);
    doReturn(vertexID2).when(ta21).getVertexID();
    AMSchedulerEventTALaunchRequest lrEvent21 = createLaunchRequestEvent(
      taID21, ta21, resource1, host1, racks, priority2);

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.