Examples of TezVertexID

org.apache.tez.dag.records.TezVertexID
TezVertexID represents the immutable and unique identifier for a Vertex in a Tez DAG. Each TezVertexID encompasses multiple Tez Tasks. TezVertezID consists of 2 parts. The first part is the {@link TezDAGID}, that is the Tez DAG that this vertex belongs to. The second part is the vertex number. @see TezDAGID @see TezTaskID

Examples of org.apache.tez.dag.records.TezVertexID

        taskAttemptNumber);
  }
  
  public static org.apache.hadoop.mapred.TaskAttemptID 
    createMockTaskAttemptIDFromTezTaskAttemptId(TezTaskAttemptID tezTaId, boolean isMap) {
    TezVertexID vId = tezTaId.getTaskID().getVertexID();
    ApplicationId appId = vId.getDAGId().getApplicationId();
    return new org.apache.hadoop.mapred.TaskAttemptID(
        new org.apache.hadoop.mapred.TaskID(String.valueOf(appId.getClusterTimestamp())
            + String.valueOf(vId.getId()), appId.getId(),
            isMap ? TaskType.MAP : TaskType.REDUCE, tezTaId.getTaskID().getId()),
        tezTaId.getId());
  }

View Full Code Here

Examples of org.apache.tez.dag.records.TezVertexID

        tezTaId.getId());
  }
  
  public static org.apache.hadoop.mapred.TaskID 
    createMockTaskAttemptIDFromTezTaskId(TezTaskID tezTaId, boolean isMap) {
    TezVertexID vId = tezTaId.getVertexID();
    ApplicationId appId = vId.getDAGId().getApplicationId();
    return new org.apache.hadoop.mapred.TaskID(String.valueOf(appId.getClusterTimestamp())
            + String.valueOf(vId.getId()), appId.getId(),
            isMap ? TaskType.MAP : TaskType.REDUCE, tezTaId.getId());
  }

View Full Code Here

Examples of org.apache.tez.dag.records.TezVertexID

    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    Path testDir = new Path(workDir, UUID.randomUUID().toString());
    String[] localDirs = new String[] { testDir.toString() };


    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
    TezVertexID vertexId = TezVertexID.getInstance(dagId, 1);
    TezTaskID taskId = TezTaskID.getInstance(vertexId, 1);
    TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 1);
    ProcessorDescriptor processorDescriptor = ProcessorDescriptor.create(processorClass)
        .setUserPayload(UserPayload.create(ByteBuffer.wrap(processorConf)));
    TaskSpec taskSpec = new TaskSpec(taskAttemptId, "dagName", "vertexName", -1, processorDescriptor,

View Full Code Here

Examples of org.apache.tez.dag.records.TezVertexID

    RackResolver.init(conf);
    TaskSchedulerAppCallback mockApp = mock(TaskSchedulerAppCallback.class);


    CapturingEventHandler eventHandler = new CapturingEventHandler();
    TezDAGID dagID = TezDAGID.getInstance("0", 0, 0);
    TezVertexID vertexID = TezVertexID.getInstance(dagID, 1);


    AMRMClient<CookieContainerRequest> rmClientCore = new AMRMClientForTest();
    TezAMRMClientAsync<CookieContainerRequest> rmClient =
      spy(new AMRMClientAsyncForTest(rmClientCore, 100));
    String appUrl = "url";

View Full Code Here

Examples of org.apache.tez.dag.records.TezVertexID

    RackResolver.init(conf);
    TaskSchedulerAppCallback mockApp = mock(TaskSchedulerAppCallback.class);


    CapturingEventHandler eventHandler = new CapturingEventHandler();
    TezDAGID dagID = TezDAGID.getInstance("0", 0, 0);
    TezVertexID vertexID = TezVertexID.getInstance(dagID, 1);


    AMRMClient<CookieContainerRequest> rmClientCore = new AMRMClientForTest();
    TezAMRMClientAsync<CookieContainerRequest> rmClient =
      spy(new AMRMClientAsyncForTest(rmClientCore, 100));
    String appUrl = "url";

View Full Code Here

Examples of org.apache.tez.dag.records.TezVertexID

    String[] host2 = {"host2"};


    String []racks = {"/default-rack"};
    Priority priority1 = Priority.newInstance(1);


    TezVertexID vertexID1 = TezVertexID.getInstance(dagID, 1);


    //Vertex 1, Task 1, Attempt 1, host1
    TezTaskAttemptID taID11 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID1, 1), 1);
    TaskAttempt ta11 = mock(TaskAttempt.class);
    AMSchedulerEventTALaunchRequest lrEvent1 = createLaunchRequestEvent(taID11, ta11, resource1, host1, racks, priority1);

View Full Code Here

Examples of org.apache.tez.dag.records.TezVertexID

    String[] host3 = {"host3"};


    String []racks = {"/default-rack"};
    Priority priority1 = Priority.newInstance(1);


    TezVertexID vertexID1 = TezVertexID.getInstance(dagID, 1);
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    String tsLaunchCmdOpts = taskSpecificLaunchCmdOption.getTaskSpecificOption("", "v1", 1);


    /**
     * Schedule 2 tasks (1 with additional launch-cmd option and another in normal mode).

View Full Code Here

Examples of org.apache.tez.dag.records.TezVertexID

    String [] emptyHosts = new String[0];
    String [] racks = { "default-rack" };


    Priority priority = Priority.newInstance(3);


    TezVertexID vertexID = TezVertexID.getInstance(dagID, 1);


    //Vertex 1, Task 1, Attempt 1, no locality information.
    TezTaskAttemptID taID11 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID, 1), 1);
    TaskAttempt ta11 = mock(TaskAttempt.class);
    doReturn(vertexID).when(ta11).getVertexID();

View Full Code Here

Examples of org.apache.tez.dag.records.TezVertexID


    String []racks = {"/default-rack"};
    Priority priority1 = Priority.newInstance(3);
    Priority priority2 = Priority.newInstance(4);


    TezVertexID vertexID1 = TezVertexID.getInstance(dagID, 1);
    TezVertexID vertexID2 = TezVertexID.getInstance(dagID, 2);


    //Vertex 1, Task 1, Attempt 1, host1
    TezTaskAttemptID taID11 = TezTaskAttemptID.getInstance(
      TezTaskID.getInstance(vertexID1, 1), 1);
    TaskAttempt ta11 = mock(TaskAttempt.class);

View Full Code Here

Examples of org.apache.tez.dag.records.TezVertexID

    AMContainerEventAssignTA assignEvent = null;
    
    Map<String, LocalResource> dag1LRs = Maps.newHashMap();
    dag1LRs.put(rsrc1, lr1);


    TezVertexID vertexID11 = TezVertexID.getInstance(dagID1, 1);
    
    //Vertex 1, Task 1, Attempt 1, host1, lr1
    TezTaskAttemptID taID111 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID11, 1), 1);
    TaskAttempt ta111 = mock(TaskAttempt.class);
    AMSchedulerEventTALaunchRequest lrEvent11 = createLaunchRequestEvent(taID111, ta111, resource1, host1, racks, priority1, dag1LRs);


    //Vertex 1, Task 2, Attempt 1, host1, lr1
    TezTaskAttemptID taID112 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID11, 2), 1);
    TaskAttempt ta112 = mock(TaskAttempt.class);
    AMSchedulerEventTALaunchRequest lrEvent12 = createLaunchRequestEvent(taID112, ta112, resource1, host1, racks, priority1, dag1LRs);


    taskSchedulerEventHandler.handleEvent(lrEvent11);
    taskSchedulerEventHandler.handleEvent(lrEvent12);


    Container container1 = createContainer(1, "host1", resource1, priority1);


    // One container allocated.
    drainNotifier.set(false);
    taskScheduler.onContainersAllocated(Collections.singletonList(container1));
    TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
    drainableAppCallback.drain();
    verify(taskSchedulerEventHandler).taskAllocated(eq(ta111), any(Object.class), eq(container1));
    assignEvent = (AMContainerEventAssignTA) eventHandler.verifyInvocation(AMContainerEventAssignTA.class);
    assertEquals(1, assignEvent.getRemoteTaskLocalResources().size());
    
    // Task assigned to container completed successfully. Container should be re-used.
    taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta111, container1.getId(), TaskAttemptState.SUCCEEDED));
    drainableAppCallback.drain();
    verify(taskScheduler).deallocateTask(eq(ta111), eq(true));
    verify(taskSchedulerEventHandler).taskAllocated(eq(ta112), any(Object.class), eq(container1));
    verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
    eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
    assignEvent = (AMContainerEventAssignTA) eventHandler.verifyInvocation(AMContainerEventAssignTA.class);
    assertEquals(1, assignEvent.getRemoteTaskLocalResources().size());
    eventHandler.reset();


    // Task assigned to container completed successfully.
    // Verify reuse across hosts.
    taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta112, container1.getId(), TaskAttemptState.SUCCEEDED));
    drainableAppCallback.drain();
    verify(taskScheduler).deallocateTask(eq(ta112), eq(true));
    verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
    eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
    eventHandler.reset();


    // Setup DAG2 with additional resources. Make sure the container, even without all resources, is reused.
    TezDAGID dagID2 = TezDAGID.getInstance("0", 2, 0);
    dagIDAnswer.setDAGID(dagID2);
    
    Map<String, LocalResource> dag2LRs = Maps.newHashMap();
    dag2LRs.put(rsrc2, lr2);
    dag2LRs.put(rsrc3, lr3);
    
    TezVertexID vertexID21 = TezVertexID.getInstance(dagID2, 1);
    
    //Vertex 2, Task 1, Attempt 1, host1, lr2
    TezTaskAttemptID taID211 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID21, 1), 1);
    TaskAttempt ta211 = mock(TaskAttempt.class);
    AMSchedulerEventTALaunchRequest lrEvent21 = createLaunchRequestEvent(taID211, ta211, resource1, host1, racks, priority1, dag2LRs);

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.