Examples of TezTaskAttemptID


Examples of org.apache.tez.dag.records.TezTaskAttemptID

    return desc;
  }

  private TezTaskAttemptID createTaskAttemptID(TezVertexID vertexId, int taskIndex) {
    TezTaskID taskId = TezTaskID.getInstance(vertexId, taskIndex);
    TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, taskIndex);
    return taskAttemptId;
  }
View Full Code Here

Examples of org.apache.tez.dag.records.TezTaskAttemptID

      }

      TezHeartbeatResponse response = new TezHeartbeatResponse();
      response.setLastRequestId(requestId);

      TezTaskAttemptID taskAttemptID = request.getCurrentTaskAttemptID();
      if (taskAttemptID != null) {
        ContainerId containerIdFromMap = attemptToInfoMap.get(taskAttemptID);
        if(containerIdFromMap == null || !containerIdFromMap.equals(containerId)) {
          throw new TezException("Attempt " + taskAttemptID
            + " is not recognized for heartbeat");
        }

        if(containerInfo.lastRequestId+1 != requestId) {
          throw new TezException("Container " + containerId
              + " has invalid request id. Expected: "
              + containerInfo.lastRequestId+1
              + " and actual: " + requestId);
        }

        List<TezEvent> inEvents = request.getEvents();
        if (LOG.isDebugEnabled()) {
          LOG.debug("Ping from " + taskAttemptID.toString() +
              " events: " + (inEvents != null? inEvents.size() : -1));
        }
        if(inEvents!=null && !inEvents.isEmpty()) {
          TezVertexID vertexId = taskAttemptID.getTaskID().getVertexID();
          context.getEventHandler().handle(
              new VertexEventRouteEvent(vertexId, inEvents));
        }
        taskHeartbeatHandler.pinged(taskAttemptID);
        List<TezEvent> outEvents = context
            .getCurrentDAG()
            .getVertex(taskAttemptID.getTaskID().getVertexID())
            .getTask(taskAttemptID.getTaskID())
            .getTaskAttemptTezEvents(taskAttemptID, request.getStartIndex(),
                request.getMaxEvents());
        response.setEvents(outEvents);
      }
      containerInfo.lastRequestId = requestId;
View Full Code Here

Examples of org.apache.tez.dag.records.TezTaskAttemptID

    String taskVertexName = "currentVertex";
    String destinationVertexName = "destinationVertex";
    TezDAGID dagID = TezDAGID.getInstance("2000", 1, 1);
    TezVertexID vertexID = TezVertexID.getInstance(dagID, 1);
    TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
    TezTaskAttemptID taskAttemptID = TezTaskAttemptID.getInstance(taskID, 1);
    TezCounters counters = new TezCounters();
    UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
    RuntimeTask runtimeTask = mock(RuntimeTask.class);
   
    int shufflePort = 2112;
View Full Code Here

Examples of org.apache.tez.dag.records.TezTaskAttemptID

    return builder.build();
  }

  static EventMetaData convertEventMetaDataFromProto(
      RecoveryProtos.EventMetaDataProto proto) {
    TezTaskAttemptID attemptID = null;
    if (proto.hasTaskAttemptId()) {
      attemptID = TezTaskAttemptID.fromString(proto.getTaskAttemptId());
    }
    return new EventMetaData(
        EventMetaData.EventProducerConsumerType.values()[proto.getProducerConsumerType()],
View Full Code Here

Examples of org.apache.tez.dag.records.TezTaskAttemptID

    Map<String, ByteBuffer> serviceConsumerMetadata = new HashMap<String, ByteBuffer>();
    Multimap<String, String> startedInputsMap = HashMultimap.create();
    TezUmbilical umbilical = mock(TezUmbilical.class);
    TezConfiguration tezConf = new TezConfiguration();

    TezTaskAttemptID taId1 = createTaskAttemptID(vertexId, 1);
    TaskSpec task1 = createTaskSpec(taId1, "vertex1");

    TezTaskAttemptID taId2 = createTaskAttemptID(vertexId, 2);
    TaskSpec task2 = createTaskSpec(taId2, "vertex1");

    LogicalIOProcessorRuntimeTask lio1 = new LogicalIOProcessorRuntimeTask(task1, 0, tezConf,
        umbilical, serviceConsumerMetadata, startedInputsMap);
View Full Code Here

Examples of org.apache.tez.dag.records.TezTaskAttemptID

    return desc;
  }

  private TezTaskAttemptID createTaskAttemptID(TezVertexID vertexId, int taskIndex) {
    TezTaskID taskId = TezTaskID.getInstance(vertexId, taskIndex);
    TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, taskIndex);
    return taskAttemptId;
  }
View Full Code Here

Examples of org.apache.tez.dag.records.TezTaskAttemptID

    return builder.build();
  }

  static EventMetaData convertEventMetaDataFromProto(
      RecoveryProtos.EventMetaDataProto proto) {
    TezTaskAttemptID attemptID = null;
    if (proto.hasTaskAttemptId()) {
      attemptID = TezTaskAttemptID.fromString(proto.getTaskAttemptId());
    }
    return new EventMetaData(
        EventMetaData.EventProducerConsumerType.values()[proto.getProducerConsumerType()],
View Full Code Here

Examples of org.apache.tez.dag.records.TezTaskAttemptID

      Collection<TezEvent> outOfBandEvents)
      throws TezException, IOException {
    TezEvent updateEvent = null;
    int eventCounter = 0;
    int eventsRange = 0;
    TezTaskAttemptID taskAttemptID = null;
    List<TezEvent> events = new ArrayList<TezEvent>();
    try {
      taskLock.readLock().lock();
      if (currentTask != null) {
        eventsToSend.drainTo(events);
        taskAttemptID = currentTaskAttemptID;
        eventCounter = currentTask.getEventCounter();
        eventsRange = maxEventsToGet;
        if (!currentTask.isTaskDone() && !currentTask.hadFatalError()) {
          updateEvent = new TezEvent(new TaskStatusUpdateEvent(
              currentTask.getCounters(), currentTask.getProgress()),
                new EventMetaData(EventProducerConsumerType.SYSTEM,
                    currentTask.getVertexName(), "", taskAttemptID));
          events.add(updateEvent);
        } else if (outOfBandEvents == null && events.isEmpty()) {
          LOG.info("Setting TaskAttemptID to null as the task has already"
            + " completed. Caused by race-condition between the normal"
            + " heartbeat and out-of-band heartbeats");
          taskAttemptID = null;
        } else {
          if (outOfBandEvents != null && !outOfBandEvents.isEmpty()) {
            events.addAll(outOfBandEvents);
          }
        }
      }
    } finally {
      taskLock.readLock().unlock();
    }

    long reqId = requestCounter.incrementAndGet();
    TezHeartbeatRequest request = new TezHeartbeatRequest(reqId, events,
        containerIdStr, taskAttemptID, eventCounter, eventsRange);
    if (LOG.isDebugEnabled()) {
      LOG.debug("Sending heartbeat to AM"
          + ", request=" + request.toString());
    }
    TezHeartbeatResponse response = umbilical.heartbeat(request);
    if (LOG.isDebugEnabled()) {
      LOG.debug("Received heartbeat response from AM"
          + ", response=" + response);
    }
    if(response.shouldDie()) {
      LOG.info("Received should die response from AM");
      return false;
    }
    if (response.getLastRequestId() != reqId) {
      throw new TezException("AM and Task out of sync"
          + ", responseReqId=" + response.getLastRequestId()
          + ", expectedReqId=" + reqId);
    }
    try {
      taskLock.readLock().lock();
      if (taskAttemptID == null
          || !taskAttemptID.equals(currentTaskAttemptID)) {
        if (response.getEvents() != null
            && !response.getEvents().isEmpty()) {
          LOG.warn("No current assigned task, ignoring all events in"
              + " heartbeat response, eventCount="
              + response.getEvents().size());
View Full Code Here

Examples of org.apache.tez.dag.records.TezTaskAttemptID

      }

      TezHeartbeatResponse response = new TezHeartbeatResponse();
      response.setLastRequestId(requestId);

      TezTaskAttemptID taskAttemptID = request.getCurrentTaskAttemptID();
      if (taskAttemptID != null) {
        ContainerId containerIdFromMap = attemptToInfoMap.get(taskAttemptID);
        if(containerIdFromMap == null || !containerIdFromMap.equals(containerId)) {
          throw new TezException("Attempt " + taskAttemptID
            + " is not recognized for heartbeat");
        }

        if(containerInfo.lastRequestId+1 != requestId) {
          throw new TezException("Container " + containerId
              + " has invalid request id. Expected: "
              + containerInfo.lastRequestId+1
              + " and actual: " + requestId);
        }

        List<TezEvent> inEvents = request.getEvents();
        if (LOG.isDebugEnabled()) {
          LOG.debug("Ping from " + taskAttemptID.toString() +
              " events: " + (inEvents != null? inEvents.size() : -1));
        }
        if(inEvents!=null && !inEvents.isEmpty()) {
          TezVertexID vertexId = taskAttemptID.getTaskID().getVertexID();
          context.getEventHandler().handle(
              new VertexEventRouteEvent(vertexId, inEvents));
        }
        taskHeartbeatHandler.pinged(taskAttemptID);
        List<TezEvent> outEvents = context
            .getCurrentDAG()
            .getVertex(taskAttemptID.getTaskID().getVertexID())
            .getTask(taskAttemptID.getTaskID())
            .getTaskAttemptTezEvents(taskAttemptID, request.getStartIndex(),
                request.getMaxEvents());
        response.setEvents(outEvents);
      }
      containerInfo.lastRequestId = requestId;
View Full Code Here

Examples of org.apache.tez.dag.records.TezTaskAttemptID

        case DATA_MOVEMENT_EVENT:
        case COMPOSITE_DATA_MOVEMENT_EVENT:
          {
            if (isEventFromVertex(vertex, sourceMeta)) {
              // event from this vertex. send to destination vertex
              TezTaskAttemptID srcTaId = sourceMeta.getTaskAttemptID();
              if (tezEvent.getEventType() == EventType.DATA_MOVEMENT_EVENT) {
                ((DataMovementEvent) tezEvent.getEvent()).setVersion(srcTaId.getId());
              } else if (tezEvent.getEventType() == EventType.COMPOSITE_DATA_MOVEMENT_EVENT) {
                ((CompositeDataMovementEvent) tezEvent.getEvent()).setVersion(srcTaId.getId());
              } else {
                ((InputFailedEvent) tezEvent.getEvent()).setVersion(srcTaId.getId());
              }
              Vertex destVertex = vertex.getDAG().getVertex(sourceMeta.getEdgeVertexName());
              Edge destEdge = vertex.targetVertices.get(destVertex);
              if (destEdge == null) {
                throw new TezUncheckedException("Bad destination vertex: " +
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.