Package org.apache.tez.dag.history

Examples of org.apache.tez.dag.history.HistoryEvent


  }

  @Test
  public void testHandlerExists() throws JSONException {
    for (HistoryEventType eventType : HistoryEventType.values()) {
      HistoryEvent event = null;
      switch (eventType) {
        case AM_LAUNCHED:
          event = new AMLaunchedEvent(applicationAttemptId, random.nextInt(), random.nextInt(),
              user);
          break;
        case AM_STARTED:
          event = new AMStartedEvent(applicationAttemptId, random.nextInt(), user);
          break;
        case DAG_SUBMITTED:
          event = new DAGSubmittedEvent(tezDAGID, random.nextInt(), dagPlan, applicationAttemptId,
              null, user);
          break;
        case DAG_INITIALIZED:
          event = new DAGInitializedEvent(tezDAGID, random.nextInt(), user, dagPlan.getName());
          break;
        case DAG_STARTED:
          event = new DAGStartedEvent(tezDAGID, random.nextInt(), user, dagPlan.getName());
          break;
        case DAG_FINISHED:
          event = new DAGFinishedEvent(tezDAGID, random.nextInt(), random.nextInt(), DAGState.ERROR,
              null, null, user, dagPlan.getName());
          break;
        case VERTEX_INITIALIZED:
          event = new VertexInitializedEvent(tezVertexID, "v1", random.nextInt(), random.nextInt(),
              random.nextInt(), "proc", null);
          break;
        case VERTEX_STARTED:
          event = new VertexStartedEvent(tezVertexID, random.nextInt(), random.nextInt());
          break;
        case VERTEX_PARALLELISM_UPDATED:
          event = new VertexParallelismUpdatedEvent();
          break;
        case VERTEX_FINISHED:
          event = new VertexFinishedEvent(tezVertexID, "v1", random.nextInt(), random.nextInt(),
              random.nextInt(), random.nextInt(), random.nextInt(), VertexState.ERROR,
              null, null, null);
          break;
        case TASK_STARTED:
          event = new TaskStartedEvent(tezTaskID, "v1", random.nextInt(), random.nextInt());
          break;
        case TASK_FINISHED:
          event = new TaskFinishedEvent(tezTaskID, "v1", random.nextInt(), random.nextInt(),
              tezTaskAttemptID, TaskState.FAILED, null, null);
          break;
        case TASK_ATTEMPT_STARTED:
          event = new TaskAttemptStartedEvent(tezTaskAttemptID, "v1", random.nextInt(), containerId,
              nodeId, null, null);
          break;
        case TASK_ATTEMPT_FINISHED:
          event = new TaskAttemptFinishedEvent(tezTaskAttemptID, "v1", random.nextInt(),
              random.nextInt(), TaskAttemptState.FAILED, null, null);
          break;
        case CONTAINER_LAUNCHED:
          event = new ContainerLaunchedEvent(containerId, random.nextInt(),
              applicationAttemptId);
          break;
        case CONTAINER_STOPPED:
          event = new ContainerStoppedEvent(containerId, random.nextInt(), -1, applicationAttemptId);
          break;
        case VERTEX_DATA_MOVEMENT_EVENTS_GENERATED:
          event = new VertexDataMovementEventsGeneratedEvent();
          break;
        case DAG_COMMIT_STARTED:
          event = new DAGCommitStartedEvent();
          break;
        case VERTEX_COMMIT_STARTED:
          event = new VertexCommitStartedEvent();
          break;
        case VERTEX_GROUP_COMMIT_STARTED:
          event = new VertexGroupCommitStartedEvent();
          break;
        case VERTEX_GROUP_COMMIT_FINISHED:
          event = new VertexGroupCommitFinishedEvent();
          break;
        default:
          Assert.fail("Unhandled event type " + eventType);
      }
      if (event == null || !event.isHistoryEvent()) {
        continue;
      }
      HistoryEventTimelineConversion.convertToTimelineEntity(event);
    }
  }
View Full Code Here


      // reached end
      throw new IOException("Corrupt data found when trying to read next event type"
          + ", eventTypeOrdinal=" + eventTypeOrdinal);
    }
    HistoryEventType eventType = HistoryEventType.values()[eventTypeOrdinal];
    HistoryEvent event;
    switch (eventType) {
      case AM_LAUNCHED:
        event = new AMLaunchedEvent();
        break;
      case AM_STARTED:
        event = new AMStartedEvent();
        break;
      case DAG_SUBMITTED:
        event = new DAGSubmittedEvent();
        break;
      case DAG_INITIALIZED:
        event = new DAGInitializedEvent();
        break;
      case DAG_STARTED:
        event = new DAGStartedEvent();
        break;
      case DAG_COMMIT_STARTED:
        event = new DAGCommitStartedEvent();
        break;
      case DAG_FINISHED:
        event = new DAGFinishedEvent();
        break;
      case CONTAINER_LAUNCHED:
        event = new ContainerLaunchedEvent();
        break;
      case CONTAINER_STOPPED:
        event = new ContainerStoppedEvent();
        break;
      case VERTEX_INITIALIZED:
        event = new VertexInitializedEvent();
        break;
      case VERTEX_STARTED:
        event = new VertexStartedEvent();
        break;
      case VERTEX_PARALLELISM_UPDATED:
        event = new VertexParallelismUpdatedEvent();
        break;
      case VERTEX_COMMIT_STARTED:
        event = new VertexCommitStartedEvent();
        break;
      case VERTEX_GROUP_COMMIT_STARTED:
        event = new VertexGroupCommitStartedEvent();
        break;
      case VERTEX_GROUP_COMMIT_FINISHED:
        event = new VertexGroupCommitFinishedEvent();
        break;
      case VERTEX_FINISHED:
        event = new VertexFinishedEvent();
        break;
      case TASK_STARTED:
        event = new TaskStartedEvent();
        break;
      case TASK_FINISHED:
        event = new TaskFinishedEvent();
        break;
      case TASK_ATTEMPT_STARTED:
        event = new TaskAttemptStartedEvent();
        break;
      case TASK_ATTEMPT_FINISHED:
        event = new TaskAttemptFinishedEvent();
        break;
      case VERTEX_DATA_MOVEMENT_EVENTS_GENERATED:
        event = new VertexDataMovementEventsGeneratedEvent();
        break;
      default:
        throw new IOException("Invalid data found, unknown event type "
            + eventType);

    }
    if (LOG.isDebugEnabled()) {
      LOG.debug("Parsing event from input stream"
          + ", eventType=" + eventType);
    }
    try {
      event.fromProtoStream(inputStream);
    } catch (EOFException eof) {
      return null;
    }
    if (LOG.isDebugEnabled()) {
      LOG.debug("Parsed event from input stream"
          + ", eventType=" + eventType
          + ", event=" + event.toString());
    }
    return event;
  }
View Full Code Here


  private static void parseDAGRecoveryFile(FSDataInputStream inputStream)
      throws IOException {
    while (true) {
      HistoryEvent historyEvent = getNextEvent(inputStream);
      if (historyEvent == null) {
        LOG.info("Reached end of stream");
        break;
      }
      LOG.info("Parsed event from recovery stream"
          + ", eventType=" + historyEvent.getEventType()
          + ", event=" + historyEvent);
    }
  }
View Full Code Here

    FSDataOutputStream newDAGRecoveryStream =
        getDAGRecoveryOutputStream(currentAttemptRecoveryDataDir, lastInProgressDAG);

    boolean skipAllOtherEvents = false;
    while (true) {
      HistoryEvent event;
      try {
        event = getNextEvent(dagRecoveryStream);
        if (event == null) {
          LOG.info("Reached end of dag recovery stream");
          break;
        }
      } catch (EOFException eof) {
        LOG.info("Reached end of dag recovery stream");
        break;
      } catch (IOException ioe) {
        LOG.warn("Corrupt data found when trying to read next event", ioe);
        break;
      }
      if (event == null || skipAllOtherEvents) {
        // reached end of data
        break;
      }
      HistoryEventType eventType = event.getEventType();
      switch (eventType) {
        case DAG_SUBMITTED:
        {
          DAGSubmittedEvent submittedEvent = (DAGSubmittedEvent) event;
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          recoveredDAGData.recoveredDAG = dagAppMaster.createDAG(submittedEvent.getDAGPlan(),
              lastInProgressDAG);
          recoveredDAGData.cumulativeAdditionalResources = submittedEvent
            .getCumulativeAdditionalLocalResources();
          recoveredDAGData.recoveredDagID = recoveredDAGData.recoveredDAG.getID();
          dagAppMaster.setCurrentDAG(recoveredDAGData.recoveredDAG);
          if (recoveredDAGData.nonRecoverable) {
            skipAllOtherEvents = true;
          }
          break;
        }
        case DAG_INITIALIZED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          recoveredDAGData.recoveredDAG.restoreFromEvent(event);
          break;
        }
        case DAG_STARTED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          recoveredDAGData.recoveredDAG.restoreFromEvent(event);
          break;
        }
        case DAG_COMMIT_STARTED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          recoveredDAGData.recoveredDAG.restoreFromEvent(event);
          break;
        }
        case VERTEX_GROUP_COMMIT_STARTED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          recoveredDAGData.recoveredDAG.restoreFromEvent(event);
          break;
        }
        case VERTEX_GROUP_COMMIT_FINISHED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          recoveredDAGData.recoveredDAG.restoreFromEvent(event);
          break;
        }
        case DAG_FINISHED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          // If this is seen, nothing to recover
          assert recoveredDAGData.recoveredDAG != null;
          recoveredDAGData.recoveredDAG.restoreFromEvent(event);
          recoveredDAGData.isCompleted = true;
          recoveredDAGData.dagState =
              ((DAGFinishedEvent) event).getState();
          skipAllOtherEvents = true;
        }
        case CONTAINER_LAUNCHED:
        {
          // Nothing to do for now
          break;
        }
        case VERTEX_INITIALIZED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          VertexInitializedEvent vEvent = (VertexInitializedEvent) event;
          Vertex v = recoveredDAGData.recoveredDAG.getVertex(vEvent.getVertexID());
          v.restoreFromEvent(vEvent);
          break;
        }
        case VERTEX_STARTED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          VertexStartedEvent vEvent = (VertexStartedEvent) event;
          Vertex v = recoveredDAGData.recoveredDAG.getVertex(vEvent.getVertexID());
          v.restoreFromEvent(vEvent);
          break;
        }
        case VERTEX_PARALLELISM_UPDATED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          VertexParallelismUpdatedEvent vEvent = (VertexParallelismUpdatedEvent) event;
          Vertex v = recoveredDAGData.recoveredDAG.getVertex(vEvent.getVertexID());
          v.restoreFromEvent(vEvent);
          break;
        }
        case VERTEX_COMMIT_STARTED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          VertexCommitStartedEvent vEvent = (VertexCommitStartedEvent) event;
          Vertex v = recoveredDAGData.recoveredDAG.getVertex(vEvent.getVertexID());
          v.restoreFromEvent(vEvent);
          break;
        }
        case VERTEX_FINISHED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          VertexFinishedEvent vEvent = (VertexFinishedEvent) event;
          Vertex v = recoveredDAGData.recoveredDAG.getVertex(vEvent.getVertexID());
          v.restoreFromEvent(vEvent);
          break;
        }
        case TASK_STARTED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          TaskStartedEvent tEvent = (TaskStartedEvent) event;
          Task task = recoveredDAGData.recoveredDAG.getVertex(
              tEvent.getTaskID().getVertexID()).getTask(tEvent.getTaskID());
          task.restoreFromEvent(tEvent);
          break;
        }
        case TASK_FINISHED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          TaskFinishedEvent tEvent = (TaskFinishedEvent) event;
          Task task = recoveredDAGData.recoveredDAG.getVertex(
              tEvent.getTaskID().getVertexID()).getTask(tEvent.getTaskID());
          task.restoreFromEvent(tEvent);
          break;
        }
        case TASK_ATTEMPT_STARTED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          TaskAttemptStartedEvent tEvent = (TaskAttemptStartedEvent) event;
          Task task =
              recoveredDAGData.recoveredDAG.getVertex(
                  tEvent.getTaskAttemptID().getTaskID().getVertexID())
                      .getTask(tEvent.getTaskAttemptID().getTaskID());
          task.restoreFromEvent(tEvent);
          break;
        }
        case TASK_ATTEMPT_FINISHED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          TaskAttemptFinishedEvent tEvent = (TaskAttemptFinishedEvent) event;
          Task task =
              recoveredDAGData.recoveredDAG.getVertex(
                  tEvent.getTaskAttemptID().getTaskID().getVertexID())
                  .getTask(tEvent.getTaskAttemptID().getTaskID());
          task.restoreFromEvent(tEvent);
          break;
        }
        case VERTEX_DATA_MOVEMENT_EVENTS_GENERATED:
        {
          LOG.info("Recovering from event"
              + ", eventType=" + eventType
              + ", event=" + event.toString());
          assert recoveredDAGData.recoveredDAG != null;
          VertexDataMovementEventsGeneratedEvent vEvent =
              (VertexDataMovementEventsGeneratedEvent) event;
          Vertex v = recoveredDAGData.recoveredDAG.getVertex(vEvent.getVertexID());
          v.restoreFromEvent(vEvent);
          break;
        }
        default:
          throw new RuntimeException("Invalid data found, unknown event type "
              + eventType);
      }
      if (LOG.isDebugEnabled()) {
        LOG.debug("[DAG RECOVERY]"
            + " dagId=" + lastInProgressDAG
            + ", eventType=" + eventType
            + ", event=" + event.toString());
      }
      newDAGRecoveryStream.writeInt(eventType.ordinal());
      event.toProtoStream(newDAGRecoveryStream);
    }
    dagRecoveryStream.close();
    newDAGRecoveryStream.hsync();
    newDAGRecoveryStream.close();
View Full Code Here

        // reached end
        LOG.warn("Corrupt data found when trying to read next event type");
        break;
      }
      HistoryEventType eventType = HistoryEventType.values()[eventTypeOrdinal];
      HistoryEvent event = null;
      switch (eventType) {
        case AM_LAUNCHED:
          event = new AMLaunchedEvent();
          break;
        case AM_STARTED:
          event = new AMStartedEvent();
          break;
        case DAG_SUBMITTED:
          event = new DAGSubmittedEvent();
          break;
        case DAG_INITIALIZED:
          event = new DAGInitializedEvent();
          break;
        case DAG_STARTED:
          event = new DAGStartedEvent();
          break;
        case DAG_FINISHED:
          event = new DAGFinishedEvent();
          break;
        case CONTAINER_LAUNCHED:
          event = new ContainerLaunchedEvent();
          break;
        case VERTEX_INITIALIZED:
          event = new VertexInitializedEvent();
          break;
        case VERTEX_STARTED:
          event = new VertexStartedEvent();
          break;
        case VERTEX_FINISHED:
          event = new VertexFinishedEvent();
          break;
        case TASK_STARTED:
          event = new TaskStartedEvent();
          break;
        case TASK_FINISHED:
          event = new TaskFinishedEvent();
          break;
        case TASK_ATTEMPT_STARTED:
          event = new TaskAttemptStartedEvent();
          break;
        case TASK_ATTEMPT_FINISHED:
          event = new TaskAttemptFinishedEvent();
          break;
        case VERTEX_DATA_MOVEMENT_EVENTS_GENERATED:
          event = new VertexDataMovementEventsGeneratedEvent();
          break;
        default:
          throw new IOException("Invalid data found, unknown event type "
              + eventType);

      }
      ++counter;
      LOG.info("Parsing event from input stream"
          + ", eventType=" + eventType
          + ", eventIndex=" + counter);
      event.fromProtoStream(inputStream);
      LOG.info("Parsed event from input stream"
          + ", eventType=" + eventType
          + ", eventIndex=" + counter
          + ", event=" + event.toString());
    }
  }
View Full Code Here

      TestHistoryEventsProtoConversion.class);


  private HistoryEvent testProtoConversion(HistoryEvent event) throws IOException {
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    HistoryEvent deserializedEvent = null;
    event.toProtoStream(os);
    os.flush();
    os.close();
    deserializedEvent = ReflectionUtils.createClazzInstance(
        event.getClass().getName());
    LOG.info("Serialized event to byte array"
        + ", eventType=" + event.getEventType()
        + ", bufLen=" + os.toByteArray().length);
    deserializedEvent.fromProtoStream(
        new ByteArrayInputStream(os.toByteArray()));
    return deserializedEvent;
  }
View Full Code Here

  private HistoryEvent testSummaryProtoConversion(HistoryEvent historyEvent)
      throws IOException {
    SummaryEvent event = (SummaryEvent) historyEvent;
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    HistoryEvent deserializedEvent = null;
    event.toSummaryProtoStream(os);
    os.flush();
    os.close();
    LOG.info("Serialized event to byte array"
        + ", eventType=" + historyEvent.getEventType()
View Full Code Here

  }

  @Test
  public void testHandlerExists() throws JSONException {
    for (HistoryEventType eventType : HistoryEventType.values()) {
      HistoryEvent event = null;
      switch (eventType) {
        case AM_LAUNCHED:
          event = new AMLaunchedEvent(applicationAttemptId, random.nextInt(), random.nextInt(),
              user);
          break;
        case AM_STARTED:
          event = new AMStartedEvent(applicationAttemptId, random.nextInt(), user);
          break;
        case DAG_SUBMITTED:
          event = new DAGSubmittedEvent(tezDAGID, random.nextInt(), dagPlan, applicationAttemptId,
              null, user);
          break;
        case DAG_INITIALIZED:
          event = new DAGInitializedEvent(tezDAGID, random.nextInt(), user, dagPlan.getName());
          break;
        case DAG_STARTED:
          event = new DAGStartedEvent(tezDAGID, random.nextInt(), user, dagPlan.getName());
          break;
        case DAG_FINISHED:
          event = new DAGFinishedEvent(tezDAGID, random.nextInt(), random.nextInt(), DAGState.ERROR,
              null, null, user, dagPlan.getName());
          break;
        case VERTEX_INITIALIZED:
          event = new VertexInitializedEvent(tezVertexID, "v1", random.nextInt(), random.nextInt(),
              random.nextInt(), "proc", null);
          break;
        case VERTEX_STARTED:
          event = new VertexStartedEvent(tezVertexID, random.nextInt(), random.nextInt());
          break;
        case VERTEX_PARALLELISM_UPDATED:
          event = new VertexParallelismUpdatedEvent();
          break;
        case VERTEX_FINISHED:
          event = new VertexFinishedEvent(tezVertexID, "v1", random.nextInt(), random.nextInt(),
              random.nextInt(), random.nextInt(), random.nextInt(), VertexState.ERROR,
              null, null, null);
          break;
        case TASK_STARTED:
          event = new TaskStartedEvent(tezTaskID, "v1", random.nextInt(), random.nextInt());
          break;
        case TASK_FINISHED:
          event = new TaskFinishedEvent(tezTaskID, "v1", random.nextInt(), random.nextInt(),
              tezTaskAttemptID, TaskState.FAILED, null, null);
          break;
        case TASK_ATTEMPT_STARTED:
          event = new TaskAttemptStartedEvent(tezTaskAttemptID, "v1", random.nextInt(), containerId,
              nodeId, null, null);
          break;
        case TASK_ATTEMPT_FINISHED:
          event = new TaskAttemptFinishedEvent(tezTaskAttemptID, "v1", random.nextInt(),
              random.nextInt(), TaskAttemptState.FAILED, null, null);
          break;
        case CONTAINER_LAUNCHED:
          event = new ContainerLaunchedEvent(containerId, random.nextInt(),
              applicationAttemptId);
          break;
        case CONTAINER_STOPPED:
          event = new ContainerStoppedEvent(containerId, random.nextInt(), -1, applicationAttemptId);
          break;
        case VERTEX_DATA_MOVEMENT_EVENTS_GENERATED:
          event = new VertexDataMovementEventsGeneratedEvent();
          break;
        case DAG_COMMIT_STARTED:
          event = new DAGCommitStartedEvent();
          break;
        case VERTEX_COMMIT_STARTED:
          event = new VertexCommitStartedEvent();
          break;
        case VERTEX_GROUP_COMMIT_STARTED:
          event = new VertexGroupCommitStartedEvent();
          break;
        case VERTEX_GROUP_COMMIT_FINISHED:
          event = new VertexGroupCommitFinishedEvent();
          break;
        default:
          Assert.fail("Unhandled event type " + eventType);
      }
      if (event == null || !event.isHistoryEvent()) {
        continue;
      }
      HistoryEventJsonConversion.convertToJson(event);
    }
  }
View Full Code Here

      TestHistoryEventsProtoConversion.class);


  private HistoryEvent testProtoConversion(HistoryEvent event) throws IOException {
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    HistoryEvent deserializedEvent = null;
    event.toProtoStream(os);
    os.flush();
    os.close();
    deserializedEvent = RuntimeUtils.createClazzInstance(
        event.getClass().getName());
    LOG.info("Serialized event to byte array"
        + ", eventType=" + event.getEventType()
        + ", bufLen=" + os.toByteArray().length);
    deserializedEvent.fromProtoStream(
        new ByteArrayInputStream(os.toByteArray()));
    return deserializedEvent;
  }
View Full Code Here

  private HistoryEvent testSummaryProtoConversion(HistoryEvent historyEvent)
      throws IOException {
    SummaryEvent event = (SummaryEvent) historyEvent;
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    HistoryEvent deserializedEvent = null;
    event.toSummaryProtoStream(os);
    os.flush();
    os.close();
    LOG.info("Serialized event to byte array"
        + ", eventType=" + historyEvent.getEventType()
View Full Code Here

TOP

Related Classes of org.apache.tez.dag.history.HistoryEvent

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.