Examples of org.apache.tez.runtime.api.events.RootInputDataInformationEvent

Package org.apache.tez.runtime.api.events

Examples of org.apache.tez.runtime.api.events.RootInputDataInformationEvent

org.apache.tez.runtime.api.events.RootInputDataInformationEvent
Events used by {@link TezRootInputInitializer} implementations to provide thebyte payload for individual tasks running as part of the Vertex for which an Initial Input has been configured.

            .setSourceIndex(ifEvt.getSourceIndex())
            .setTargetIndex(ifEvt.getTargetIndex())
            .setVersion(ifEvt.getVersion()).build().toByteArray();
        break;
      case ROOT_INPUT_DATA_INFORMATION_EVENT:
        RootInputDataInformationEvent liEvent = (RootInputDataInformationEvent) event;
        eventBytes = RootInputDataInformationEventProto.newBuilder()
            .setIndex(liEvent.getIndex())
            .setUserPayload(ByteString.copyFrom(liEvent.getUserPayload()))
            .build().toByteArray();
        break;
      default:
        throw new TezUncheckedException("Unknown TezEvent"
           + ", type=" + eventType);

View Full Code Here

            ifProto.getTargetIndex(), ifProto.getVersion());
        break;
      case ROOT_INPUT_DATA_INFORMATION_EVENT:
        RootInputDataInformationEventProto difProto = RootInputDataInformationEventProto
            .parseFrom(eventBytes);
        event = new RootInputDataInformationEvent(difProto.getIndex(), difProto
            .getUserPayload().toByteArray());
        break;
      default:
        // RootInputUpdatePayload event not wrapped in a TezEvent.
        throw new TezUncheckedException("Unexpected TezEvent"

View Full Code Here

    assertEquals(3, events.size());
    assertTrue(events.get(0) instanceof RootInputUpdatePayloadEvent);
    assertTrue(events.get(1) instanceof RootInputDataInformationEvent);
    assertTrue(events.get(2) instanceof RootInputDataInformationEvent);


    RootInputDataInformationEvent diEvent1 = (RootInputDataInformationEvent) (events.get(1));
    RootInputDataInformationEvent diEvent2 = (RootInputDataInformationEvent) (events.get(2));


    assertNull(diEvent1.getDeserializedUserPayload());
    assertNull(diEvent2.getDeserializedUserPayload());


    assertNotNull(diEvent1.getUserPayload());
    assertNotNull(diEvent2.getUserPayload());


    MRSplitProto event1Proto = MRSplitProto.parseFrom(diEvent1.getUserPayload());
    InputSplit is1 = MRInput.getOldSplitDetailsFromEvent(event1Proto, new Configuration());
    assertTrue(is1 instanceof InputSplitForTest);
    assertEquals(1, ((InputSplitForTest) is1).identifier);


    MRSplitProto event2Proto = MRSplitProto.parseFrom(diEvent2.getUserPayload());
    InputSplit is2 = MRInput.getOldSplitDetailsFromEvent(event2Proto, new Configuration());
    assertTrue(is2 instanceof InputSplitForTest);
    assertEquals(2, ((InputSplitForTest) is2).identifier);
  }

View Full Code Here

    assertEquals(3, events.size());
    assertTrue(events.get(0) instanceof RootInputUpdatePayloadEvent);
    assertTrue(events.get(1) instanceof RootInputDataInformationEvent);
    assertTrue(events.get(2) instanceof RootInputDataInformationEvent);


    RootInputDataInformationEvent diEvent1 = (RootInputDataInformationEvent) (events.get(1));
    RootInputDataInformationEvent diEvent2 = (RootInputDataInformationEvent) (events.get(2));


    assertNull(diEvent1.getUserPayload());
    assertNull(diEvent2.getUserPayload());


    assertNotNull(diEvent1.getDeserializedUserPayload());
    assertNotNull(diEvent2.getDeserializedUserPayload());


    assertTrue(diEvent1.getDeserializedUserPayload() instanceof InputSplitForTest);
    assertEquals(1, ((InputSplitForTest) diEvent1.getDeserializedUserPayload()).identifier);


    assertTrue(diEvent2.getDeserializedUserPayload() instanceof InputSplitForTest);
    assertEquals(2, ((InputSplitForTest) diEvent2.getDeserializedUserPayload()).identifier);
  }

View Full Code Here

    events.add(updatePayloadEvent);
    int count = 0;


    for (MRSplitProto mrSplit : this.splitsProto.getSplitsList()) {


      RootInputDataInformationEvent diEvent;


      if (sendSerializedEvents) {
        // Unnecessary array copy, can be avoided by using ByteBuffer instead of
        // a raw array.
        diEvent = new RootInputDataInformationEvent(count++, mrSplit.toByteArray());
      } else {
        if (useNewApi) {
          org.apache.hadoop.mapreduce.InputSplit newInputSplit = MRInput
              .getNewSplitDetailsFromEvent(mrSplit, conf);
          diEvent = new RootInputDataInformationEvent(count++, newInputSplit);
        } else {
          org.apache.hadoop.mapred.InputSplit oldInputSplit = MRInput.getOldSplitDetailsFromEvent(
              mrSplit, conf);
          diEvent = new RootInputDataInformationEvent(count++, oldInputSplit);
        }
      }
      events.add(diEvent);
    }

View Full Code Here

    if (sendSerializedEvents) {
      MRSplitsProto splitsProto = inputSplitInfo.getSplitsProto();
      int count = 0;
      for (MRSplitProto mrSplit : splitsProto.getSplitsList()) {
        // Unnecessary array copy, can be avoided by using ByteBuffer instead of a raw array.
        RootInputDataInformationEvent diEvent = new RootInputDataInformationEvent(count++,
            mrSplit.toByteArray());
        events.add(diEvent);
      }
    } else {
      int count = 0;
      if (inputSplitInfo.holdsNewFormatSplits()) {
        for (org.apache.hadoop.mapreduce.InputSplit split : inputSplitInfo.getNewFormatSplits()) {
          RootInputDataInformationEvent diEvent = new RootInputDataInformationEvent(count++, split);
          events.add(diEvent);
        }
      } else {
        for (org.apache.hadoop.mapred.InputSplit split : inputSplitInfo.getOldFormatSplits()) {
          RootInputDataInformationEvent diEvent = new RootInputDataInformationEvent(count++, split);
          events.add(diEvent);
        }
      }
    }

View Full Code Here

  }


  public static RootInputDataInformationEvent
      convertRootInputDataInformationEventFromProto(
      EventProtos.RootInputDataInformationEventProto proto) {
    RootInputDataInformationEvent diEvent = new RootInputDataInformationEvent(
        proto.getSourceIndex(), proto.getUserPayload() != null ? proto.getUserPayload()
            .toByteArray() : null);
    diEvent.setTargetIndex(proto.getTargetIndex());
    return diEvent;
  }

View Full Code Here


      RootInputConfigureVertexTasksEvent configEvent = new RootInputConfigureVertexTasksEvent(
          targetTasks, locationHints);
      events.add(configEvent);
      for (int i = 0; i < targetTasks; i++) {
        RootInputDataInformationEvent diEvent = new RootInputDataInformationEvent(
            i, null);
        events.add(diEvent);
      }
      eventHandler.handle(new VertexEventRootInputInitialized(vertexID, inputs
          .get(0).getEntityName(), events));

View Full Code Here

      if (event instanceof RootInputUpdatePayloadEvent) {
        // this event can never occur. If it does, fail.
        Preconditions.checkState(false);
      } else if (event instanceof RootInputDataInformationEvent) {
        dataInformationEventSeen = true;
        RootInputDataInformationEvent diEvent = (RootInputDataInformationEvent) event;
        dataInformationEvents.add(diEvent);
        FileSplit fileSplit;
        try {
          fileSplit = getFileSplitFromEvent(diEvent);
        } catch (IOException e) {

View Full Code Here

    List<RootInputDataInformationEvent> taskEvents = Lists.newArrayListWithCapacity(finalSplits.size());
    // Re-serialize the splits after grouping.
    int count = 0;
    for (InputSplit inputSplit : finalSplits) {
      MRSplitProto serializedSplit = MRHelpers.createSplitProto(inputSplit);
      RootInputDataInformationEvent diEvent = new RootInputDataInformationEvent(
          count, serializedSplit.toByteArray());
      diEvent.setTargetIndex(count);
      count++;
      taskEvents.add(diEvent);
    }


    // Replace the Edge Managers

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.tez.runtime.api.events.RootInputDataInformationEvent

org.apache.hadoop.hive.ql.exec.tez.CustomPartitionVertex

org.apache.tez.common.ProtoConverters

org.apache.tez.dag.app.dag.impl.RootInputVertexManager

org.apache.tez.dag.app.dag.impl.TestVertexImpl$RootInputInitializerRunnerControlled

org.apache.tez.dag.app.dag.impl.VertexImpl$RouteEventTransition

org.apache.tez.mapreduce.common.MRInputAMSplitGenerator

org.apache.tez.mapreduce.common.MRInputSplitDistributor

org.apache.tez.mapreduce.common.TestMRInputSplitDistributor

org.apache.tez.runtime.api.impl.TezEvent

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.