Examples of org.apache.tez.runtime.api.events.InputDataInformationEvent

Package org.apache.tez.runtime.api.events

Examples of org.apache.tez.runtime.api.events.InputDataInformationEvent

org.apache.tez.runtime.api.events.InputDataInformationEvent
Events used by {@link InputInitializer} implementations to provide theuser payload for individual tasks running as part of the Vertex for which an Initial Input has been configured. This event is used by InputInitialziers to configure tasks belonging to a Vertex. The event may be processed by a @link {@link VertexManagerPlugin}before being sent to tasks. A {@link InputInitializer} may send Events with or without aserialized user payload. Events, after being processed by a {@link VertexManagerPlugin}, must contain the payload in a serialized form.

  }


  public static InputDataInformationEvent
      convertRootInputDataInformationEventFromProto(
      EventProtos.RootInputDataInformationEventProto proto) {
    InputDataInformationEvent diEvent = InputDataInformationEvent.createWithSerializedPayload(
        proto.getSourceIndex(),
        proto.hasUserPayload() ? proto.getUserPayload().asReadOnlyByteBuffer() : null);
    diEvent.setTargetIndex(proto.getTargetIndex());
    return diEvent;
  }

View Full Code Here

        case ROOT_INPUT_DATA_INFORMATION_EVENT:
          if (vertex.tasksNotYetScheduled) {
            vertex.pendingTaskEvents.add(tezEvent);
          } else {
            checkEventSourceMetadata(vertex, sourceMeta);
            InputDataInformationEvent riEvent = (InputDataInformationEvent) tezEvent
                .getEvent();
            Task targetTask = vertex.getTask(riEvent.getTargetIndex());
            targetTask.registerTezEvent(tezEvent);
          }
          break;
        case VERTEX_MANAGER_EVENT:
        {
          VertexManagerEvent vmEvent = (VertexManagerEvent) tezEvent.getEvent();
          Vertex target = vertex.getDAG().getVertex(vmEvent.getTargetVertexName());
          Preconditions.checkArgument(target != null,
              "Event sent to unkown vertex: " + vmEvent.getTargetVertexName());
          if (target == vertex) {
            vertex.vertexManager.onVertexManagerEventReceived(vmEvent);
          } else {
            checkEventSourceMetadata(vertex, sourceMeta);
            vertex.eventHandler.handle(new VertexEventRouteEvent(target
                .getVertexId(), Collections.singletonList(tezEvent)));
          }
        }
          break;
        case ROOT_INPUT_INITIALIZER_EVENT:
        {
          InputInitializerEvent riEvent = (InputInitializerEvent) tezEvent.getEvent();
          Vertex target = vertex.getDAG().getVertex(riEvent.getTargetVertexName());
          Preconditions.checkArgument(target != null,
              "Event sent to unkown vertex: " + riEvent.getTargetVertexName());
          if (target == vertex) {
            vertex.rootInputInitializerManager.handleInitializerEvent(riEvent);
          } else {
            checkEventSourceMetadata(vertex, sourceMeta);
            vertex.eventHandler.handle(new VertexEventRouteEvent(target.getVertexId(),

View Full Code Here

            "RootInputVertexManager cannot configure multiple inputs. Use a custom VertexManager"
                + ", VertexName:" + getContext().getVertexName() + ", ConfiguredInput: "
                + configuredInputName + ", CurrentInput: " + inputName);
        configuredInputName = inputName;
        
        InputDataInformationEvent rEvent = (InputDataInformationEvent)event;
        rEvent.setTargetIndex(rEvent.getSourceIndex()); // 1:1 routing
        riEvents.add(rEvent);
      }
    }
    getContext().addRootInputEvents(inputName, riEvents);
  }

View Full Code Here


    if (sendSerializedEvents) {
      MRSplitsProto splitsProto = inputSplitInfo.getSplitsProto();
      int count = 0;
      for (MRSplitProto mrSplit : splitsProto.getSplitsList()) {
        InputDataInformationEvent diEvent = InputDataInformationEvent.createWithSerializedPayload(
            count++, mrSplit.toByteString().asReadOnlyByteBuffer());
        events.add(diEvent);
      }
    } else {
      int count = 0;
      for (org.apache.hadoop.mapred.InputSplit split : inputSplitInfo.getOldFormatSplits()) {
        InputDataInformationEvent diEvent = InputDataInformationEvent.createWithObjectPayload(
            count++, split);
        events.add(diEvent);
      }
    }
    return events;

View Full Code Here

      if (event instanceof InputUpdatePayloadEvent) {
        // this event can never occur. If it does, fail.
        Preconditions.checkState(false);
      } else if (event instanceof InputDataInformationEvent) {
        dataInformationEventSeen = true;
        InputDataInformationEvent diEvent = (InputDataInformationEvent) event;
        FileSplit fileSplit;
        try {
          fileSplit = getFileSplitFromEvent(diEvent);
        } catch (IOException e) {
          throw new RuntimeException("Failed to get file split for event: " + diEvent);

View Full Code Here

      Collection<Integer> destTasks = bucketToTaskMap.get(entry.getKey());
      for (Integer task : destTasks) {
        int count = 0;
        for (ByteBuffer buf : entry.getValue()) {
          count++;
          InputDataInformationEvent diEvent =
              InputDataInformationEvent.createWithSerializedPayload(count, buf);
          diEvent.setTargetIndex(task);
          taskEvents.add(diEvent);
        }
        numSplitsForTask[task] = count;
      }
    }

View Full Code Here

          if ((subSplit instanceof TezGroupedSplit) == false) {
            throw new IOException("Unexpected split type found: "
                + subSplit.getClass().getCanonicalName());
          }
          MRSplitProto serializedSplit = MRInputHelpers.createSplitProto(subSplit);
          InputDataInformationEvent diEvent =
              InputDataInformationEvent.createWithSerializedPayload(count, serializedSplit
                  .toByteString().asReadOnlyByteBuffer());
          diEvent.setTargetIndex(count);
          taskEvents.add(diEvent);
        }
      } else {
        MRSplitProto serializedSplit = MRInputHelpers.createSplitProto(inputSplit);
        InputDataInformationEvent diEvent =
            InputDataInformationEvent.createWithSerializedPayload(count, serializedSplit
                .toByteString().asReadOnlyByteBuffer());
        diEvent.setTargetIndex(count);
        taskEvents.add(diEvent);
      }
      count++;
    }

View Full Code Here

0 1

TOP

Related Classes of org.apache.tez.runtime.api.events.InputDataInformationEvent

org.apache.hadoop.hive.ql.exec.tez.CustomPartitionVertex

org.apache.hadoop.hive.ql.exec.tez.HiveSplitGenerator

org.apache.tez.common.ProtoConverters

org.apache.tez.dag.app.dag.impl.RootInputVertexManager

org.apache.tez.dag.app.dag.impl.TestRootInputVertexManager

org.apache.tez.dag.app.dag.impl.TestVertexImpl$EventHandlingRootInputInitializer

org.apache.tez.dag.app.dag.impl.TestVertexImpl$RootInputInitializerManagerControlled

org.apache.tez.dag.app.dag.impl.VertexImpl$RouteEventTransition

org.apache.tez.mapreduce.common.MRInputAMSplitGenerator

org.apache.tez.mapreduce.common.MRInputSplitDistributor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.