Package org.apache.tez.runtime.api

Examples of org.apache.tez.runtime.api.LogicalInput



    // then add the non-grouped inputs
    for (InputSpec inputSpec : inputSpecs) {
      if (!groupInputs.contains(inputSpec.getSourceVertexName())) {
        LogicalInput input = inputsMap.get(inputSpec.getSourceVertexName());
        runInputMap.put(inputSpec.getSourceVertexName(), input);
      }
    }

    for (OutputSpec outputSpec : outputSpecs) {
View Full Code Here


  }

  private LogicalInput createMergedInput(InputDescriptor inputDesc,
                                         MergedInputContext mergedInputContext,
                                         List<Input> constituentInputs) {
    LogicalInput input = ReflectionUtils.createClazzInstance(inputDesc.getClassName(),
        new Class[]{MergedInputContext.class, List.class},
        new Object[]{mergedInputContext, constituentInputs});
    if (!(input instanceof LogicalInput)) {
      throw new TezUncheckedException(inputDesc.getClass().getName()
          + " is not a sub-type of LogicalInput."
View Full Code Here

          + ", eventDestinationInfo=" + e.getDestinationInfo());
    }
    try {
      switch (e.getDestinationInfo().getEventGenerator()) {
      case INPUT:
        LogicalInput input = inputsMap.get(
            e.getDestinationInfo().getEdgeVertexName());
        if (input != null) {
          ((InputFrameworkInterface)input).handleEvents(Collections.singletonList(e.getEvent()));
        } else {
          throw new TezUncheckedException("Unhandled event for invalid target: "
View Full Code Here

    @Override
    public Void call() throws Exception {
      LOG.info("Initializing Input using InputSpec: " + inputSpec);
      String edgeName = inputSpec.getSourceVertexName();
      InputContext inputContext = createInputContext(inputsMap, inputSpec, inputIndex);
      LogicalInput input = createInput(inputSpec, inputContext);

      inputsMap.put(edgeName, input);
      inputContextMap.put(edgeName, inputContext);

      LOG.info("Initializing Input with src edge: " + edgeName);
View Full Code Here

    }
    for (LogicalOutput output : outputs.values()) {
      output.start();
    }

    LogicalInput li = inputs.values().iterator().next();
    if (! (li instanceof MRInput)) {
      throw new IllegalStateException("FilterByWordInputProcessor processor can only work with MRInput");
    }

    LogicalOutput lo = outputs.values().iterator().next();
View Full Code Here

   

    // then add the non-grouped inputs
    for (InputSpec inputSpec : inputSpecs) {
      if (!groupInputs.contains(inputSpec.getSourceVertexName())) {
        LogicalInput input = inputsMap.get(inputSpec.getSourceVertexName());
        runInputMap.put(inputSpec.getSourceVertexName(), input);
      }
    }

    for (OutputSpec outputSpec : outputSpecs) {
View Full Code Here

          + ", eventDestinationInfo=" + e.getDestinationInfo());
    }
    try {
      switch (e.getDestinationInfo().getEventGenerator()) {
      case INPUT:
        LogicalInput input = inputsMap.get(
            e.getDestinationInfo().getEdgeVertexName());
        if (input != null) {
          input.handleEvents(Collections.singletonList(e.getEvent()));
        } else {
          throw new TezUncheckedException("Unhandled event for invalid target: "
              + e);
        }
        break;
View Full Code Here

    @Override
    public Void call() throws Exception {
      LOG.info("Initializing Input using InputSpec: " + inputSpec);
      String edgeName = inputSpec.getSourceVertexName();
      LogicalInput input = createInput(inputSpec);
      TezInputContext inputContext = createInputContext(input, inputSpec, inputIndex);
      inputsMap.put(edgeName, input);
      inputContextMap.put(edgeName, inputContext);

      if (input instanceof LogicalInput) {
        ((LogicalInput) input).setNumPhysicalInputs(inputSpec
            .getPhysicalEdgeCount());
      }
      LOG.info("Initializing Input with src edge: " + edgeName);
      List<Event> events = input.initialize(inputContext);
      sendTaskGeneratedEvents(events, EventProducerConsumerType.INPUT,
          inputContext.getTaskVertexName(), inputContext.getSourceVertexName(),
          taskSpec.getTaskAttemptID());
      LOG.info("Initialized Input with src edge: " + edgeName);
      return null;
View Full Code Here

    if (inputs.size() <= 0 || inputs.size() > 1) {
      throw new IOException("Invalid number of inputs"
          + ", inputCount=" + inputs.size());
    }

    LogicalInput in = inputs.values().iterator().next();
    in.start();

    List<Input> pendingInputs = new LinkedList<Input>();
    pendingInputs.add(in);
    processorContext.waitForAllInputsReady(pendingInputs);
    LOG.info("Input is ready for consumption. Starting Output");

    LogicalOutput out = outputs.values().iterator().next();
    out.start();

    initTask(out);

    this.statusUpdate();

    Class keyClass = ConfigUtils.getIntermediateInputKeyClass(jobConf);
    Class valueClass = ConfigUtils.getIntermediateInputValueClass(jobConf);
    LOG.info("Using keyClass: " + keyClass);
    LOG.info("Using valueClass: " + valueClass);
    RawComparator comparator =
        ConfigUtils.getInputKeySecondaryGroupingComparator(jobConf);
    LOG.info("Using comparator: " + comparator);

    reduceInputKeyCounter =
        mrReporter.getCounter(TaskCounter.REDUCE_INPUT_GROUPS);
    reduceInputValueCounter =
        mrReporter.getCounter(TaskCounter.REDUCE_INPUT_RECORDS);

    // Sanity check
    if (!(in instanceof ShuffledMergedInputLegacy)) {
      throw new IOException("Illegal input to reduce: " + in.getClass());
    }
    ShuffledMergedInputLegacy shuffleInput = (ShuffledMergedInputLegacy)in;
    KeyValuesReader kvReader = shuffleInput.getReader();

    KeyValueWriter kvWriter = null;
    if((out instanceof MROutputLegacy)) {
      kvWriter = ((MROutputLegacy) out).getWriter();
    } else if ((out instanceof OnFileSortedOutput)) {
      kvWriter = ((OnFileSortedOutput) out).getWriter();
    } else {
      throw new IOException("Illegal output to reduce: " + in.getClass());
    }

    if (useNewApi) {
      try {
        runNewReducer(
View Full Code Here

        || outputs.size() != 1) {
      throw new IOException("Cannot handle multiple inputs or outputs"
          + ", inputCount=" + inputs.size()
          + ", outputCount=" + outputs.size());
    }
    LogicalInput in = inputs.values().iterator().next();
    LogicalOutput out = outputs.values().iterator().next();

    initTask(out);

    // Sanity check
    if (!(in instanceof MRInputLegacy)) {
      throw new IOException(new TezException(
          "Only MRInputLegacy supported. Input: " + in.getClass()));
    }
    MRInputLegacy input = (MRInputLegacy)in;
    input.init();
    Configuration incrementalConf = input.getConfigUpdates();
    if (incrementalConf != null) {
View Full Code Here

TOP

Related Classes of org.apache.tez.runtime.api.LogicalInput

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.