Examples of org.apache.tez.runtime.api.LogicalOutput

org.apache.tez.runtime.api.LogicalOutput
An @link {@link Output} which handles all outgoing physical connections on anedge. A {@link LogicalIOProcessor} sees a single Logical Output per outgoingedge. It's expected to hide the details of output partitioning and physical outputs from the {@link Processor}Users are expected to derive from {@link AbstractLogicalOutput}

          throw new TezUncheckedException("Unhandled event for invalid target: "
              + e);
        }
        break;
      case OUTPUT:
        LogicalOutput output = outputsMap.get(
            e.getDestinationInfo().getEdgeVertexName());
        if (output != null) {
          ((OutputFrameworkInterface)output).handleEvents(Collections.singletonList(e.getEvent()));
        } else {
          throw new TezUncheckedException("Unhandled event for invalid target: "

View Full Code Here

    @Override
    public Void call() throws Exception {
      LOG.info("Initializing Output using OutputSpec: " + outputSpec);
      String edgeName = outputSpec.getDestinationVertexName();
      OutputContext outputContext = createOutputContext(outputSpec, outputIndex);
      LogicalOutput output = createOutput(outputSpec, outputContext);


      outputsMap.put(edgeName, output);
      outputContextMap.put(edgeName, outputContext);


      LOG.info("Initializing Output with dest edge: " + edgeName);

View Full Code Here

    LogicalInput li = inputs.values().iterator().next();
    if (! (li instanceof MRInput)) {
      throw new IllegalStateException("FilterByWordInputProcessor processor can only work with MRInput");
    }


    LogicalOutput lo = outputs.values().iterator().next();
    if (! (lo instanceof OnFileUnorderedKVOutput)) {
      throw new IllegalStateException("FilterByWordInputProcessor processor can only work with OnFileUnorderedKVOutput");
    }

View Full Code Here

        runInputMap.put(inputSpec.getSourceVertexName(), input);
      }
    }


    for (OutputSpec outputSpec : outputSpecs) {
      LogicalOutput output = outputsMap.get(outputSpec.getDestinationVertexName());
      String outputName = outputSpec.getDestinationVertexName();
      runOutputMap.put(outputName, output);
    }
    
    // TODO Maybe close initialized inputs / outputs in case of failure to

View Full Code Here

          throw new TezUncheckedException("Unhandled event for invalid target: "
              + e);
        }
        break;
      case OUTPUT:
        LogicalOutput output = outputsMap.get(
            e.getDestinationInfo().getEdgeVertexName());
        if (output != null) {
          output.handleEvents(Collections.singletonList(e.getEvent()));
        } else {
          throw new TezUncheckedException("Unhandled event for invalid target: "
              + e);
        }
        break;

View Full Code Here


    @Override
    public Void call() throws Exception {
      LOG.info("Initializing Output using OutputSpec: " + outputSpec);
      String edgeName = outputSpec.getDestinationVertexName();
      LogicalOutput output = createOutput(outputSpec);
      TezOutputContext outputContext = createOutputContext(outputSpec, outputIndex);
      outputsMap.put(edgeName, output);
      outputContextMap.put(edgeName, outputContext);


      if (output instanceof LogicalOutput) {
        ((LogicalOutput) output).setNumPhysicalOutputs(outputSpec
            .getPhysicalEdgeCount());
      }
      LOG.info("Initializing Output with dest edge: " + edgeName);
      List<Event> events = output.initialize(outputContext);
      sendTaskGeneratedEvents(events, EventProducerConsumerType.OUTPUT,
          outputContext.getTaskVertexName(),
          outputContext.getDestinationVertexName(), taskSpec.getTaskAttemptID());
      LOG.info("Initialized Output with dest edge: " + edgeName);
      return null;

View Full Code Here

    List<Input> pendingInputs = new LinkedList<Input>();
    pendingInputs.add(in);
    processorContext.waitForAllInputsReady(pendingInputs);
    LOG.info("Input is ready for consumption. Starting Output");


    LogicalOutput out = outputs.values().iterator().next();
    out.start();


    initTask(out);


    this.statusUpdate();

View Full Code Here

      throw new IOException("Cannot handle multiple inputs or outputs"
          + ", inputCount=" + inputs.size()
          + ", outputCount=" + outputs.size());
    }
    LogicalInput in = inputs.values().iterator().next();
    LogicalOutput out = outputs.values().iterator().next();


    initTask(out);


    // Sanity check
    if (!(in instanceof MRInputLegacy)) {
      throw new IOException(new TezException(
          "Only MRInputLegacy supported. Input: " + in.getClass()));
    }
    MRInputLegacy input = (MRInputLegacy)in;
    input.init();
    Configuration incrementalConf = input.getConfigUpdates();
    if (incrementalConf != null) {
      for (Entry<String, String> entry : incrementalConf) {
        jobConf.set(entry.getKey(), entry.getValue());
      }
    }


    KeyValueWriter kvWriter = null;
    if ((out instanceof MROutputLegacy)) {
      kvWriter = ((MROutputLegacy)out).getWriter();
    } else if ((out instanceof OnFileSortedOutput)){
      kvWriter = ((OnFileSortedOutput)out).getWriter();
    } else {
      throw new IOException("Illegal output to map, outputClass="
          + out.getClass());
    }


    if (useNewApi) {
      runNewMapper(jobConf, mrReporter, input, kvWriter);
    } else {

View Full Code Here

    LogicalInput li = inputs.values().iterator().next();
    if (! (li instanceof ShuffledUnorderedKVInput)) {
      throw new IllegalStateException("FilterByWordOutputProcessor processor can only work with ShuffledUnorderedKVInput");
    }


    LogicalOutput lo = outputs.values().iterator().next();
    if (! (lo instanceof MROutput)) {
      throw new IllegalStateException("FilterByWordOutputProcessor processor can only work with MROutput");
    }


    ShuffledUnorderedKVInput kvInput = (ShuffledUnorderedKVInput) li;

View Full Code Here

    @Override
    public void attachOutputs(Map<String, LogicalOutput> outputs,
            Configuration conf) throws ExecException {
        writers = new ArrayList<KeyValueWriter>();
        for (String outputKey : outputKeys) {
            LogicalOutput output = outputs.get(outputKey);
            if (output == null) {
                throw new ExecException("Output to vertex " + outputKey
                        + " is missing");
            }
            try {
                KeyValueWriter writer = (KeyValueWriter) output.getWriter();
                writers.add(writer);
                LOG.info("Attached output to vertex " + outputKey + " : output=" + output + ", writer=" + writer);
            } catch (Exception e) {
                throw new ExecException(e);
            }

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.tez.runtime.api.LogicalOutput

cascading.flow.tez.stream.graph.Hadoop2TezStreamGraph

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POCounterStatsTez

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POCounterTez

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POIdentityInOutTez

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POLocalRearrangeTez

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POStoreTez

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POValueOutputTez

org.apache.pig.backend.hadoop.executionengine.tez.runtime.PigProcessor

org.apache.tez.examples.JoinExample$ForwardingProcessor

org.apache.tez.examples.JoinExample$JoinProcessor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.