Examples of org.apache.tez.runtime.api.LogicalOutput

org.apache.tez.runtime.api.LogicalOutput
An @link {@link Output} which handles all outgoing physical connections on anedge. A {@link LogicalIOProcessor} sees a single Logical Output per outgoingedge. It's expected to hide the details of output partitioning and physical outputs from the {@link Processor}Users are expected to derive from {@link AbstractLogicalOutput}

    }


    private void initializeOutputs(Map<String, LogicalOutput> outputs) throws Exception {


        for (Entry<String, LogicalOutput> entry : outputs.entrySet()) {
            LogicalOutput output = entry.getValue();
            LOG.info("Starting output " + output + " to vertex " + entry.getKey());
            output.start();
            if (output instanceof MROutput){
                MROutput mrOut = (MROutput) output;
                fileOutputs.add(mrOut);
            }
        }

View Full Code Here

      throw new IOException("Cannot handle multiple inputs or outputs"
          + ", inputCount=" + inputs.size()
          + ", outputCount=" + outputs.size());
    }
    LogicalInput in = inputs.values().iterator().next();
    LogicalOutput out = outputs.values().iterator().next();


    initTask(out);


    // Sanity check
    if (!(in instanceof MRInputLegacy)) {
      throw new IOException(new TezException(
          "Only MRInputLegacy supported. Input: " + in.getClass()));
    }
    MRInputLegacy input = (MRInputLegacy)in;
    input.init();
    Configuration incrementalConf = input.getConfigUpdates();
    if (incrementalConf != null) {
      for (Entry<String, String> entry : incrementalConf) {
        jobConf.set(entry.getKey(), entry.getValue());
      }
    }


    KeyValueWriter kvWriter = null;
    if ((out instanceof MROutputLegacy)) {
      kvWriter = ((MROutputLegacy)out).getWriter();
    } else if ((out instanceof OrderedPartitionedKVOutput)){
      kvWriter = ((OrderedPartitionedKVOutput)out).getWriter();
    } else {
      throw new IOException("Illegal output to map, outputClass="
          + out.getClass());
    }


    if (useNewApi) {
      runNewMapper(jobConf, mrReporter, input, kvWriter);
    } else {

View Full Code Here

    List<Input> pendingInputs = new LinkedList<Input>();
    pendingInputs.add(in);
    processorContext.waitForAllInputsReady(pendingInputs);
    LOG.info("Input is ready for consumption. Starting Output");


    LogicalOutput out = outputs.values().iterator().next();
    out.start();


    initTask(out);


    this.statusUpdate();

View Full Code Here

      // instance and this processor is used in many different DAGs with 
      // different names for inputs and outputs
      LogicalInput input = getInputs().values().iterator().next();
      Reader rawReader = input.getReader();
      Preconditions.checkState(rawReader instanceof KeyValueReader);
      LogicalOutput output = getOutputs().values().iterator().next();


      KeyValueReader reader = (KeyValueReader) rawReader;
      KeyValueWriter writer = (KeyValueWriter) output.getWriter();


      while (reader.next()) {
        Object val = reader.getCurrentValue();
        // The data value itself is the join key. Simply write it out as the key.
        // The output value is null.

View Full Code Here

      LogicalInput hashInput = getInputs().get(hashSide);
      Reader rawStreamReader = streamInput.getReader();
      Reader rawHashReader = hashInput.getReader();
      Preconditions.checkState(rawStreamReader instanceof KeyValueReader);
      Preconditions.checkState(rawHashReader instanceof KeyValueReader);
      LogicalOutput lo = getOutputs().get(joinOutput);
      Preconditions.checkState(lo.getWriter() instanceof KeyValueWriter);
      KeyValueWriter writer = (KeyValueWriter) lo.getWriter();


      // create a hash table for the hash side
      KeyValueReader hashKvReader = (KeyValueReader) rawHashReader;
      Set<Text> keySet = new HashSet<Text>();
      while (hashKvReader.next()) {

View Full Code Here

    LogicalInput li = inputs.values().iterator().next();
    if (! (li instanceof MRInput)) {
      throw new IllegalStateException("FilterByWordInputProcessor processor can only work with MRInput");
    }


    LogicalOutput lo = outputs.values().iterator().next();
    if (! (lo instanceof UnorderedKVOutput)) {
      throw new IllegalStateException("FilterByWordInputProcessor processor can only work with OnFileUnorderedKVOutput");
    }


    MRInputLegacy mrInput = (MRInputLegacy) li;

View Full Code Here

    LogicalInput li = inputs.values().iterator().next();
    if (! (li instanceof UnorderedKVInput)) {
      throw new IllegalStateException("FilterByWordOutputProcessor processor can only work with ShuffledUnorderedKVInput");
    }


    LogicalOutput lo = outputs.values().iterator().next();
    if (! (lo instanceof MROutput)) {
      throw new IllegalStateException("FilterByWordOutputProcessor processor can only work with MROutput");
    }


    UnorderedKVInput kvInput = (UnorderedKVInput) li;

View Full Code Here


    @Override
    public Void call() throws Exception {
      LOG.info("Initializing Output using OutputSpec: " + outputSpec);
      String edgeName = outputSpec.getDestinationVertexName();
      LogicalOutput output = createOutput(outputSpec);
      TezOutputContext outputContext = createOutputContext(outputSpec, outputIndex);
      outputsMap.put(edgeName, output);
      outputContextMap.put(edgeName, outputContext);


      if (output instanceof LogicalOutput) {
        ((LogicalOutput) output).setNumPhysicalOutputs(outputSpec
            .getPhysicalEdgeCount());
      }
      LOG.info("Initializing Output with dest edge: " + edgeName);
      List<Event> events = output.initialize(outputContext);
      sendTaskGeneratedEvents(events, EventProducerConsumerType.OUTPUT,
          outputContext.getTaskVertexName(),
          outputContext.getDestinationVertexName(), taskSpec.getTaskAttemptID());
      LOG.info("Initialized Output with dest edge: " + edgeName);
      return null;

View Full Code Here

    List<Input> pendingInputs = new LinkedList<Input>();
    pendingInputs.add(in);
    processorContext.waitForAllInputsReady(pendingInputs);
    LOG.info("Input is ready for consumption. Starting Output");


    LogicalOutput out = outputs.values().iterator().next();
    out.start();


    initTask(out);


    this.statusUpdate();

View Full Code Here

        runInputMap.put(inputSpec.getSourceVertexName(), input);
      }
    }


    for (OutputSpec outputSpec : outputSpecs) {
      LogicalOutput output = outputsMap.get(outputSpec.getDestinationVertexName());
      String outputName = outputSpec.getDestinationVertexName();
      runOutputMap.put(outputName, output);
    }
    
    // TODO Maybe close initialized inputs / outputs in case of failure to

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.tez.runtime.api.LogicalOutput

cascading.flow.tez.stream.graph.Hadoop2TezStreamGraph

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POCounterStatsTez

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POCounterTez

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POIdentityInOutTez

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POLocalRearrangeTez

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POStoreTez

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POValueOutputTez

org.apache.pig.backend.hadoop.executionengine.tez.runtime.PigProcessor

org.apache.tez.examples.JoinExample$ForwardingProcessor

org.apache.tez.examples.JoinExample$JoinProcessor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.