Examples of OutputSpec

org.apache.tez.runtime.api.impl.OutputSpec

Examples of org.apache.tez.runtime.api.impl.OutputSpec

        edgeManager.getNumDestinationTaskPhysicalInputs(sourceVertex.getTotalTasks(),
            destinationTaskIndex));
  }


  public OutputSpec getSourceSpec(int sourceTaskIndex) {
    return new OutputSpec(destinationVertex.getName(),
        edgeProperty.getEdgeSource(), edgeManager.getNumSourceTaskPhysicalOutputs(
        destinationVertex.getTotalTasks(), sourceTaskIndex));
  }

View Full Code Here

Examples of org.apache.tez.runtime.api.impl.OutputSpec

    
    InputSpec mapInputSpec = new InputSpec("NullSrcVertex",
        new InputDescriptor(MRInputLegacy.class.getName())
            .setUserPayload(MRHelpers.createMRInputPayload(mapConf, null)),
        0);
    OutputSpec mapOutputSpec = new OutputSpec("NullDestVertex", new OutputDescriptor(LocalOnFileSorterOutput.class.getName()), 1);
    // Run a map
    LogicalIOProcessorRuntimeTask mapTask = MapUtils.createLogicalTask(localFs, workDir, mapConf, 0,
        mapInput, new TestUmbilical(), mapVertexName,
        Collections.singletonList(mapInputSpec),
        Collections.singletonList(mapOutputSpec));


    mapTask.initialize();
    mapTask.run();
    mapTask.close();
    
    LOG.info("Starting reduce...");
    
    Token<JobTokenIdentifier> shuffleToken = new Token<JobTokenIdentifier>();
    
    Configuration reduceStageConf = MultiStageMRConfigUtil.getConfForVertex(conf,
        reduceVertexName);
    JobConf reduceConf = new JobConf(reduceStageConf);
    reduceConf.setOutputFormat(SequenceFileOutputFormat.class);
    reduceConf.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR, new Path(workDir,
        "localized-resources").toUri().toString());
    FileOutputFormat.setOutputPath(reduceConf, new Path(workDir, "output"));
    ProcessorDescriptor reduceProcessorDesc = new ProcessorDescriptor(
        ReduceProcessor.class.getName()).setUserPayload(TezUtils.createUserPayloadFromConf(reduceConf));
    
    InputSpec reduceInputSpec = new InputSpec(mapVertexName, new InputDescriptor(LocalMergedInput.class.getName()), 1);
    OutputSpec reduceOutputSpec = new OutputSpec("NullDestinationVertex", new OutputDescriptor(MROutput.class.getName()), 1);
    
    // Now run a reduce
    TaskSpec taskSpec = new TaskSpec(
        TezTestUtils.getMockTaskAttemptId(0, 1, 0, 0),
        "testUser",

View Full Code Here

Examples of org.apache.tez.runtime.api.impl.OutputSpec

    AMSchedulerEventTALaunchRequest lr = new AMSchedulerEventTALaunchRequest(
      taID, capability, new TaskSpec(taID, "user", "vertexName",
      new ProcessorDescriptor("processorClassName"),
      Collections.singletonList(new InputSpec("vertexName",
        new InputDescriptor("inputClassName"), 1)),
      Collections.singletonList(new OutputSpec("vertexName",
        new OutputDescriptor("outputClassName"), 1))), ta, hosts, racks,
      priority, containerContext);
    return lr;
  }

View Full Code Here

Examples of org.apache.tez.runtime.api.impl.OutputSpec

      this.additionalOutputs.put(
          output.getName(),
          new RootInputLeafOutputDescriptor<OutputDescriptor>(output.getName(), od,
              output.hasInitializerClassName() ? output
                  .getInitializerClassName() : null));
      OutputSpec outputSpec = new OutputSpec(output.getName(), od, 0);
      additionalOutputSpecs.add(outputSpec);
    }
  }

View Full Code Here

Examples of org.apache.tez.runtime.api.impl.OutputSpec

    if (this.outputSpecList == null) {
      outputSpecList = new ArrayList<OutputSpec>(this.getOutputVerticesCount()
          + this.additionalOutputSpecs.size());
      outputSpecList.addAll(additionalOutputSpecs);
      for (Entry<Vertex, Edge> entry : this.getOutputVertices().entrySet()) {
        OutputSpec outputSpec = entry.getValue().getSourceSpec(taskIndex);
        outputSpecList.add(outputSpec);
      }
    }
    return outputSpecList;
  }

View Full Code Here

Examples of org.apache.tez.runtime.api.impl.OutputSpec

        edgeManager.getNumDestinationTaskInputs(sourceVertex.getTotalTasks(),
            destinationTaskIndex));
  }


  public OutputSpec getSourceSpec(int sourceTaskIndex) {
    return new OutputSpec(destinationVertex.getName(),
        edgeProperty.getEdgeSource(), edgeManager.getNumSourceTaskOutputs(
            destinationVertex.getTotalTasks(), sourceTaskIndex));
  }

View Full Code Here

Examples of org.apache.tez.runtime.api.impl.OutputSpec

  }


  public OutputSpec getSourceSpec(int sourceTaskIndex) {
    Preconditions.checkState(edgeManager != null, 
        "Edge Manager must be initialized by this time");
    return new OutputSpec(destinationVertex.getName(),
        edgeProperty.getEdgeSource(), edgeManager.getNumSourceTaskPhysicalOutputs(
        sourceTaskIndex));
  }

View Full Code Here

Examples of org.apache.tez.runtime.api.impl.OutputSpec

              new RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>(
                  output.getName(), od,
                  output.hasControllerDescriptor() ? DagTypeConverters
                      .convertOutputCommitterDescriptorFromDAGPlan(output
                          .getControllerDescriptor()) : null));
      OutputSpec outputSpec = new OutputSpec(output.getName(), od, 0);
      additionalOutputSpecs.add(outputSpec);
    }
  }

View Full Code Here

Examples of org.apache.tez.runtime.api.impl.OutputSpec

    if (this.outputSpecList == null) {
      outputSpecList = new ArrayList<OutputSpec>(this.getOutputVerticesCount()
          + this.additionalOutputSpecs.size());
      outputSpecList.addAll(additionalOutputSpecs);
      for (Entry<Vertex, Edge> entry : this.getOutputVertices().entrySet()) {
        OutputSpec outputSpec = entry.getValue().getSourceSpec(taskIndex);
        outputSpecList.add(outputSpec);
      }
    }
    return outputSpecList;
  }

View Full Code Here

Examples of org.apache.tez.runtime.api.impl.OutputSpec

    
    InputSpec mapInputSpec = new InputSpec("NullSrcVertex",
        new InputDescriptor(MRInputLegacy.class.getName())
            .setUserPayload(MRHelpers.createMRInputPayload(mapConf, null)),
        0);
    OutputSpec mapOutputSpec = new OutputSpec("NullDestVertex", new OutputDescriptor(LocalOnFileSorterOutput.class.getName()), 1);
    // Run a map
    LogicalIOProcessorRuntimeTask mapTask = MapUtils.createLogicalTask(localFs, workDir, mapConf, 0,
        mapInput, new TestUmbilical(), dagName, mapVertexName,
        Collections.singletonList(mapInputSpec),
        Collections.singletonList(mapOutputSpec));


    mapTask.initialize();
    mapTask.run();
    mapTask.close();
    
    LOG.info("Starting reduce...");
    
    Token<JobTokenIdentifier> shuffleToken = new Token<JobTokenIdentifier>();
    
    Configuration reduceStageConf = MultiStageMRConfigUtil.getConfForVertex(conf,
        reduceVertexName);
    JobConf reduceConf = new JobConf(reduceStageConf);
    reduceConf.setOutputFormat(SequenceFileOutputFormat.class);
    reduceConf.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR, new Path(workDir,
        "localized-resources").toUri().toString());
    FileOutputFormat.setOutputPath(reduceConf, new Path(workDir, "output"));
    ProcessorDescriptor reduceProcessorDesc = new ProcessorDescriptor(
        ReduceProcessor.class.getName()).setUserPayload(TezUtils.createUserPayloadFromConf(reduceConf));
    
    InputSpec reduceInputSpec = new InputSpec(mapVertexName,
        new InputDescriptor(LocalMergedInput.class.getName()), 1);
    OutputSpec reduceOutputSpec = new OutputSpec("NullDestinationVertex",
        new OutputDescriptor(MROutputLegacy.class.getName()), 1);


    // Now run a reduce
    TaskSpec taskSpec = new TaskSpec(
        TezTestUtils.getMockTaskAttemptId(0, 1, 0, 0),

View Full Code Here

0 1 2

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.