Examples of org.apache.tez.dag.api.Vertex.addOutput()

org.apache.tez.dag.api.Vertex.addOutput()
Specifies an Output for a Vertex. This is meant to be used when a Vertex writes Output directly to an external destination.
If an output of the vertex is meant to be consumed by another Vertex in the DAG - use the {@link DAG addEdge} method.If a vertex needs generate data to an external source as well as for another Vertex in the DAG, a combination of this API and the DAG.addEdge API can be used. @param outputName the name of the output. This will be used when accessing the output in the {@link LogicalIOProcessor} @param outputDescriptor @return

    }




    // final vertices need to have at least one output
    if (!hasChildren) {
      v.addOutput("out_"+work.getName(),
          new OutputDescriptor(MROutput.class.getName())
          .setUserPayload(MRHelpers.createUserPayloadFromConf(conf)));
    }


    return v;

View Full Code Here

    Map<String, String> reduceEnv = new HashMap<String, String>();
    MRHelpers.updateEnvironmentForMRTasks(finalReduceConf, reduceEnv, false);
    summerVertex.setTaskEnvironment(reduceEnv);
    OutputDescriptor od = new OutputDescriptor(MROutput.class.getName())
        .setUserPayload(finalReducePayload);
    summerVertex.addOutput("MROutput", od, MROutputCommitter.class);
    
    DAG dag = new DAG("WordCount");
    dag.addVertex(tokenizerVertex)
        .addVertex(summerVertex)
        .addEdge(

View Full Code Here

    Map<String, String> stage2Env = new HashMap<String, String>();
    MRHelpers.updateEnvironmentForMRTasks(stage2Conf, stage2Env, false);
    stage2Vertex.setTaskEnvironment(stage2Env);


    // Configure the Output for stage2
    stage2Vertex.addOutput("MROutput",
        new OutputDescriptor(MROutput.class.getName()).setUserPayload(MRHelpers
            .createUserPayloadFromConf(stage2Conf)));


    DAG dag = new DAG("FilterLinesByWord");
    Edge edge = new Edge(stage1Vertex, stage2Vertex, new EdgeProperty(

View Full Code Here

    Map<String, String> stage2Env = new HashMap<String, String>();
    MRHelpers.updateEnvironmentForMRTasks(stage2Conf, stage2Env, false);
    stage2Vertex.setTaskEnvironment(stage2Env);


    // Configure the Output for stage2
    stage2Vertex.addOutput("MROutput",
        new OutputDescriptor(MROutput.class.getName()).setUserPayload(MRHelpers
            .createUserPayloadFromConf(stage2Conf)));


    DAG dag = new DAG("FilterLinesByWord");
    Edge edge = new Edge(stage1Vertex, stage2Vertex, new EdgeProperty(

View Full Code Here

    stage2Vertex.setTaskEnvironment(stage2Env);


    // Configure the Output for stage2
    OutputDescriptor od = new OutputDescriptor(MROutput.class.getName())
        .setUserPayload(MRHelpers.createUserPayloadFromConf(stage2Conf));
    stage2Vertex.addOutput("MROutput", od, MROutputCommitter.class);


    DAG dag = new DAG("FilterLinesByWord");
    Edge edge = new Edge(stage1Vertex, stage2Vertex, new EdgeProperty(
        DataMovementType.BROADCAST, DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL, new OutputDescriptor(

View Full Code Here

    Map<String, String> reduceEnv = new HashMap<String, String>();
    MRHelpers.updateEnvironmentForMRTasks(finalReduceConf, reduceEnv, false);
    checkerVertex.setTaskEnvironment(reduceEnv);
    OutputDescriptor od = new OutputDescriptor(MROutput.class.getName())
      .setUserPayload(finalReducePayload);
    checkerVertex.addOutput("union", od, MROutputCommitter.class);


    Configuration partsConf = new Configuration(finalReduceConf);
    partsConf.set(FileOutputFormat.OUTDIR, outputPath+"-parts");
    byte[] partsPayload = MRHelpers.createUserPayloadFromConf(partsConf);

View Full Code Here

    allPartsConf.set(FileOutputFormat.OUTDIR, outputPath+"-all-parts");
    byte[] allPartsPayload = MRHelpers.createUserPayloadFromConf(allPartsConf);
    OutputDescriptor od2 = new OutputDescriptor(MROutput.class.getName())
      .setUserPayload(allPartsPayload);
    unionVertex.addOutput("parts", od1, MROutputCommitter.class);
    checkerVertex.addOutput("all-parts", od2, MROutputCommitter.class);
    
    
    dag.addVertex(mapVertex1)
        .addVertex(mapVertex2)
        .addVertex(mapVertex3)

View Full Code Here

    Map<String, String> reduceEnv = new HashMap<String, String>();
    MRHelpers.updateEnvironmentForMRTasks(finalReduceConf, reduceEnv, false);
    summerVertex.setTaskEnvironment(reduceEnv);
    OutputDescriptor od = new OutputDescriptor(MROutput.class.getName())
        .setUserPayload(finalReducePayload);
    summerVertex.addOutput("MROutput", od, MROutputCommitter.class);
    
    DAG dag = new DAG("WordCount");
    dag.addVertex(tokenizerVertex)
        .addVertex(summerVertex)
        .addEdge(

View Full Code Here

    Map<String, String> stage2Env = new HashMap<String, String>();
    MRHelpers.updateEnvironmentForMRTasks(stage2Conf, stage2Env, false);
    stage2Vertex.setTaskEnvironment(stage2Env);


    // Configure the Output for stage2
    stage2Vertex.addOutput("MROutput",
        new OutputDescriptor(MROutput.class.getName()).setUserPayload(MRHelpers
            .createUserPayloadFromConf(stage2Conf)),
        MROutputCommitter.class);


    DAG dag = new DAG("FilterLinesByWord");

View Full Code Here

    }




    // final vertices need to have at least one output
    if (!hasChildren) {
      v.addOutput("out_"+work.getName(),
          new OutputDescriptor(MROutput.class.getName())
          .setUserPayload(MRHelpers.createUserPayloadFromConf(conf)));
    }


    return v;

View Full Code Here

0 1

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.