Examples of DAG

hidden.org.codehaus.plexus.util.dag.DAG
org.apache.tez.dag.api.DAG
Top level entity that defines the DAG (Directed Acyclic Graph) representing the data flow graph. Consists of a set of Vertices and Edges connecting the vertices. Vertices represent transformations of data and edges represent movement of data between vertices.
org.apache.tez.dag.app.dag.DAG
Main interface to interact with the job.
org.codehaus.plexus.util.dag.DAG
DAG = Directed Acyclic Graph @author Michal Maczka @version $Id$ @todo this class should be reanmed from DAG to Dag
org.eclipse.ui.internal.texteditor.rulers.DAG
kipedia.org/wiki/Directed_acyclic_graph @since 3.3

Examples of org.apache.tez.dag.api.DAG

    
    TezClient tezClient = TezClient.create("OrderedWordCount", tezConf);
    tezClient.start();


    try {
        DAG dag = createDAG(tezConf, inputPath, outputPath, numPartitions, "OrderedWordCount");


        tezClient.waitTillReady();
        DAGClient dagClient = tezClient.submitDAG(dag);


        // monitoring

View Full Code Here

Examples of org.apache.tez.dag.api.DAG

    if (numTasks <= 0) {
      System.err.println("NumTasks must be > 0");
      return 4;
    }


    DAG dag = createDag(tezConf, largeOutPath, smallOutPath, expectedOutputPath, numTasks,
        largeOutSize, smallOutSize);


    tezClient.waitTillReady();
    DAGClient dagClient = tezClient.submitDAG(dag);
    DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(null);

View Full Code Here

Examples of org.apache.tez.dag.api.DAG

      throws IOException {


    long largeOutSizePerTask = largeOutSize / numTasks;
    long smallOutSizePerTask = smallOutSize / numTasks;


    DAG dag = DAG.create("JoinDataGen");


    Vertex genDataVertex = Vertex.create("datagen", ProcessorDescriptor.create(
        GenDataProcessor.class.getName()).setUserPayload(
        UserPayload.create(ByteBuffer.wrap(GenDataProcessor.createConfiguration(largeOutSizePerTask,
            smallOutSizePerTask)))), numTasks);
    genDataVertex.addDataSink(STREAM_OUTPUT_NAME, 
        MROutput.createConfigBuilder(new Configuration(tezConf),
            TextOutputFormat.class, largeOutPath.toUri().toString()).build());
    genDataVertex.addDataSink(HASH_OUTPUT_NAME, 
        MROutput.createConfigBuilder(new Configuration(tezConf),
            TextOutputFormat.class, smallOutPath.toUri().toString()).build());
    genDataVertex.addDataSink(EXPECTED_OUTPUT_NAME, 
        MROutput.createConfigBuilder(new Configuration(tezConf),
            TextOutputFormat.class, expectedOutputPath.toUri().toString()).build());


    dag.addVertex(genDataVertex);


    return dag;
  }

View Full Code Here

Examples of org.apache.tez.dag.api.DAG

    tezConf1.set("fs.defaultFS", "file:///");
    tezConf1.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
    TezClient tezClient1 = TezClient.create("commonName", tezConf1, true);
    tezClient1.start();


    DAG dag1 = createSimpleDAG("dag1", SleepProcessor.class.getName());


    DAGClient dagClient1 = tezClient1.submitDAG(dag1);
    dagClient1.waitForCompletion();
    assertEquals(DAGStatus.State.SUCCEEDED, dagClient1.getDAGStatus(null).getState());


    dagClient1.close();
    tezClient1.stop();




    TezConfiguration tezConf2 = new TezConfiguration();
    tezConf2.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
    tezConf2.set("fs.defaultFS", "file:///");
    tezConf2.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
    DAG dag2 = createSimpleDAG("dag2", SleepProcessor.class.getName());
    TezClient tezClient2 = TezClient.create("commonName", tezConf2, true);
    tezClient2.start();
    DAGClient dagClient2 = tezClient2.submitDAG(dag2);
    dagClient2.waitForCompletion();
    assertEquals(DAGStatus.State.SUCCEEDED, dagClient2.getDAGStatus(null).getState());

View Full Code Here

Examples of org.apache.tez.dag.api.DAG

    tezConf1.set("fs.defaultFS", "file:///");
    tezConf1.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
    TezClient tezClient1 = TezClient.create("commonName", tezConf1, false);
    tezClient1.start();


    DAG dag1 = createSimpleDAG("dag1", SleepProcessor.class.getName());


    DAGClient dagClient1 = tezClient1.submitDAG(dag1);
    dagClient1.waitForCompletion();
    assertEquals(DAGStatus.State.SUCCEEDED, dagClient1.getDAGStatus(null).getState());


    dagClient1.close();
    tezClient1.stop();




    TezConfiguration tezConf2 = new TezConfiguration();
    tezConf2.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
    tezConf2.set("fs.defaultFS", "file:///");
    tezConf2.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
    DAG dag2 = createSimpleDAG("dag2", SleepProcessor.class.getName());
    TezClient tezClient2 = TezClient.create("commonName", tezConf2, false);
    tezClient2.start();
    DAGClient dagClient2 = tezClient2.submitDAG(dag2);
    dagClient2.waitForCompletion();
    assertEquals(DAGStatus.State.SUCCEEDED, dagClient2.getDAGStatus(null).getState());

View Full Code Here

Examples of org.apache.tez.dag.api.DAG

    tezConf1.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
    // Run in non-session mode so that the AM terminates
    TezClient tezClient1 = TezClient.create("commonName", tezConf1, false);
    tezClient1.start();


    DAG dag1 = createSimpleDAG("dag1", SleepProcessor.class.getName());


    DAGClient dagClient1 = tezClient1.submitDAG(dag1);
    dagClient1.waitForCompletion();
    assertEquals(DAGStatus.State.SUCCEEDED, dagClient1.getDAGStatus(null).getState());

View Full Code Here

Examples of org.apache.tez.dag.api.DAG

    tezConf1.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
    // Run in non-session mode so that the AM terminates
    TezClient tezClient1 = TezClient.create("commonName", tezConf1, false);
    tezClient1.start();


    DAG dag1 = createSimpleDAG("dag1", FailingProcessor.class.getName());


    DAGClient dagClient1 = tezClient1.submitDAG(dag1);
    dagClient1.waitForCompletion();
    assertEquals(DAGStatus.State.FAILED, dagClient1.getDAGStatus(null).getState());

View Full Code Here

Examples of org.apache.tez.dag.api.DAG


    }
  }


  private DAG createSimpleDAG(String dagName, String processorName) {
    DAG dag = DAG.create(dagName).addVertex(Vertex.create("Sleep", ProcessorDescriptor.create(
        processorName).setUserPayload(
        new SleepProcessor.SleepProcessorConfig(1).toUserPayload()), 1));
    return dag;


  }

View Full Code Here

Examples of org.apache.tez.dag.api.DAG

    Assert.assertEquals(finalState, dagStatus.getState());
  }
  
  @Test (timeout=60000)
  public void testBasicSuccessScatterGather() throws Exception {
    DAG dag = SimpleTestDAG.createDAG("testBasicSuccessScatterGather", null);
    runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
  }

View Full Code Here

Examples of org.apache.tez.dag.api.DAG

    runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
  }
  
  @Test (timeout=60000)
  public void testBasicSuccessBroadcast() throws Exception {
    DAG dag = DAG.create("testBasicSuccessBroadcast");
    Vertex v1 =
        Vertex.create("v1", TestProcessor.getProcDesc(null), 2, SimpleTestDAG.defaultResource);
    Vertex v2 =
        Vertex.create("v2", TestProcessor.getProcDesc(null), 2, SimpleTestDAG.defaultResource);
    dag.addVertex(v1).addVertex(v2).addEdge(Edge.create(v1, v2,
        EdgeProperty.create(DataMovementType.BROADCAST,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            TestOutput.getOutputDesc(null),
            TestInput.getInputDesc(null))));

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.