Package org.apache.tez.dag.api

Examples of org.apache.tez.dag.api.DAG

428
429
430
431
432
433
434
435
436
437
            TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 5);
    // Work-around till TEZ-877 gets fixed
    //testConf.setInt(TestProcessor.getVertexConfName(
    //        TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 4);
 
    DAG dag = SimpleTestDAG.createDAG(
            "testInputFailureCausesRerunAttemptWithinMaxAttemptSuccess", testConf);
    //Job should succeed.
    runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
  }
View Full Code Here
499
500
501
502
503
504
505
506
507
    testConf.setInt(TestProcessor.getVertexConfName(
            TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 9);
    testConf.setInt(TestProcessor.getVertexConfName(
            TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 1), 9);
   
    DAG dag = SimpleTestDAG3Vertices.createDAG(
              "testCascadingInputFailureWithoutExitSuccess", testConf);
    runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
  }
View Full Code Here
529
530
531
532
533
534
535
536
537
    testConf.setInt(TestProcessor.getVertexConfName(
            TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 11);
    testConf.setInt(TestProcessor.getVertexConfName(
            TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 1), 11);
   
    DAG dag = SimpleTestDAG3Vertices.createDAG(
              "testCascadingInputFailureWithExitSuccess", testConf);
    runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
  }
View Full Code Here
566
567
568
569
570
571
572
573
574
    testConf.set(TestProcessor.getVertexConfName(
            TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), "0");
    testConf.setInt(TestProcessor.getVertexConfName(
            TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 13);
   
    DAG dag = SimpleVTestDAG.createDAG(
            "testInputFailureCausesRerunOfTwoVerticesWithoutExit", testConf);
    runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
  }
View Full Code Here
600
601
602
603
604
605
606
607
608
    testConf.setInt(TestProcessor.getVertexConfName(
            TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 7);
    testConf.setInt(TestProcessor.getVertexConfName(
            TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 1), 7);
   
    DAG dag = SimpleVTestDAG.createDAG(
            "testAttemptOfDownstreamVertexConnectedWithTwoUpstreamVerticesFailure", testConf);
    runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
  }
View Full Code Here
656
657
658
659
660
661
662
663
664
    testConf.set(TestProcessor.getVertexConfName(
            TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), "0");
    testConf.setInt(TestProcessor.getVertexConfName(
            TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 5);
       
    DAG dag = SimpleReverseVTestDAG.createDAG(
            "testInputFailureRerunCanSendOutputToTwoDownstreamVertices", testConf);
    runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
  }
View Full Code Here
699
700
701
702
703
704
705
706
            TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 5);
    //similarly for v2 task1
    testConf.setInt(TestProcessor.getVertexConfName(
            TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 5);
   
    DAG dag = SimpleTestDAG.createDAG("testTwoTasksHaveInputFailuresSuccess", testConf);
    runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
  }
View Full Code Here
148
149
150
151
152
153
154
155
156
157
158
    Map<String, LocalResource> lrDAG = Collections.singletonMap(mockLR1Name, LocalResource
        .newInstance(URL.newInstance("file:///", "localhost", 0, "test"), LocalResourceType.FILE,
            LocalResourceVisibility.PUBLIC, 1, 1));
    Vertex vertex = Vertex.create("Vertex", ProcessorDescriptor.create("P"), 1,
        Resource.newInstance(1, 1));
    DAG dag = DAG.create("DAG").addVertex(vertex).addTaskLocalFiles(lrDAG);
    DAGClient dagClient = client.submitDAG(dag);
   
    // verify that both DAG and TezClient localResources are added to the vertex
    Map<String, LocalResource> vertexLR = vertex.getTaskLocalFiles();
    Assert.assertTrue(vertexLR.containsKey(mockLR1Name));
View Full Code Here
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
    finalReduceVertex.addDataSink("MROutput",
        MROutputLegacy.createConfigBuilder(finalReduceConf, TextOutputFormat.class, outputPath)
            .build());
    vertices.add(finalReduceVertex);

    DAG dag = DAG.create("OrderedWordCount" + dagIndex);
    for (int i = 0; i < vertices.size(); ++i) {
      dag.addVertex(vertices.get(i));
    }

    OrderedPartitionedKVEdgeConfig edgeConf1 = OrderedPartitionedKVEdgeConfig
        .newBuilder(Text.class.getName(), IntWritable.class.getName(),
            HashPartitioner.class.getName()).setFromConfiguration(conf)
      .configureInput().useLegacyInput().done().build();
    dag.addEdge(
        Edge.create(dag.getVertex("initialmap"), dag.getVertex("intermediate_reducer"),
            edgeConf1.createDefaultEdgeProperty()));

    OrderedPartitionedKVEdgeConfig edgeConf2 = OrderedPartitionedKVEdgeConfig
        .newBuilder(IntWritable.class.getName(), Text.class.getName(),
            HashPartitioner.class.getName()).setFromConfiguration(conf)
            .configureInput().useLegacyInput().done().build();
    dag.addEdge(
        Edge.create(dag.getVertex("intermediate_reducer"), dag.getVertex("finalreduce"),
            edgeConf2.createDefaultEdgeProperty()));

    return dag;
  }
View Full Code Here
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
            + ", outputPath=" + outputPath);

        Map<String, LocalResource> localResources =
          new TreeMap<String, LocalResource>();
       
        DAG dag = instance.createDAG(fs, conf, localResources,
            stagingDir, dagIndex, inputPath, outputPath,
            generateSplitsInClient);

        boolean doPreWarm = dagIndex == 1 && useTezSession
            && conf.getBoolean("PRE_WARM_SESSION", true);
        int preWarmNumContainers = 0;
        if (doPreWarm) {
          preWarmNumContainers = conf.getInt("PRE_WARM_NUM_CONTAINERS", 0);
          if (preWarmNumContainers <= 0) {
            doPreWarm = false;
          }
        }
        if (doPreWarm) {
          LOG.info("Pre-warming Session");
          PreWarmVertex preWarmVertex = PreWarmVertex.create("PreWarm", preWarmNumContainers, dag
              .getVertex("initialmap").getTaskResource());
          preWarmVertex.addTaskLocalFiles(dag.getVertex("initialmap").getTaskLocalFiles());
          preWarmVertex.setTaskEnvironment(dag.getVertex("initialmap").getTaskEnvironment());
          preWarmVertex.setTaskLaunchCmdOpts(dag.getVertex("initialmap").getTaskLaunchCmdOpts());
         
          tezSession.preWarm(preWarmVertex);
        }

        if (useTezSession) {
View Full Code Here
TOP

Related Classes of org.apache.tez.dag.api.DAG

  • cascading.flow.tez.Hadoop2TezFlowStep
  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.hive.ql.exec.tez.TestTezTask
  • org.apache.hadoop.hive.ql.exec.tez.TezTask
  • org.apache.hadoop.yarn.api.records.LocalResource
  • org.apache.hadoop.yarn.api.records.Resource
  • org.apache.pig.backend.hadoop.executionengine.tez.TezJobCompiler
  • org.apache.pig.backend.hadoop.executionengine.tez.TezLauncher$ProgressReporter
  • org.apache.pig.tez.TestGroupConstParallelTez
  • org.apache.pig.tez.TestJobSubmissionTez
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.