Examples of DAGPlan

org.apache.tez.dag.api.records.DAGProtos.DAGPlan

Examples of org.apache.tez.dag.api.records.DAGProtos.DAGPlan


  @Override
  public SubmitDAGResponseProto submitDAG(RpcController controller,
      SubmitDAGRequestProto request) throws ServiceException {
    try{
      DAGPlan dagPlan = request.getDAGPlan();
      String dagId = real.submitDAG(dagPlan);
      return SubmitDAGResponseProto.newBuilder().setDagId(dagId).build();
    } catch(TezException e) {
      throw wrapException(e);
    }

View Full Code Here

Examples of org.apache.tez.dag.api.records.DAGProtos.DAGPlan

      Path binaryPath =  new Path(amConfig.getStagingDir(),
          TezConfiguration.TEZ_PB_PLAN_BINARY_NAME + "." + appId.toString());
      amConfig.getAMConf().set(TezConfiguration.TEZ_AM_PLAN_REMOTE_PATH,
          binaryPath.toUri().toString());


      DAGPlan dagPB = dag.createDag(null);


      FSDataOutputStream dagPBOutBinaryStream = null;


      try {
        //binary output
        dagPBOutBinaryStream = FileSystem.create(fs, binaryPath,
            new FsPermission(TEZ_AM_FILE_PERMISSION));
        dagPB.writeTo(dagPBOutBinaryStream);
      } finally {
        if(dagPBOutBinaryStream != null){
          dagPBOutBinaryStream.close();
        }
      }

View Full Code Here

Examples of org.apache.tez.dag.api.records.DAGProtos.DAGPlan

        .setGroupsWithViewACLs(Arrays.asList("g1"))
        .setGroupsWithModifyACLs(Arrays.asList("g2"));
    dag.setAccessControls(dagAccessControls);


    Configuration conf = new Configuration(false);
    DAGPlan dagPlan = dag.createDag(conf);
    Assert.assertNull(conf.get(TezConstants.TEZ_DAG_VIEW_ACLS));
    Assert.assertNull(conf.get(TezConstants.TEZ_DAG_MODIFY_ACLS));


    ConfigurationProto confProto = dagPlan.getDagKeyValues();
    boolean foundViewAcls = false;
    boolean foundModifyAcls = false;


    for (PlanKeyValuePair pair : confProto.getConfKeyValuesList()) {
      if (pair.getKey().equals(TezConstants.TEZ_DAG_VIEW_ACLS)) {

View Full Code Here

Examples of org.apache.tez.dag.api.records.DAGProtos.DAGPlan

  public TemporaryFolder tempFolder = new TemporaryFolder(); //TODO: doesn't seem to be deleting this folder automatically as expected.


  @Test(timeout = 5000)
  public void testBasicJobPlanSerde() throws IOException {


    DAGPlan job = DAGPlan.newBuilder()
       .setName("test")
       .addVertex(
           VertexPlan.newBuilder()
             .setName("vertex1")
             .setType(PlanVertexType.NORMAL)
             .addTaskLocationHint(PlanTaskLocationHint.newBuilder().addHost("machineName").addRack("rack1").build())
             .setTaskConfig(
                 PlanTaskConfiguration.newBuilder()
                   .setNumTasks(2)
                   .setVirtualCores(4)
                   .setMemoryMb(1024)
                   .setJavaOpts("")
                   .setTaskModule("x.y")
                   .build())
             .build())
        .build();
   File file = tempFolder.newFile("jobPlan");
   FileOutputStream outStream = null;
   try {
     outStream = new FileOutputStream(file);
     job.writeTo(outStream);
   }
   finally {
     if(outStream != null){
       outStream.close();
     }
   }


   DAGPlan inJob;
   FileInputStream inputStream;
   try {
     inputStream = new FileInputStream(file);
     inJob = DAGPlan.newBuilder().mergeFrom(inputStream).build();
   }

View Full Code Here

Examples of org.apache.tez.dag.api.records.DAGProtos.DAGPlan

            UserPayload.create(ByteBuffer.wrap("emPayload".getBytes()))),
        DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));


    dag.addVertex(v1).addVertex(v2).addEdge(edge);


    DAGPlan dagProto = dag.createDag(new TezConfiguration());


    EdgeProperty edgeProperty = DagTypeConverters.createEdgePropertyMapFromDAGPlan(dagProto
        .getEdgeList().get(0));


    EdgeManagerPluginDescriptor emDesc = edgeProperty.getEdgeManagerDescriptor();
    Assert.assertNotNull(emDesc);
    Assert.assertEquals("emClass", emDesc.getClassName());

View Full Code Here

Examples of org.apache.tez.dag.api.records.DAGProtos.DAGPlan

        DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));


    dag.addVertex(v1).addVertex(v2).addEdge(edge);


    DAGPlan dagProto = dag.createDag(new TezConfiguration());


    assertEquals(2, dagProto.getVertexCount());
    assertEquals(1, dagProto.getEdgeCount());


    VertexPlan v1Proto = dagProto.getVertex(0);
    VertexPlan v2Proto = dagProto.getVertex(1);
    EdgePlan edgeProto = dagProto.getEdge(0);


    assertEquals("processor1Bytes", new String(v1Proto.getProcessorDescriptor()
        .getUserPayload().toByteArray()));
    assertEquals("processor1", v1Proto.getProcessorDescriptor().getClassName());


    assertEquals("processor2Bytes", new String(v2Proto.getProcessorDescriptor()
        .getUserPayload().toByteArray()));
    assertEquals("processor2", v2Proto.getProcessorDescriptor().getClassName());


    assertEquals("inputBytes", new String(edgeProto.getEdgeDestination()
        .getUserPayload().toByteArray()));
    assertEquals("input", edgeProto.getEdgeDestination().getClassName());


    assertEquals("outputBytes", new String(edgeProto.getEdgeSource()
        .getUserPayload().toByteArray()));
    assertEquals("output", edgeProto.getEdgeSource().getClassName());


    EdgeProperty edgeProperty = DagTypeConverters
        .createEdgePropertyMapFromDAGPlan(dagProto.getEdgeList().get(0));


    byte[] ib = edgeProperty.getEdgeDestination().getUserPayload().deepCopyAsArray();
    assertEquals("inputBytes", new String(ib));
    assertEquals("input", edgeProperty.getEdgeDestination().getClassName());

View Full Code Here

Examples of org.apache.tez.dag.api.records.DAGProtos.DAGPlan

        DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));


    dag.addVertex(v1).addVertex(v2).addEdge(edge).addVertex(v3);


    DAGPlan dagProto = dag.createDag(new TezConfiguration());


    assertEquals(3, dagProto.getVertexCount());
    assertEquals(1, dagProto.getEdgeCount());


    VertexPlan v1Proto = dagProto.getVertex(0);
    VertexPlan v2Proto = dagProto.getVertex(1);
    VertexPlan v3Proto = dagProto.getVertex(2);
    EdgePlan edgeProto = dagProto.getEdge(0);


    // either v1 or v2 will be on top based on topological order 
    String v1ProtoPayload = new String(v1Proto.getProcessorDescriptor().getUserPayload().toByteArray());
    String v2ProtoPayload = new String(v2Proto.getProcessorDescriptor().getUserPayload().toByteArray());
    assertTrue(v1ProtoPayload.equals("processor1Bytes") || v1ProtoPayload.equals("processor3Bytes"));
    assertTrue(v2ProtoPayload.equals("processor1Bytes") || v2ProtoPayload.equals("processor3Bytes"));
    assertTrue(v1Proto.getProcessorDescriptor().getClassName().equals("processor1") ||
        v1Proto.getProcessorDescriptor().getClassName().equals("processor3"));
    assertTrue(v2Proto.getProcessorDescriptor().getClassName().equals("processor1") ||
        v2Proto.getProcessorDescriptor().getClassName().equals("processor3"));


    assertEquals("processor2Bytes", new String(v3Proto.getProcessorDescriptor()
        .getUserPayload().toByteArray()));
    assertEquals("processor2", v3Proto.getProcessorDescriptor().getClassName());


    assertEquals("inputBytes", new String(edgeProto.getEdgeDestination()
        .getUserPayload().toByteArray()));
    assertEquals("input", edgeProto.getEdgeDestination().getClassName());


    assertEquals("outputBytes", new String(edgeProto.getEdgeSource()
        .getUserPayload().toByteArray()));
    assertEquals("output", edgeProto.getEdgeSource().getClassName());


    EdgeProperty edgeProperty = DagTypeConverters
        .createEdgePropertyMapFromDAGPlan(dagProto.getEdgeList().get(0));


    byte[] ib = edgeProperty.getEdgeDestination().getUserPayload().deepCopyAsArray();
    assertEquals("inputBytes", new String(ib));
    assertEquals("input", edgeProperty.getEdgeDestination().getClassName());

View Full Code Here

Examples of org.apache.tez.dag.api.records.DAGProtos.DAGPlan

    dagCredentials.addToken(new Text("Token1"), token1);
    dagCredentials.addToken(new Text("Token2"), token2);
    
    dag.setCredentials(dagCredentials);


    DAGPlan dagProto = dag.createDag(new TezConfiguration());


    assertTrue(dagProto.hasCredentialsBinary());
    
    Credentials fetchedCredentials = DagTypeConverters.convertByteStringToCredentials(dagProto
        .getCredentialsBinary());
    
    assertEquals(2, fetchedCredentials.numberOfTokens());
    assertNotNull(fetchedCredentials.getToken(new Text("Token1")));
    assertNotNull(fetchedCredentials.getToken(new Text("Token2")));

View Full Code Here

Examples of org.apache.tez.dag.api.records.DAGProtos.DAGPlan

  }


  @Test
  @SuppressWarnings("unchecked")
  public void testConvertDAGPlanToATSMap() throws IOException, JSONException {
    DAGPlan dagPlan = createDAG();
    Map<String, Object> atsMap = DAGUtils.convertDAGPlanToATSMap(dagPlan);
    Assert.assertTrue(atsMap.containsKey(DAGUtils.DAG_NAME_KEY));
    Assert.assertEquals(dagPlan.getName(), atsMap.get(DAGUtils.DAG_NAME_KEY));
    Assert.assertTrue(atsMap.containsKey("version"));
    Assert.assertEquals(1, atsMap.get("version"));
    Assert.assertTrue(atsMap.containsKey(DAGUtils.VERTICES_KEY));
    Assert.assertTrue(atsMap.containsKey(DAGUtils.EDGES_KEY));
    Assert.assertTrue(atsMap.containsKey(DAGUtils.VERTEX_GROUPS_KEY));

View Full Code Here

Examples of org.apache.tez.dag.api.records.DAGProtos.DAGPlan

    }
  }


  private DAGPlan createTestMRRDAGPlan() {
    LOG.info("Setting up MRR dag plan");
    DAGPlan dag = DAGPlan.newBuilder()
        .setName("testverteximpl")
        .addVertex(
            VertexPlan.newBuilder()
            .setName("vertex1")
            .setType(PlanVertexType.NORMAL)

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.