Examples of EdgeManagerDescriptor

org.apache.tez.dag.api.EdgeManagerDescriptor

Examples of org.apache.tez.dag.api.EdgeManagerDescriptor

        taskCount++;
      }
    }


    // Construct the EdgeManager descriptor to be used by all edges which need the routing table.
    EdgeManagerDescriptor hiveEdgeManagerDesc = new EdgeManagerDescriptor(
        CustomPartitionEdge.class.getName());    
    byte[] payload = getBytePayload(bucketToTaskMap);
    hiveEdgeManagerDesc.setUserPayload(payload);


    Map<String, EdgeManagerDescriptor> emMap = Maps.newHashMap();


    // Replace the edge manager for all vertices which have routing type custom.
    for (Entry<String, EdgeProperty> edgeEntry : context.getInputVertexEdgeProperties().entrySet()) {

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeManagerDescriptor

      case CUSTOM_EDGE:


        dataMovementType = DataMovementType.CUSTOM;
        logicalOutputClass = OnFileUnorderedPartitionedKVOutput.class;
        logicalInputClass = ShuffledUnorderedKVInput.class;
        EdgeManagerDescriptor edgeDesc = new EdgeManagerDescriptor(
            CustomPartitionEdge.class.getName());
        CustomEdgeConfiguration edgeConf =
            new CustomEdgeConfiguration(edgeProp.getNumBuckets(), null);
          DataOutputBuffer dob = new DataOutputBuffer();
          edgeConf.write(dob);
          byte[] userPayload = dob.getData();
        edgeDesc.setUserPayload(userPayload);
        edgeProperty =
          new EdgeProperty(edgeDesc,
              DataSourceType.PERSISTED,
              SchedulingType.SEQUENTIAL,
              new OutputDescriptor(logicalOutputClass.getName()),

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeManagerDescriptor

        taskCount++;
      }
    }


    // Construct the EdgeManager descriptor to be used by all edges which need the routing table.
    EdgeManagerDescriptor hiveEdgeManagerDesc = new EdgeManagerDescriptor(
        CustomPartitionEdge.class.getName());    
    byte[] payload = getBytePayload(bucketToTaskMap);
    hiveEdgeManagerDesc.setUserPayload(payload);


    Map<String, EdgeManagerDescriptor> emMap = Maps.newHashMap();


    // Replace the edge manager for all vertices which have routing type custom.
    for (Entry<String, EdgeProperty> edgeEntry : context.getInputVertexEdgeProperties().entrySet()) {

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeManagerDescriptor

    if (proto.getEdgeManagerDescriptorsCount() > 0) {
      this.sourceEdgeManagers = new HashMap<String, EdgeManagerDescriptor>(
          proto.getEdgeManagerDescriptorsCount());
      for (EdgeManagerDescriptorProto edgeManagerProto :
        proto.getEdgeManagerDescriptorsList()) {
        EdgeManagerDescriptor edgeManagerDescriptor =
            DagTypeConverters.convertEdgeManagerDescriptorFromDAGPlan(
                edgeManagerProto.getEntityDescriptor());
        sourceEdgeManagers.put(edgeManagerProto.getEdgeName(),
            edgeManagerDescriptor);
      }

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeManagerDescriptor

      case CUSTOM_EDGE:


        dataMovementType = DataMovementType.CUSTOM;
        logicalOutputClass = OnFileUnorderedPartitionedKVOutput.class;
        logicalInputClass = ShuffledUnorderedKVInput.class;
        EdgeManagerDescriptor edgeDesc = new EdgeManagerDescriptor(
            CustomPartitionEdge.class.getName());
        CustomEdgeConfiguration edgeConf =
            new CustomEdgeConfiguration(edgeProp.getNumBuckets(), null);
          DataOutputBuffer dob = new DataOutputBuffer();
          edgeConf.write(dob);
          byte[] userPayload = dob.getData();
        edgeDesc.setUserPayload(userPayload);
        edgeProperty =
          new EdgeProperty(edgeDesc,
              DataSourceType.PERSISTED,
              SchedulingType.SEQUENTIAL,
              new OutputDescriptor(logicalOutputClass.getName()),

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeManagerDescriptor

      case CUSTOM_EDGE:
        
        dataMovementType = DataMovementType.CUSTOM;
        logicalOutputClass = OnFileUnorderedPartitionedKVOutput.class;
        logicalInputClass = ShuffledUnorderedKVInput.class;
        EdgeManagerDescriptor edgeDesc = new EdgeManagerDescriptor(
            CustomPartitionEdge.class.getName());
        CustomEdgeConfiguration edgeConf = 
            new CustomEdgeConfiguration(edgeProp.getNumBuckets(), null);
          DataOutputBuffer dob = new DataOutputBuffer();
          edgeConf.write(dob);
          byte[] userPayload = dob.getData();
        edgeDesc.setUserPayload(userPayload);
        edgeProperty =
          new EdgeProperty(edgeDesc,
              DataSourceType.PERSISTED,
              SchedulingType.SEQUENTIAL,
              new OutputDescriptor(logicalOutputClass.getName()),

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeManagerDescriptor

        taskCount++;
      }
    }


    // Construct the EdgeManager descriptor to be used by all edges which need the routing table.
    EdgeManagerDescriptor hiveEdgeManagerDesc = new EdgeManagerDescriptor(
        CustomPartitionEdge.class.getName());    
    byte[] payload = getBytePayload(bucketToTaskMap);
    hiveEdgeManagerDesc.setUserPayload(payload);


    Map<String, EdgeManagerDescriptor> emMap = Maps.newHashMap();


    // Replace the edge manager for all vertices which have routing type custom.
    for (Entry<String, EdgeProperty> edgeEntry : context.getInputVertexEdgeProperties().entrySet()) {

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeManagerDescriptor

    TezTaskID firstTask = tasks.keySet().iterator().next();


    startVertex(v3);


    Vertex v1 = vertices.get("vertex1");
    EdgeManagerDescriptor mockEdgeManagerDescriptor =
        new EdgeManagerDescriptor(EdgeManagerForTest.class.getName());


    Map<String, EdgeManagerDescriptor> edgeManagerDescriptors =
        Collections.singletonMap(
       v1.getName(), mockEdgeManagerDescriptor);
    Assert.assertTrue(v3.setParallelism(1, null, edgeManagerDescriptors));

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeManagerDescriptor

    EdgeManagerForTest originalEm = (EdgeManagerForTest) em;
    Assert.assertTrue(Arrays.equals(edgePayload, originalEm.getEdgeManagerContext()
        .getUserPayload()));


    byte[] userPayload = new String("foo").getBytes();
    EdgeManagerDescriptor edgeManagerDescriptor =
        new EdgeManagerDescriptor(EdgeManagerForTest.class.getName());
    edgeManagerDescriptor.setUserPayload(userPayload);


    Vertex v1 = vertices.get("vertex1");
    Vertex v3 = vertices.get("vertex3"); // Vertex3 linked to v1 (v1 src, v3
                                         // dest)

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeManagerDescriptor

    if (proto.getEdgeManagerDescriptorsCount() > 0) {
      this.sourceEdgeManagers = new HashMap<String, EdgeManagerDescriptor>(
          proto.getEdgeManagerDescriptorsCount());
      for (EdgeManagerDescriptorProto edgeManagerProto :
        proto.getEdgeManagerDescriptorsList()) {
        EdgeManagerDescriptor edgeManagerDescriptor =
            DagTypeConverters.convertEdgeManagerDescriptorFromDAGPlan(
                edgeManagerProto.getEntityDescriptor());
        sourceEdgeManagers.put(edgeManagerProto.getEdgeName(),
            edgeManagerDescriptor);
      }

View Full Code Here

0 1 2

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.