Examples of EdgeManagerDescriptor

org.apache.tez.dag.api.EdgeManagerDescriptor

Examples of org.apache.tez.dag.api.EdgeManagerDescriptor

        taskCount++;
      }
    }


    // Construct the EdgeManager descriptor to be used by all edges which need the routing table.
    EdgeManagerDescriptor hiveEdgeManagerDesc = new EdgeManagerDescriptor(
        CustomPartitionEdge.class.getName());    
    byte[] payload = getBytePayload(bucketToTaskMap);
    hiveEdgeManagerDesc.setUserPayload(payload);


    Map<String, EdgeManagerDescriptor> emMap = Maps.newHashMap();


    // Replace the edge manager for all vertices which have routing type custom.
    for (Entry<String, EdgeProperty> edgeEntry : context.getInputVertexEdgeProperties().entrySet()) {

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeManagerDescriptor

      case CUSTOM_EDGE:
        
        dataMovementType = DataMovementType.CUSTOM;
        logicalOutputClass = OnFileUnorderedPartitionedKVOutput.class;
        logicalInputClass = ShuffledUnorderedKVInput.class;
        EdgeManagerDescriptor edgeDesc = new EdgeManagerDescriptor(
            CustomPartitionEdge.class.getName());
        CustomEdgeConfiguration edgeConf = 
            new CustomEdgeConfiguration(edgeProp.getNumBuckets(), null);
          DataOutputBuffer dob = new DataOutputBuffer();
          edgeConf.write(dob);
          byte[] userPayload = dob.getData();
        edgeDesc.setUserPayload(userPayload);
        edgeProperty =
          new EdgeProperty(edgeDesc,
              DataSourceType.PERSISTED,
              SchedulingType.SEQUENTIAL,
              new OutputDescriptor(logicalOutputClass.getName()),

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeManagerDescriptor

        CustomShuffleEdgeManagerConfig edgeManagerConfig =
            new CustomShuffleEdgeManagerConfig(
                currentParallelism, finalTaskParallelism, basePartitionRange,
                ((remainderRangeForLastShuffler > 0) ?
                    remainderRangeForLastShuffler : basePartitionRange));
        EdgeManagerDescriptor edgeManagerDescriptor =
            new EdgeManagerDescriptor(CustomShuffleEdgeManager.class.getName());
        edgeManagerDescriptor.setUserPayload(edgeManagerConfig.toUserPayload());
        edgeManagers.put(vertex, edgeManagerDescriptor);
      }
      
      context.setVertexParallelism(finalTaskParallelism, null, edgeManagers);
      updatePendingTasks();

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeManagerDescriptor

    TezTaskID firstTask = tasks.keySet().iterator().next();


    startVertex(v3);


    Vertex v1 = vertices.get("vertex1");
    EdgeManagerDescriptor mockEdgeManagerDescriptor =
        new EdgeManagerDescriptor(EdgeManagerForTest.class.getName());


    Map<String, EdgeManagerDescriptor> edgeManagerDescriptors =
        Collections.singletonMap(
       v1.getName(), mockEdgeManagerDescriptor);
    Assert.assertTrue(v3.setParallelism(1, null, edgeManagerDescriptors));

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeManagerDescriptor

    EdgeManagerForTest originalEm = (EdgeManagerForTest) em;
    Assert.assertTrue(Arrays.equals(edgePayload, originalEm.getEdgeManagerContext()
        .getUserPayload()));


    byte[] userPayload = new String("foo").getBytes();
    EdgeManagerDescriptor edgeManagerDescriptor =
        new EdgeManagerDescriptor(EdgeManagerForTest.class.getName());
    edgeManagerDescriptor.setUserPayload(userPayload);


    Vertex v1 = vertices.get("vertex1");
    Vertex v3 = vertices.get("vertex3"); // Vertex3 linked to v1 (v1 src, v3
                                         // dest)

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeManagerDescriptor

        
        // If CUSTOM without an edge manager, setup a fake edge manager. Avoid
        // referencing the fake edge manager within the API module.
        if (edgeProperty.getDataMovementType() == DataMovementType.CUSTOM
            && edgeProperty.getEdgeManagerDescriptor() == null) {
          EdgeManagerDescriptor edgeDesc = new EdgeManagerDescriptor(
              NullEdgeManager.class.getName());
          EdgeProperty ep = new EdgeProperty(edgeDesc, edgeProperty.getDataSourceType(),
              edgeProperty.getSchedulingType(), edgeProperty.getEdgeSource(),
              edgeProperty.getEdgeDestination());
          edgeProperty = ep;

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeManagerDescriptor

        CustomShuffleEdgeManagerConfig edgeManagerConfig =
            new CustomShuffleEdgeManagerConfig(
                currentParallelism, finalTaskParallelism, basePartitionRange,
                ((remainderRangeForLastShuffler > 0) ?
                    remainderRangeForLastShuffler : basePartitionRange));
        EdgeManagerDescriptor edgeManagerDescriptor =
            new EdgeManagerDescriptor(CustomShuffleEdgeManager.class.getName());
        edgeManagerDescriptor.setUserPayload(edgeManagerConfig.toUserPayload());
        edgeManagers.put(vertex, edgeManagerDescriptor);
      }
      
      context.setVertexParallelism(finalTaskParallelism, null, edgeManagers);
      updatePendingTasks();

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeManagerDescriptor

      logEvents(event, deserializedEvent);
    }
    {
      Map<String,EdgeManagerDescriptor> sourceEdgeManagers
          = new LinkedHashMap<String, EdgeManagerDescriptor>();
      sourceEdgeManagers.put("foo", new EdgeManagerDescriptor("bar"));
      sourceEdgeManagers.put("foo1", new EdgeManagerDescriptor("bar1").setUserPayload(
          new String("payload").getBytes()));
      VertexParallelismUpdatedEvent event =
          new VertexParallelismUpdatedEvent(
              TezVertexID.getInstance(
                  TezDAGID.getInstance(ApplicationId.newInstance(0, 1), 1), 111),

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeManagerDescriptor

    TezTaskID firstTask = tasks.keySet().iterator().next();


    startVertex(v3);


    Vertex v1 = vertices.get("vertex1");
    EdgeManagerDescriptor mockEdgeManagerDescriptor =
        new EdgeManagerDescriptor(EdgeManagerForTest.class.getName());


    Map<String, EdgeManagerDescriptor> edgeManagerDescriptors =
        Collections.singletonMap(
       v1.getName(), mockEdgeManagerDescriptor);
    Assert.assertTrue(v3.setParallelism(1, null, edgeManagerDescriptors));

View Full Code Here

Examples of org.apache.tez.dag.api.EdgeManagerDescriptor

    EdgeManagerForTest originalEm = (EdgeManagerForTest) em;
    Assert.assertTrue(Arrays.equals(edgePayload, originalEm.getEdgeManagerContext()
        .getUserPayload()));


    byte[] userPayload = new String("foo").getBytes();
    EdgeManagerDescriptor edgeManagerDescriptor =
        new EdgeManagerDescriptor(EdgeManagerForTest.class.getName());
    edgeManagerDescriptor.setUserPayload(userPayload);


    Vertex v1 = vertices.get("vertex1");
    Vertex v3 = vertices.get("vertex3"); // Vertex3 linked to v1 (v1 src, v3
                                         // dest)

View Full Code Here

0 1 2

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.