Examples of org.apache.tez.dag.api.EdgeManagerPluginDescriptor

org.apache.tez.dag.api.EdgeManagerPluginDescriptor
Describes the @link {@link EdgeManagerPlugin}

                LOG.info("Pig Partitioner Defined Vertex Manager: reset parallelism to " + dynamicParallelism
                        + " from " + currentParallelism);
                Map<String, EdgeManagerPluginDescriptor> edgeManagers =
                    new HashMap<String, EdgeManagerPluginDescriptor>();
                for(String vertex : getContext().getInputVertexEdgeProperties().keySet()) {
                    EdgeManagerPluginDescriptor edgeManagerDescriptor =
                            EdgeManagerPluginDescriptor.create(ScatterGatherEdgeManager.class.getName());
                    edgeManagers.put(vertex, edgeManagerDescriptor);
                }
                getContext().setVertexParallelism(dynamicParallelism, null, edgeManagers, null);
            }

View Full Code Here

    TezTaskID firstTask = tasks.keySet().iterator().next();


    VertexImpl v1 = vertices.get("vertex1");
    startVertex(vertices.get("vertex2"));
    startVertex(v1);
    EdgeManagerPluginDescriptor mockEdgeManagerDescriptor =
        EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName());


    Map<String, EdgeManagerPluginDescriptor> edgeManagerDescriptors =
        Collections.singletonMap(
       v1.getName(), mockEdgeManagerDescriptor);

View Full Code Here

    EdgeManagerForTest originalEm = (EdgeManagerForTest) em;
    Assert.assertTrue(Arrays.equals(edgePayload, originalEm.getEdgeManagerContext()
        .getUserPayload().deepCopyAsArray()));


    UserPayload userPayload = UserPayload.create(ByteBuffer.wrap(new String("foo").getBytes()));
    EdgeManagerPluginDescriptor edgeManagerDescriptor =
        EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName());
    edgeManagerDescriptor.setUserPayload(userPayload);


    Vertex v3 = vertices.get("vertex3");
    Vertex v5 = vertices.get("vertex5"); // Vertex5 linked to v3 (v3 src, v5
                                         // dest)

View Full Code Here

    
    Assert.assertEquals(VertexState.INITIALIZING, v1.getState());
    
    // setting the edge manager should vertex1 should not INIT/START it since
    // input initialization is not complete. v5 should be inited
    EdgeManagerPluginDescriptor mockEdgeManagerDescriptor =
        EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName());
    Edge e = v5.sourceVertices.get(v1);
    Assert.assertNull(e.getEdgeManager());
    e.setCustomEdgeManager(mockEdgeManagerDescriptor);
    dispatcher.await();

View Full Code Here

    Assert.assertEquals(VertexState.INITIALIZING, v1.getState());
    Assert.assertEquals(true, initializerManager1.hasShutDown);
    Assert.assertEquals(2, v1.getTotalTasks());
    Assert.assertArrayEquals(payload,
        v1.getInputSpecList(0).get(0).getInputDescriptor().getUserPayload().deepCopyAsArray());
    EdgeManagerPluginDescriptor mockEdgeManagerDescriptor =
        EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName());
    Edge e = v2.sourceVertices.get(v1);
    Assert.assertNull(e.getEdgeManager());
    e.setCustomEdgeManager(mockEdgeManagerDescriptor);
    dispatcher.await();

View Full Code Here

    Assert.assertEquals(VertexState.INITIALIZING, vA.getState());
    Assert.assertEquals(VertexState.INITIALIZING, vB.getState());
    Assert.assertEquals(VertexState.INITIALIZING, vC.getState());
    
    // setting the edge manager should vA to start
    EdgeManagerPluginDescriptor mockEdgeManagerDescriptor =
        EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName());
    Edge e = vC.sourceVertices.get(vA);
    Assert.assertNull(e.getEdgeManager());
    e.setCustomEdgeManager(mockEdgeManagerDescriptor);
    dispatcher.await();

View Full Code Here

            new CustomShuffleEdgeManagerConfig(
                currentParallelism, finalTaskParallelism, 
                getContext().getVertexNumTasks(vertex), basePartitionRange,
                ((remainderRangeForLastShuffler > 0) ?
                    remainderRangeForLastShuffler : basePartitionRange));
        EdgeManagerPluginDescriptor edgeManagerDescriptor =
            EdgeManagerPluginDescriptor.create(CustomShuffleEdgeManager.class.getName());
        edgeManagerDescriptor.setUserPayload(edgeManagerConfig.toUserPayload());
        edgeManagers.put(vertex, edgeManagerDescriptor);
      }
      
      getContext().setVertexParallelism(finalTaskParallelism, null, edgeManagers, null);
      updatePendingTasks();

View Full Code Here

    if (proto.getEdgeManagerDescriptorsCount() > 0) {
      this.sourceEdgeManagers = new HashMap<String, EdgeManagerPluginDescriptor>(
          proto.getEdgeManagerDescriptorsCount());
      for (EdgeManagerDescriptorProto edgeManagerProto :
        proto.getEdgeManagerDescriptorsList()) {
        EdgeManagerPluginDescriptor edgeManagerDescriptor =
            DagTypeConverters.convertEdgeManagerPluginDescriptorFromDAGPlan(
                edgeManagerProto.getEntityDescriptor());
        sourceEdgeManagers.put(edgeManagerProto.getEdgeName(),
            edgeManagerDescriptor);
      }

View Full Code Here

    inputNameInputSpecMap.put(inputName,
        InputSpecUpdate.createPerTaskInputSpecUpdate(numSplitsForTask));


    // Construct the EdgeManager descriptor to be used by all edges which need
    // the routing table.
    EdgeManagerPluginDescriptor hiveEdgeManagerDesc = null;
    if ((vertexType == VertexType.MULTI_INPUT_INITIALIZED_EDGES)
        || (vertexType == VertexType.INITIALIZED_EDGES)) {
      hiveEdgeManagerDesc = EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName());
      UserPayload payload = getBytePayload(bucketToTaskMap);
      hiveEdgeManagerDesc.setUserPayload(payload);
    }


    // Replace the edge manager for all vertices which have routing type custom.
    for (Entry<String, EdgeProperty> edgeEntry : context.getInputVertexEdgeProperties().entrySet()) {
      if (edgeEntry.getValue().getDataMovementType() == DataMovementType.CUSTOM

View Full Code Here

          .newBuilder(keyClass, valClass, MRPartitioner.class.getName(), partitionerConf)
          .setFromConfiguration(conf)
          .setKeySerializationClass(TezBytesWritableSerialization.class.getName(), null)
          .setValueSerializationClass(TezBytesWritableSerialization.class.getName(), null)
          .build();
      EdgeManagerPluginDescriptor edgeDesc =
          EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName());
      CustomEdgeConfiguration edgeConf =
          new CustomEdgeConfiguration(edgeProp.getNumBuckets(), null);
      DataOutputBuffer dob = new DataOutputBuffer();
      edgeConf.write(dob);
      byte[] userPayload = dob.getData();
      edgeDesc.setUserPayload(UserPayload.create(ByteBuffer.wrap(userPayload)));
      return et2Conf.createDefaultCustomEdgeProperty(edgeDesc);
    case CUSTOM_SIMPLE_EDGE:
      assert partitionerClassName != null;
      partitionerConf = createPartitionerConf(partitionerClassName, conf);
      UnorderedPartitionedKVEdgeConfig et3Conf = UnorderedPartitionedKVEdgeConfig

View Full Code Here

TOP

Related Classes of org.apache.tez.dag.api.EdgeManagerPluginDescriptor

org.apache.hadoop.hive.ql.exec.tez.CustomPartitionVertex

org.apache.hadoop.hive.ql.exec.tez.DagUtils

org.apache.pig.backend.hadoop.executionengine.tez.runtime.PartitionerDefinedVertexManager

org.apache.tez.dag.app.dag.impl.TestVertexImpl

org.apache.tez.dag.history.events.VertexParallelismUpdatedEvent

org.apache.tez.dag.library.vertexmanager.ShuffleVertexManager

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.