Examples of org.apache.flink.runtime.jobgraph.AbstractJobVertex

Package org.apache.flink.runtime.jobgraph

Examples of org.apache.flink.runtime.jobgraph.AbstractJobVertex

org.apache.flink.runtime.jobgraph.AbstractJobVertex
An abstract base class for a job vertex.


  private static AbstractJobVertex createIterationIntermediate(JobGraph jobGraph, int numSubTasks,
      TypeSerializerFactory<?> serializer, TypeComparatorFactory<?> comparator)
  {
    // --------------- the intermediate (reduce to min id) ---------------
    AbstractJobVertex intermediate = JobGraphUtils.createTask(IterationIntermediatePactTask.class,
      "Find Min Component-ID", jobGraph, numSubTasks);
    TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());
    {
      intermediateConfig.setIterationId(ITERATION_ID);


      intermediateConfig.addInputToGroup(0);
      intermediateConfig.setInputSerializer(serializer, 0);

View Full Code Here


    return output;
  }


  private static AbstractJobVertex createSync(JobGraph jobGraph, int numSubTasks, int maxIterations) {
    AbstractJobVertex sync = JobGraphUtils.createSync(jobGraph, numSubTasks);
    TaskConfig syncConfig = new TaskConfig(sync.getConfiguration());
    syncConfig.setNumberOfIterations(maxIterations);
    syncConfig.setIterationId(ITERATION_ID);
    syncConfig.addIterationAggregator(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME,
      new LongSumAggregator());
    syncConfig.setConvergenceCriterion(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME,

View Full Code Here

    JobGraph jobGraph = new JobGraph("Connected Components (Unified Tails)");


    // -- invariant vertices -----------------------------------------------------------------------------------
    InputFormatVertex vertices = createVerticesInput(jobGraph, verticesPath, numSubTasks, serializer, comparator);
    InputFormatVertex edges = createEdgesInput(jobGraph, edgesPath, numSubTasks, serializer, comparator);
    AbstractJobVertex head = createIterationHead(jobGraph, numSubTasks, serializer, comparator, pairComparator);


    AbstractJobVertex intermediate = createIterationIntermediate(jobGraph, numSubTasks, serializer, comparator);
    TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());


    OutputFormatVertex output = createOutput(jobGraph, resultPath, numSubTasks, serializer);
    AbstractJobVertex sync = createSync(jobGraph, numSubTasks, maxIterations);


    // --------------- the tail (solution set join) ---------------
    AbstractJobVertex tail = JobGraphUtils.createTask(IterationTailPactTask.class, "IterationTail", jobGraph, numSubTasks);
    TaskConfig tailConfig = new TaskConfig(tail.getConfiguration());
    {
      tailConfig.setIterationId(ITERATION_ID);


      tailConfig.setIsWorksetIteration();
      tailConfig.setIsWorksetUpdate();


      tailConfig.setIsSolutionSetUpdate();
      tailConfig.setIsSolutionSetUpdateWithoutReprobe();


      // inputs and driver
      tailConfig.addInputToGroup(0);
      tailConfig.setInputSerializer(serializer, 0);


      // output
      tailConfig.setOutputSerializer(serializer);


      // the driver
      tailConfig.setDriver(JoinWithSolutionSetSecondDriver.class);
      tailConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
      tailConfig.setDriverComparator(comparator, 0);
      tailConfig.setDriverPairComparator(pairComparator);
      
      tailConfig.setStubWrapper(new UserCodeClassWrapper<UpdateComponentIdMatch>(UpdateComponentIdMatch.class));
    }


    // -- edges ------------------------------------------------------------------------------------------------
    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(edges, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);


    JobGraphUtils.connect(head, intermediate, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    intermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);


    JobGraphUtils.connect(intermediate, tail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    tailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);


    SlotSharingGroup sharingGroup = new SlotSharingGroup();
    vertices.setSlotSharingGroup(sharingGroup);
    edges.setSlotSharingGroup(sharingGroup);
    head.setSlotSharingGroup(sharingGroup);
    intermediate.setSlotSharingGroup(sharingGroup);
    tail.setSlotSharingGroup(sharingGroup);
    output.setSlotSharingGroup(sharingGroup);
    sync.setSlotSharingGroup(sharingGroup);
    
    intermediate.setStrictlyCoLocatedWith(head);
    tail.setStrictlyCoLocatedWith(head);


    return jobGraph;
  }

View Full Code Here

    // input
    InputFormatVertex vertices = createVerticesInput(jobGraph, verticesPath, numSubTasks, serializer, comparator);
    InputFormatVertex edges = createEdgesInput(jobGraph, edgesPath, numSubTasks, serializer, comparator);


    // head
    AbstractJobVertex head = createIterationHead(jobGraph, numSubTasks, serializer, comparator, pairComparator);
    TaskConfig headConfig = new TaskConfig(head.getConfiguration());
    headConfig.setWaitForSolutionSetUpdate();


    // intermediate
    AbstractJobVertex intermediate = createIterationIntermediate(jobGraph, numSubTasks, serializer, comparator);
    TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());


    // output and auxiliaries
    OutputFormatVertex output = createOutput(jobGraph, resultPath, numSubTasks, serializer);
    AbstractJobVertex sync = createSync(jobGraph, numSubTasks, maxIterations);


    // ------------------ the intermediate (ss join) ----------------------
    AbstractJobVertex ssJoinIntermediate = JobGraphUtils.createTask(IterationIntermediatePactTask.class,
      "Solution Set Join", jobGraph, numSubTasks);
    TaskConfig ssJoinIntermediateConfig = new TaskConfig(ssJoinIntermediate.getConfiguration());
    {
      ssJoinIntermediateConfig.setIterationId(ITERATION_ID);


      // inputs
      ssJoinIntermediateConfig.addInputToGroup(0);
      ssJoinIntermediateConfig.setInputSerializer(serializer, 0);


      // output
      ssJoinIntermediateConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      ssJoinIntermediateConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      ssJoinIntermediateConfig.setOutputComparator(comparator, 0);
      ssJoinIntermediateConfig.setOutputComparator(comparator, 1);


      ssJoinIntermediateConfig.setOutputSerializer(serializer);


      // driver
      ssJoinIntermediateConfig.setDriver(JoinWithSolutionSetSecondDriver.class);
      ssJoinIntermediateConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
      ssJoinIntermediateConfig.setDriverComparator(comparator, 0);
      ssJoinIntermediateConfig.setDriverPairComparator(pairComparator);
      
      ssJoinIntermediateConfig.setStubWrapper(
        new UserCodeClassWrapper<UpdateComponentIdMatch>(UpdateComponentIdMatch.class));
    }


    // -------------------------- ss tail --------------------------------
    AbstractJobVertex ssTail = JobGraphUtils.createTask(IterationTailPactTask.class, "IterationSolutionSetTail",
      jobGraph, numSubTasks);
    TaskConfig ssTailConfig = new TaskConfig(ssTail.getConfiguration());
    {
      ssTailConfig.setIterationId(ITERATION_ID);
      ssTailConfig.setIsSolutionSetUpdate();
      ssTailConfig.setIsWorksetIteration();


      // inputs and driver
      ssTailConfig.addInputToGroup(0);
      ssTailConfig.setInputSerializer(serializer, 0);
      ssTailConfig.setInputAsynchronouslyMaterialized(0, true);
      ssTailConfig.setRelativeInputMaterializationMemory(0, MEM_FRAC_PER_CONSUMER);


      // output
      ssTailConfig.setOutputSerializer(serializer);


      // the driver
      ssTailConfig.setDriver(CollectorMapDriver.class);
      ssTailConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      ssTailConfig.setStubWrapper(new UserCodeClassWrapper<DummyMapper>(DummyMapper.class));
    }


    // -------------------------- ws tail --------------------------------
    AbstractJobVertex wsTail = JobGraphUtils.createTask(IterationTailPactTask.class, "IterationWorksetTail",
      jobGraph, numSubTasks);
    TaskConfig wsTailConfig = new TaskConfig(wsTail.getConfiguration());
    {
      wsTailConfig.setIterationId(ITERATION_ID);
      wsTailConfig.setIsWorksetIteration();
      wsTailConfig.setIsWorksetUpdate();


      // inputs and driver
      wsTailConfig.addInputToGroup(0);
      wsTailConfig.setInputSerializer(serializer, 0);


      // output
      wsTailConfig.setOutputSerializer(serializer);


      // the driver
      wsTailConfig.setDriver(CollectorMapDriver.class);
      wsTailConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      wsTailConfig.setStubWrapper(new UserCodeClassWrapper<DummyMapper>(DummyMapper.class));
    }


    // --------------- the wiring ---------------------


    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(edges, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);


    JobGraphUtils.connect(head, intermediate, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    intermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);


    JobGraphUtils.connect(intermediate, ssJoinIntermediate, ChannelType.NETWORK, DistributionPattern.POINTWISE);
    ssJoinIntermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(ssJoinIntermediate, ssTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    ssTailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(ssJoinIntermediate, wsTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    wsTailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);


    SlotSharingGroup sharingGroup = new SlotSharingGroup();
    vertices.setSlotSharingGroup(sharingGroup);
    edges.setSlotSharingGroup(sharingGroup);
    head.setSlotSharingGroup(sharingGroup);
    intermediate.setSlotSharingGroup(sharingGroup);
    ssJoinIntermediate.setSlotSharingGroup(sharingGroup);
    wsTail.setSlotSharingGroup(sharingGroup);
    ssTail.setSlotSharingGroup(sharingGroup);
    output.setSlotSharingGroup(sharingGroup);
    sync.setSlotSharingGroup(sharingGroup);
    
    intermediate.setStrictlyCoLocatedWith(head);
    ssJoinIntermediate.setStrictlyCoLocatedWith(head);
    wsTail.setStrictlyCoLocatedWith(head);
    ssTail.setStrictlyCoLocatedWith(head);


    return jobGraph;
  }

View Full Code Here

    // input
    InputFormatVertex vertices = createVerticesInput(jobGraph, verticesPath, numSubTasks, serializer, comparator);
    InputFormatVertex edges = createEdgesInput(jobGraph, edgesPath, numSubTasks, serializer, comparator);


    // head
    AbstractJobVertex head = createIterationHead(jobGraph, numSubTasks, serializer, comparator, pairComparator);
    TaskConfig headConfig = new TaskConfig(head.getConfiguration());
    headConfig.setWaitForSolutionSetUpdate();


    // intermediate
    AbstractJobVertex intermediate = createIterationIntermediate(jobGraph, numSubTasks, serializer, comparator);
    TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());


    // output and auxiliaries
    AbstractJobVertex output = createOutput(jobGraph, resultPath, numSubTasks, serializer);
    AbstractJobVertex sync = createSync(jobGraph, numSubTasks, maxIterations);


    // ------------------ the intermediate (ws update) ----------------------
    AbstractJobVertex wsUpdateIntermediate = 
      JobGraphUtils.createTask(IterationIntermediatePactTask.class, "WorksetUpdate", jobGraph, numSubTasks);
    TaskConfig wsUpdateConfig = new TaskConfig(wsUpdateIntermediate.getConfiguration());
    {
      wsUpdateConfig.setIterationId(ITERATION_ID);
      wsUpdateConfig.setIsWorksetIteration();
      wsUpdateConfig.setIsWorksetUpdate();


      // inputs
      wsUpdateConfig.addInputToGroup(0);
      wsUpdateConfig.setInputSerializer(serializer, 0);


      // output
      wsUpdateConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      wsUpdateConfig.setOutputComparator(comparator, 0);


      wsUpdateConfig.setOutputSerializer(serializer);


      // driver
      wsUpdateConfig.setDriver(JoinWithSolutionSetSecondDriver.class);
      wsUpdateConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
      wsUpdateConfig.setDriverComparator(comparator, 0);
      wsUpdateConfig.setDriverPairComparator(pairComparator);
      
      wsUpdateConfig.setStubWrapper(new UserCodeClassWrapper<UpdateComponentIdMatch>(
        UpdateComponentIdMatch.class));
    }


    // -------------------------- ss tail --------------------------------
    AbstractJobVertex ssTail =
      JobGraphUtils.createTask(IterationTailPactTask.class, "IterationSolutionSetTail", jobGraph, numSubTasks);
    TaskConfig ssTailConfig = new TaskConfig(ssTail.getConfiguration());
    {
      ssTailConfig.setIterationId(ITERATION_ID);
      ssTailConfig.setIsSolutionSetUpdate();
      ssTailConfig.setIsWorksetIteration();


      // inputs and driver
      ssTailConfig.addInputToGroup(0);
      ssTailConfig.setInputSerializer(serializer, 0);


      // output
      ssTailConfig.setOutputSerializer(serializer);


      // the driver
      ssTailConfig.setDriver(CollectorMapDriver.class);
      ssTailConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      ssTailConfig.setStubWrapper(new UserCodeClassWrapper<DummyMapper>(DummyMapper.class));
    }


    // edges


    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(edges, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);


    JobGraphUtils.connect(head, intermediate, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    intermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);


    JobGraphUtils.connect(intermediate, wsUpdateIntermediate, ChannelType.NETWORK,
      DistributionPattern.POINTWISE);
    wsUpdateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(wsUpdateIntermediate, ssTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    ssTailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);


    SlotSharingGroup sharingGroup = new SlotSharingGroup();
    vertices.setSlotSharingGroup(sharingGroup);
    edges.setSlotSharingGroup(sharingGroup);
    head.setSlotSharingGroup(sharingGroup);
    intermediate.setSlotSharingGroup(sharingGroup);
    wsUpdateIntermediate.setSlotSharingGroup(sharingGroup);
    ssTail.setSlotSharingGroup(sharingGroup);
    output.setSlotSharingGroup(sharingGroup);
    sync.setSlotSharingGroup(sharingGroup);


    intermediate.setStrictlyCoLocatedWith(head);
    wsUpdateIntermediate.setStrictlyCoLocatedWith(head);
    ssTail.setStrictlyCoLocatedWith(head);


    return jobGraph;
  }

View Full Code Here

    // input
    InputFormatVertex vertices = createVerticesInput(jobGraph, verticesPath, numSubTasks, serializer, comparator);
    InputFormatVertex edges = createEdgesInput(jobGraph, edgesPath, numSubTasks, serializer, comparator);


    // head
    AbstractJobVertex head = createIterationHead(jobGraph, numSubTasks, serializer, comparator, pairComparator);


    // intermediate
    AbstractJobVertex intermediate = createIterationIntermediate(jobGraph, numSubTasks, serializer, comparator);
    TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());


    // output and auxiliaries
    AbstractJobVertex output = createOutput(jobGraph, resultPath, numSubTasks, serializer);
    AbstractJobVertex sync = createSync(jobGraph, numSubTasks, maxIterations);


    // ------------------ the intermediate (ss update) ----------------------
    AbstractJobVertex ssJoinIntermediate = JobGraphUtils.createTask(IterationIntermediatePactTask.class,
      "Solution Set Update", jobGraph, numSubTasks);
    TaskConfig ssJoinIntermediateConfig = new TaskConfig(ssJoinIntermediate.getConfiguration());
    {
      ssJoinIntermediateConfig.setIterationId(ITERATION_ID);
      ssJoinIntermediateConfig.setIsSolutionSetUpdate();
      ssJoinIntermediateConfig.setIsSolutionSetUpdateWithoutReprobe();


      // inputs
      ssJoinIntermediateConfig.addInputToGroup(0);
      ssJoinIntermediateConfig.setInputSerializer(serializer, 0);


      // output
      ssJoinIntermediateConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      ssJoinIntermediateConfig.setOutputComparator(comparator, 0);


      ssJoinIntermediateConfig.setOutputSerializer(serializer);


      // driver
      ssJoinIntermediateConfig.setDriver(JoinWithSolutionSetSecondDriver.class);
      ssJoinIntermediateConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
      ssJoinIntermediateConfig.setDriverComparator(comparator, 0);
      ssJoinIntermediateConfig.setDriverPairComparator(pairComparator);
      
      ssJoinIntermediateConfig.setStubWrapper(new UserCodeClassWrapper<UpdateComponentIdMatch>(UpdateComponentIdMatch.class));
    }


    // -------------------------- ws tail --------------------------------
    AbstractJobVertex wsTail = JobGraphUtils.createTask(IterationTailPactTask.class, "IterationWorksetTail", jobGraph, numSubTasks);
    TaskConfig wsTailConfig = new TaskConfig(wsTail.getConfiguration());
    {
      wsTailConfig.setIterationId(ITERATION_ID);
      wsTailConfig.setIsWorksetIteration();
      wsTailConfig.setIsWorksetUpdate();


      // inputs and driver
      wsTailConfig.addInputToGroup(0);
      wsTailConfig.setInputSerializer(serializer, 0);


      // output
      wsTailConfig.setOutputSerializer(serializer);


      // the driver
      wsTailConfig.setDriver(CollectorMapDriver.class);
      wsTailConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      wsTailConfig.setStubWrapper(new UserCodeClassWrapper<DummyMapper>(DummyMapper.class));
    }


    // --------------- the wiring ---------------------


    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(edges, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);


    JobGraphUtils.connect(head, intermediate, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    intermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);


    JobGraphUtils.connect(intermediate, ssJoinIntermediate, ChannelType.NETWORK, DistributionPattern.POINTWISE);
    ssJoinIntermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(ssJoinIntermediate, wsTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    wsTailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);


    
    SlotSharingGroup sharingGroup = new SlotSharingGroup();
    vertices.setSlotSharingGroup(sharingGroup);
    edges.setSlotSharingGroup(sharingGroup);
    head.setSlotSharingGroup(sharingGroup);
    intermediate.setSlotSharingGroup(sharingGroup);
    ssJoinIntermediate.setSlotSharingGroup(sharingGroup);
    wsTail.setSlotSharingGroup(sharingGroup);
    output.setSlotSharingGroup(sharingGroup);
    sync.setSlotSharingGroup(sharingGroup);


    intermediate.setStrictlyCoLocatedWith(head);
    ssJoinIntermediate.setStrictlyCoLocatedWith(head);
    wsTail.setStrictlyCoLocatedWith(head);


    return jobGraph;
  }

View Full Code Here

        boolean isSlowReceiver, int numSubtasks)
    {
      JobGraph jobGraph = new JobGraph("Speed Test");
      SlotSharingGroup sharingGroup = new SlotSharingGroup();


      AbstractJobVertex producer = new AbstractJobVertex("Speed Test Producer");
      jobGraph.addVertex(producer);
      producer.setSlotSharingGroup(sharingGroup);
      
      producer.setInvokableClass(SpeedTestProducer.class);
      producer.setParallelism(numSubtasks);
      producer.getConfiguration().setInteger(DATA_VOLUME_GB_CONFIG_KEY, dataVolumeGb);
      producer.getConfiguration().setBoolean(IS_SLOW_SENDER_CONFIG_KEY, isSlowSender);


      AbstractJobVertex forwarder = null;
      if (useForwarder) {
        forwarder = new AbstractJobVertex("Speed Test Forwarder");
        jobGraph.addVertex(forwarder);
        forwarder.setSlotSharingGroup(sharingGroup);
        
        forwarder.setInvokableClass(SpeedTestForwarder.class);
        forwarder.setParallelism(numSubtasks);
      }


      AbstractJobVertex consumer = new AbstractJobVertex("Speed Test Consumer");
      jobGraph.addVertex(consumer);
      consumer.setSlotSharingGroup(sharingGroup);
      
      consumer.setInvokableClass(SpeedTestConsumer.class);
      consumer.setParallelism(numSubtasks);
      consumer.getConfiguration().setBoolean(IS_SLOW_RECEIVER_CONFIG_KEY, isSlowReceiver);


      if (useForwarder) {
        forwarder.connectNewDataSetAsInput(producer, DistributionPattern.BIPARTITE);
        consumer.connectNewDataSetAsInput(forwarder, DistributionPattern.BIPARTITE);
      }
      else {
        consumer.connectNewDataSetAsInput(producer, DistributionPattern.BIPARTITE);
      }


      return jobGraph;
    }

View Full Code Here

  }


  @SuppressWarnings("rawtypes") 
  public static AbstractJobVertex createTask(Class<? extends RegularPactTask> task, String name, JobGraph graph, int parallelism)
  {
    AbstractJobVertex taskVertex = new AbstractJobVertex(name);
    graph.addVertex(taskVertex);
    
    taskVertex.setInvokableClass(task);
    taskVertex.setParallelism(parallelism);
    return taskVertex;
  }

View Full Code Here

    taskVertex.setParallelism(parallelism);
    return taskVertex;
  }


  public static AbstractJobVertex createSync(JobGraph jobGraph, int parallelism) {
    AbstractJobVertex sync = new AbstractJobVertex("BulkIterationSync");
    jobGraph.addVertex(sync);
    
    sync.setInvokableClass(IterationSynchronizationSinkTask.class);
    sync.setParallelism(1);
    
    TaskConfig syncConfig = new TaskConfig(sync.getConfiguration());
    syncConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, parallelism);
    return sync;
  }

View Full Code Here

      inputConfig.setOutputSerializer(serializer);
      inputConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
    }


    // - head ------------------------------------------------------------------------------------------------------
    AbstractJobVertex head = JobGraphUtils.createTask(IterationHeadPactTask.class, "Iteration Head", jobGraph, numSubTasks);
    TaskConfig headConfig = new TaskConfig(head.getConfiguration());
    {
      headConfig.setIterationId(ITERATION_ID);


      // input to iteration head
      headConfig.addInputToGroup(0);
      headConfig.setInputSerializer(serializer, 0);
      headConfig.setInputLocalStrategy(0, LocalStrategy.NONE);
      headConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);


      // output into iteration
      headConfig.setOutputSerializer(serializer);
      headConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
      headConfig.setOutputComparator(comparator, 0);


      // final output
      TaskConfig headFinalOutConfig = new TaskConfig(new Configuration());
      headFinalOutConfig.setOutputSerializer(serializer);
      headFinalOutConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      headConfig.setIterationHeadFinalOutputConfig(headFinalOutConfig);


      // the sync
      headConfig.setIterationHeadIndexOfSyncOutput(2);


      // driver
      headConfig.setDriver(CollectorMapDriver.class);
      headConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      headConfig.setStubWrapper(new UserCodeClassWrapper<DummyMapper>(DummyMapper.class));


      // back channel
      headConfig.setRelativeBackChannelMemory(1.0);
    }


    // - tail ------------------------------------------------------------------------------------------------------
    AbstractJobVertex tail = JobGraphUtils.createTask(IterationTailPactTask.class, "Chained Iteration Tail", jobGraph, numSubTasks);
    TaskConfig tailConfig = new TaskConfig(tail.getConfiguration());
    {
      tailConfig.setIterationId(ITERATION_ID);


      // inputs and driver
      tailConfig.addInputToGroup(0);
      tailConfig.setInputSerializer(serializer, 0);


      // output
      tailConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      tailConfig.setOutputSerializer(serializer);


      // the driver
      tailConfig.setDriver(GroupReduceDriver.class);
      tailConfig.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
      tailConfig.setDriverComparator(comparator, 0);
      tailConfig.setStubWrapper(new UserCodeClassWrapper<DummyReducer>(DummyReducer.class));


      // chained mapper
      TaskConfig chainedMapperConfig = new TaskConfig(new Configuration());
      chainedMapperConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      chainedMapperConfig.setStubWrapper(new UserCodeClassWrapper<IncrementCoordinatesMapper>(
        IncrementCoordinatesMapper.class));


      chainedMapperConfig.setInputLocalStrategy(0, LocalStrategy.NONE);
      chainedMapperConfig.setInputSerializer(serializer, 0);


      chainedMapperConfig.setOutputSerializer(serializer);


      chainedMapperConfig.setIsWorksetUpdate();


      tailConfig.addChainedTask(ChainedCollectorMapDriver.class, chainedMapperConfig, "Chained ID Mapper");
    }


    // - output ----------------------------------------------------------------------------------------------------
    OutputFormatVertex output = JobGraphUtils.createFileOutput(jobGraph, "Output", numSubTasks);
    TaskConfig outputConfig = new TaskConfig(output.getConfiguration());
    {
      outputConfig.addInputToGroup(0);
      outputConfig.setInputSerializer(serializer, 0);


      outputConfig.setStubWrapper(new UserCodeClassWrapper<PointOutFormat>(PointOutFormat.class));
      outputConfig.setStubParameter(FileOutputFormat.FILE_PARAMETER_KEY, outputPath);
    }


    // - sync ------------------------------------------------------------------------------------------------------
    AbstractJobVertex sync = JobGraphUtils.createSync(jobGraph, numSubTasks);
    TaskConfig syncConfig = new TaskConfig(sync.getConfiguration());
    syncConfig.setNumberOfIterations(maxIterations);
    syncConfig.setIterationId(ITERATION_ID);


    // --------------------------------------------------------------------------------------------------------------
    // 2. EDGES
    // --------------------------------------------------------------------------------------------------------------
    JobGraphUtils.connect(input, head, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(head, tail, ChannelType.IN_MEMORY, DistributionPattern.BIPARTITE);
    tailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);


    JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);


    // --------------------------------------------------------------------------------------------------------------
    // 3. INSTANCE SHARING
    // --------------------------------------------------------------------------------------------------------------
    
    SlotSharingGroup sharingGroup = new SlotSharingGroup();
    
    input.setSlotSharingGroup(sharingGroup);
    head.setSlotSharingGroup(sharingGroup);
    tail.setSlotSharingGroup(sharingGroup);
    output.setSlotSharingGroup(sharingGroup);
    sync.setSlotSharingGroup(sharingGroup);
    
    tail.setStrictlyCoLocatedWith(head);


    return jobGraph;
  }

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of org.apache.flink.runtime.jobgraph.AbstractJobVertex

org.apache.flink.compiler.plantranslate.NepheleJobGraphGenerator

org.apache.flink.configuration.Configuration

org.apache.flink.runtime.executiongraph.AllVerticesIteratorTest

org.apache.flink.runtime.executiongraph.ExecutionGraphConstructionTest

org.apache.flink.runtime.executiongraph.ExecutionGraphDeploymentTest

org.apache.flink.runtime.executiongraph.ExecutionGraphRestartTest

org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils

org.apache.flink.runtime.executiongraph.ExecutionStateProgressTest

org.apache.flink.runtime.executiongraph.PointwisePatternTest

org.apache.flink.runtime.executiongraph.TaskManagerLossFailsTasksTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.