Examples of org.apache.flink.runtime.jobgraph.InputFormatVertex

Package org.apache.flink.runtime.jobgraph

Examples of org.apache.flink.runtime.jobgraph.InputFormatVertex

org.apache.flink.runtime.jobgraph.InputFormatVertex

  private static InputFormatVertex createVerticesInput(JobGraph jobGraph, String verticesPath, int numSubTasks,
      TypeSerializerFactory<?> serializer, TypeComparatorFactory<?> comparator)
  {
    @SuppressWarnings("unchecked")
    CsvInputFormat verticesInFormat = new CsvInputFormat(' ', LongValue.class);
    InputFormatVertex verticesInput = JobGraphUtils.createInput(verticesInFormat, verticesPath, "VerticesInput",
      jobGraph, numSubTasks);
    TaskConfig verticesInputConfig = new TaskConfig(verticesInput.getConfiguration());
    {
      verticesInputConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      verticesInputConfig.setOutputSerializer(serializer);


      // chained mapper that duplicates the id

View Full Code Here

      TypeSerializerFactory<?> serializer, TypeComparatorFactory<?> comparator)
  {
    // edges
    @SuppressWarnings("unchecked")
    CsvInputFormat edgesInFormat = new CsvInputFormat(' ', LongValue.class, LongValue.class);
    InputFormatVertex edgesInput = JobGraphUtils.createInput(edgesInFormat, edgesPath, "EdgesInput", jobGraph,
      numSubTasks);
    TaskConfig edgesInputConfig = new TaskConfig(edgesInput.getConfiguration());
    {
      edgesInputConfig.setOutputSerializer(serializer);
      edgesInputConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
      edgesInputConfig.setOutputComparator(comparator, 0);
    }

View Full Code Here

    final TypePairComparatorFactory<?, ?> pairComparator = RecordPairComparatorFactory.get();


    JobGraph jobGraph = new JobGraph("Connected Components (Unified Tails)");


    // -- invariant vertices -----------------------------------------------------------------------------------
    InputFormatVertex vertices = createVerticesInput(jobGraph, verticesPath, numSubTasks, serializer, comparator);
    InputFormatVertex edges = createEdgesInput(jobGraph, edgesPath, numSubTasks, serializer, comparator);
    AbstractJobVertex head = createIterationHead(jobGraph, numSubTasks, serializer, comparator, pairComparator);


    AbstractJobVertex intermediate = createIterationIntermediate(jobGraph, numSubTasks, serializer, comparator);
    TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());


    OutputFormatVertex output = createOutput(jobGraph, resultPath, numSubTasks, serializer);
    AbstractJobVertex sync = createSync(jobGraph, numSubTasks, maxIterations);


    // --------------- the tail (solution set join) ---------------
    AbstractJobVertex tail = JobGraphUtils.createTask(IterationTailPactTask.class, "IterationTail", jobGraph, numSubTasks);
    TaskConfig tailConfig = new TaskConfig(tail.getConfiguration());
    {
      tailConfig.setIterationId(ITERATION_ID);


      tailConfig.setIsWorksetIteration();
      tailConfig.setIsWorksetUpdate();


      tailConfig.setIsSolutionSetUpdate();
      tailConfig.setIsSolutionSetUpdateWithoutReprobe();


      // inputs and driver
      tailConfig.addInputToGroup(0);
      tailConfig.setInputSerializer(serializer, 0);


      // output
      tailConfig.setOutputSerializer(serializer);


      // the driver
      tailConfig.setDriver(JoinWithSolutionSetSecondDriver.class);
      tailConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
      tailConfig.setDriverComparator(comparator, 0);
      tailConfig.setDriverPairComparator(pairComparator);
      
      tailConfig.setStubWrapper(new UserCodeClassWrapper<UpdateComponentIdMatch>(UpdateComponentIdMatch.class));
    }


    // -- edges ------------------------------------------------------------------------------------------------
    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(edges, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);


    JobGraphUtils.connect(head, intermediate, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    intermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);


    JobGraphUtils.connect(intermediate, tail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    tailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);


    SlotSharingGroup sharingGroup = new SlotSharingGroup();
    vertices.setSlotSharingGroup(sharingGroup);
    edges.setSlotSharingGroup(sharingGroup);
    head.setSlotSharingGroup(sharingGroup);
    intermediate.setSlotSharingGroup(sharingGroup);
    tail.setSlotSharingGroup(sharingGroup);
    output.setSlotSharingGroup(sharingGroup);
    sync.setSlotSharingGroup(sharingGroup);

View Full Code Here

    final TypePairComparatorFactory<?, ?> pairComparator = RecordPairComparatorFactory.get();


    JobGraph jobGraph = new JobGraph("Connected Components (Unified Tails)");


    // input
    InputFormatVertex vertices = createVerticesInput(jobGraph, verticesPath, numSubTasks, serializer, comparator);
    InputFormatVertex edges = createEdgesInput(jobGraph, edgesPath, numSubTasks, serializer, comparator);


    // head
    AbstractJobVertex head = createIterationHead(jobGraph, numSubTasks, serializer, comparator, pairComparator);
    TaskConfig headConfig = new TaskConfig(head.getConfiguration());
    headConfig.setWaitForSolutionSetUpdate();


    // intermediate
    AbstractJobVertex intermediate = createIterationIntermediate(jobGraph, numSubTasks, serializer, comparator);
    TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());


    // output and auxiliaries
    OutputFormatVertex output = createOutput(jobGraph, resultPath, numSubTasks, serializer);
    AbstractJobVertex sync = createSync(jobGraph, numSubTasks, maxIterations);


    // ------------------ the intermediate (ss join) ----------------------
    AbstractJobVertex ssJoinIntermediate = JobGraphUtils.createTask(IterationIntermediatePactTask.class,
      "Solution Set Join", jobGraph, numSubTasks);
    TaskConfig ssJoinIntermediateConfig = new TaskConfig(ssJoinIntermediate.getConfiguration());
    {
      ssJoinIntermediateConfig.setIterationId(ITERATION_ID);


      // inputs
      ssJoinIntermediateConfig.addInputToGroup(0);
      ssJoinIntermediateConfig.setInputSerializer(serializer, 0);


      // output
      ssJoinIntermediateConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      ssJoinIntermediateConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      ssJoinIntermediateConfig.setOutputComparator(comparator, 0);
      ssJoinIntermediateConfig.setOutputComparator(comparator, 1);


      ssJoinIntermediateConfig.setOutputSerializer(serializer);


      // driver
      ssJoinIntermediateConfig.setDriver(JoinWithSolutionSetSecondDriver.class);
      ssJoinIntermediateConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
      ssJoinIntermediateConfig.setDriverComparator(comparator, 0);
      ssJoinIntermediateConfig.setDriverPairComparator(pairComparator);
      
      ssJoinIntermediateConfig.setStubWrapper(
        new UserCodeClassWrapper<UpdateComponentIdMatch>(UpdateComponentIdMatch.class));
    }


    // -------------------------- ss tail --------------------------------
    AbstractJobVertex ssTail = JobGraphUtils.createTask(IterationTailPactTask.class, "IterationSolutionSetTail",
      jobGraph, numSubTasks);
    TaskConfig ssTailConfig = new TaskConfig(ssTail.getConfiguration());
    {
      ssTailConfig.setIterationId(ITERATION_ID);
      ssTailConfig.setIsSolutionSetUpdate();
      ssTailConfig.setIsWorksetIteration();


      // inputs and driver
      ssTailConfig.addInputToGroup(0);
      ssTailConfig.setInputSerializer(serializer, 0);
      ssTailConfig.setInputAsynchronouslyMaterialized(0, true);
      ssTailConfig.setRelativeInputMaterializationMemory(0, MEM_FRAC_PER_CONSUMER);


      // output
      ssTailConfig.setOutputSerializer(serializer);


      // the driver
      ssTailConfig.setDriver(CollectorMapDriver.class);
      ssTailConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      ssTailConfig.setStubWrapper(new UserCodeClassWrapper<DummyMapper>(DummyMapper.class));
    }


    // -------------------------- ws tail --------------------------------
    AbstractJobVertex wsTail = JobGraphUtils.createTask(IterationTailPactTask.class, "IterationWorksetTail",
      jobGraph, numSubTasks);
    TaskConfig wsTailConfig = new TaskConfig(wsTail.getConfiguration());
    {
      wsTailConfig.setIterationId(ITERATION_ID);
      wsTailConfig.setIsWorksetIteration();
      wsTailConfig.setIsWorksetUpdate();


      // inputs and driver
      wsTailConfig.addInputToGroup(0);
      wsTailConfig.setInputSerializer(serializer, 0);


      // output
      wsTailConfig.setOutputSerializer(serializer);


      // the driver
      wsTailConfig.setDriver(CollectorMapDriver.class);
      wsTailConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      wsTailConfig.setStubWrapper(new UserCodeClassWrapper<DummyMapper>(DummyMapper.class));
    }


    // --------------- the wiring ---------------------


    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(edges, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);


    JobGraphUtils.connect(head, intermediate, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    intermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);


    JobGraphUtils.connect(intermediate, ssJoinIntermediate, ChannelType.NETWORK, DistributionPattern.POINTWISE);
    ssJoinIntermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(ssJoinIntermediate, ssTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    ssTailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(ssJoinIntermediate, wsTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    wsTailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);


    SlotSharingGroup sharingGroup = new SlotSharingGroup();
    vertices.setSlotSharingGroup(sharingGroup);
    edges.setSlotSharingGroup(sharingGroup);
    head.setSlotSharingGroup(sharingGroup);
    intermediate.setSlotSharingGroup(sharingGroup);
    ssJoinIntermediate.setSlotSharingGroup(sharingGroup);
    wsTail.setSlotSharingGroup(sharingGroup);
    ssTail.setSlotSharingGroup(sharingGroup);

View Full Code Here

    final TypePairComparatorFactory<?, ?> pairComparator = RecordPairComparatorFactory.get();


    JobGraph jobGraph = new JobGraph("Connected Components (Intermediate Workset Update, Solution Set Tail)");


    // input
    InputFormatVertex vertices = createVerticesInput(jobGraph, verticesPath, numSubTasks, serializer, comparator);
    InputFormatVertex edges = createEdgesInput(jobGraph, edgesPath, numSubTasks, serializer, comparator);


    // head
    AbstractJobVertex head = createIterationHead(jobGraph, numSubTasks, serializer, comparator, pairComparator);
    TaskConfig headConfig = new TaskConfig(head.getConfiguration());
    headConfig.setWaitForSolutionSetUpdate();


    // intermediate
    AbstractJobVertex intermediate = createIterationIntermediate(jobGraph, numSubTasks, serializer, comparator);
    TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());


    // output and auxiliaries
    AbstractJobVertex output = createOutput(jobGraph, resultPath, numSubTasks, serializer);
    AbstractJobVertex sync = createSync(jobGraph, numSubTasks, maxIterations);


    // ------------------ the intermediate (ws update) ----------------------
    AbstractJobVertex wsUpdateIntermediate = 
      JobGraphUtils.createTask(IterationIntermediatePactTask.class, "WorksetUpdate", jobGraph, numSubTasks);
    TaskConfig wsUpdateConfig = new TaskConfig(wsUpdateIntermediate.getConfiguration());
    {
      wsUpdateConfig.setIterationId(ITERATION_ID);
      wsUpdateConfig.setIsWorksetIteration();
      wsUpdateConfig.setIsWorksetUpdate();


      // inputs
      wsUpdateConfig.addInputToGroup(0);
      wsUpdateConfig.setInputSerializer(serializer, 0);


      // output
      wsUpdateConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      wsUpdateConfig.setOutputComparator(comparator, 0);


      wsUpdateConfig.setOutputSerializer(serializer);


      // driver
      wsUpdateConfig.setDriver(JoinWithSolutionSetSecondDriver.class);
      wsUpdateConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
      wsUpdateConfig.setDriverComparator(comparator, 0);
      wsUpdateConfig.setDriverPairComparator(pairComparator);
      
      wsUpdateConfig.setStubWrapper(new UserCodeClassWrapper<UpdateComponentIdMatch>(
        UpdateComponentIdMatch.class));
    }


    // -------------------------- ss tail --------------------------------
    AbstractJobVertex ssTail =
      JobGraphUtils.createTask(IterationTailPactTask.class, "IterationSolutionSetTail", jobGraph, numSubTasks);
    TaskConfig ssTailConfig = new TaskConfig(ssTail.getConfiguration());
    {
      ssTailConfig.setIterationId(ITERATION_ID);
      ssTailConfig.setIsSolutionSetUpdate();
      ssTailConfig.setIsWorksetIteration();


      // inputs and driver
      ssTailConfig.addInputToGroup(0);
      ssTailConfig.setInputSerializer(serializer, 0);


      // output
      ssTailConfig.setOutputSerializer(serializer);


      // the driver
      ssTailConfig.setDriver(CollectorMapDriver.class);
      ssTailConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      ssTailConfig.setStubWrapper(new UserCodeClassWrapper<DummyMapper>(DummyMapper.class));
    }


    // edges


    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(edges, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);


    JobGraphUtils.connect(head, intermediate, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    intermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);


    JobGraphUtils.connect(intermediate, wsUpdateIntermediate, ChannelType.NETWORK,
      DistributionPattern.POINTWISE);
    wsUpdateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(wsUpdateIntermediate, ssTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    ssTailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);


    SlotSharingGroup sharingGroup = new SlotSharingGroup();
    vertices.setSlotSharingGroup(sharingGroup);
    edges.setSlotSharingGroup(sharingGroup);
    head.setSlotSharingGroup(sharingGroup);
    intermediate.setSlotSharingGroup(sharingGroup);
    wsUpdateIntermediate.setSlotSharingGroup(sharingGroup);
    ssTail.setSlotSharingGroup(sharingGroup);
    output.setSlotSharingGroup(sharingGroup);

View Full Code Here

    final TypePairComparatorFactory<?, ?> pairComparator = RecordPairComparatorFactory.get();


    JobGraph jobGraph = new JobGraph("Connected Components (Intermediate Solution Set Update, Workset Tail)");


    // input
    InputFormatVertex vertices = createVerticesInput(jobGraph, verticesPath, numSubTasks, serializer, comparator);
    InputFormatVertex edges = createEdgesInput(jobGraph, edgesPath, numSubTasks, serializer, comparator);


    // head
    AbstractJobVertex head = createIterationHead(jobGraph, numSubTasks, serializer, comparator, pairComparator);


    // intermediate
    AbstractJobVertex intermediate = createIterationIntermediate(jobGraph, numSubTasks, serializer, comparator);
    TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());


    // output and auxiliaries
    AbstractJobVertex output = createOutput(jobGraph, resultPath, numSubTasks, serializer);
    AbstractJobVertex sync = createSync(jobGraph, numSubTasks, maxIterations);


    // ------------------ the intermediate (ss update) ----------------------
    AbstractJobVertex ssJoinIntermediate = JobGraphUtils.createTask(IterationIntermediatePactTask.class,
      "Solution Set Update", jobGraph, numSubTasks);
    TaskConfig ssJoinIntermediateConfig = new TaskConfig(ssJoinIntermediate.getConfiguration());
    {
      ssJoinIntermediateConfig.setIterationId(ITERATION_ID);
      ssJoinIntermediateConfig.setIsSolutionSetUpdate();
      ssJoinIntermediateConfig.setIsSolutionSetUpdateWithoutReprobe();


      // inputs
      ssJoinIntermediateConfig.addInputToGroup(0);
      ssJoinIntermediateConfig.setInputSerializer(serializer, 0);


      // output
      ssJoinIntermediateConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      ssJoinIntermediateConfig.setOutputComparator(comparator, 0);


      ssJoinIntermediateConfig.setOutputSerializer(serializer);


      // driver
      ssJoinIntermediateConfig.setDriver(JoinWithSolutionSetSecondDriver.class);
      ssJoinIntermediateConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
      ssJoinIntermediateConfig.setDriverComparator(comparator, 0);
      ssJoinIntermediateConfig.setDriverPairComparator(pairComparator);
      
      ssJoinIntermediateConfig.setStubWrapper(new UserCodeClassWrapper<UpdateComponentIdMatch>(UpdateComponentIdMatch.class));
    }


    // -------------------------- ws tail --------------------------------
    AbstractJobVertex wsTail = JobGraphUtils.createTask(IterationTailPactTask.class, "IterationWorksetTail", jobGraph, numSubTasks);
    TaskConfig wsTailConfig = new TaskConfig(wsTail.getConfiguration());
    {
      wsTailConfig.setIterationId(ITERATION_ID);
      wsTailConfig.setIsWorksetIteration();
      wsTailConfig.setIsWorksetUpdate();


      // inputs and driver
      wsTailConfig.addInputToGroup(0);
      wsTailConfig.setInputSerializer(serializer, 0);


      // output
      wsTailConfig.setOutputSerializer(serializer);


      // the driver
      wsTailConfig.setDriver(CollectorMapDriver.class);
      wsTailConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      wsTailConfig.setStubWrapper(new UserCodeClassWrapper<DummyMapper>(DummyMapper.class));
    }


    // --------------- the wiring ---------------------


    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(edges, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);


    JobGraphUtils.connect(head, intermediate, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    intermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);


    JobGraphUtils.connect(intermediate, ssJoinIntermediate, ChannelType.NETWORK, DistributionPattern.POINTWISE);
    ssJoinIntermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(ssJoinIntermediate, wsTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    wsTailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);


    
    SlotSharingGroup sharingGroup = new SlotSharingGroup();
    vertices.setSlotSharingGroup(sharingGroup);
    edges.setSlotSharingGroup(sharingGroup);
    head.setSlotSharingGroup(sharingGroup);
    intermediate.setSlotSharingGroup(sharingGroup);
    ssJoinIntermediate.setSlotSharingGroup(sharingGroup);
    wsTail.setSlotSharingGroup(sharingGroup);
    output.setSlotSharingGroup(sharingGroup);

View Full Code Here

  }


  private static <T extends InputFormat<?,?>> InputFormatVertex createInput(UserCodeWrapper<T> stub, String name, JobGraph graph,
      int degreeOfParallelism)
  {
    InputFormatVertex inputVertex = new InputFormatVertex(name);
    graph.addVertex(inputVertex);
    
    inputVertex.setInvokableClass(DataSourceTask.class);
    inputVertex.setParallelism(degreeOfParallelism);


    TaskConfig inputConfig = new TaskConfig(inputVertex.getConfiguration());
    inputConfig.setStubWrapper(stub);
    
    return inputVertex;
  }

View Full Code Here

    // --------------------------------------------------------------------------------------------------------------
    // 1. VERTICES
    // --------------------------------------------------------------------------------------------------------------


    // - input -----------------------------------------------------------------------------------------------------
    InputFormatVertex input = JobGraphUtils.createInput(
      new PointInFormat(), inputPath, "Input", jobGraph, numSubTasks);
    TaskConfig inputConfig = new TaskConfig(input.getConfiguration());
    {
      inputConfig.setOutputSerializer(serializer);
      inputConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
    }


    // - head ------------------------------------------------------------------------------------------------------
    AbstractJobVertex head = JobGraphUtils.createTask(IterationHeadPactTask.class, "Iteration Head", jobGraph, numSubTasks);
    TaskConfig headConfig = new TaskConfig(head.getConfiguration());
    {
      headConfig.setIterationId(ITERATION_ID);


      // input to iteration head
      headConfig.addInputToGroup(0);
      headConfig.setInputSerializer(serializer, 0);
      headConfig.setInputLocalStrategy(0, LocalStrategy.NONE);
      headConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);


      // output into iteration
      headConfig.setOutputSerializer(serializer);
      headConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
      headConfig.setOutputComparator(comparator, 0);


      // final output
      TaskConfig headFinalOutConfig = new TaskConfig(new Configuration());
      headFinalOutConfig.setOutputSerializer(serializer);
      headFinalOutConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      headConfig.setIterationHeadFinalOutputConfig(headFinalOutConfig);


      // the sync
      headConfig.setIterationHeadIndexOfSyncOutput(2);


      // driver
      headConfig.setDriver(CollectorMapDriver.class);
      headConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      headConfig.setStubWrapper(new UserCodeClassWrapper<DummyMapper>(DummyMapper.class));


      // back channel
      headConfig.setRelativeBackChannelMemory(1.0);
    }


    // - tail ------------------------------------------------------------------------------------------------------
    AbstractJobVertex tail = JobGraphUtils.createTask(IterationTailPactTask.class, "Chained Iteration Tail", jobGraph, numSubTasks);
    TaskConfig tailConfig = new TaskConfig(tail.getConfiguration());
    {
      tailConfig.setIterationId(ITERATION_ID);


      // inputs and driver
      tailConfig.addInputToGroup(0);
      tailConfig.setInputSerializer(serializer, 0);


      // output
      tailConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      tailConfig.setOutputSerializer(serializer);


      // the driver
      tailConfig.setDriver(GroupReduceDriver.class);
      tailConfig.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
      tailConfig.setDriverComparator(comparator, 0);
      tailConfig.setStubWrapper(new UserCodeClassWrapper<DummyReducer>(DummyReducer.class));


      // chained mapper
      TaskConfig chainedMapperConfig = new TaskConfig(new Configuration());
      chainedMapperConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      chainedMapperConfig.setStubWrapper(new UserCodeClassWrapper<IncrementCoordinatesMapper>(
        IncrementCoordinatesMapper.class));


      chainedMapperConfig.setInputLocalStrategy(0, LocalStrategy.NONE);
      chainedMapperConfig.setInputSerializer(serializer, 0);


      chainedMapperConfig.setOutputSerializer(serializer);


      chainedMapperConfig.setIsWorksetUpdate();


      tailConfig.addChainedTask(ChainedCollectorMapDriver.class, chainedMapperConfig, "Chained ID Mapper");
    }


    // - output ----------------------------------------------------------------------------------------------------
    OutputFormatVertex output = JobGraphUtils.createFileOutput(jobGraph, "Output", numSubTasks);
    TaskConfig outputConfig = new TaskConfig(output.getConfiguration());
    {
      outputConfig.addInputToGroup(0);
      outputConfig.setInputSerializer(serializer, 0);


      outputConfig.setStubWrapper(new UserCodeClassWrapper<PointOutFormat>(PointOutFormat.class));
      outputConfig.setStubParameter(FileOutputFormat.FILE_PARAMETER_KEY, outputPath);
    }


    // - sync ------------------------------------------------------------------------------------------------------
    AbstractJobVertex sync = JobGraphUtils.createSync(jobGraph, numSubTasks);
    TaskConfig syncConfig = new TaskConfig(sync.getConfiguration());
    syncConfig.setNumberOfIterations(maxIterations);
    syncConfig.setIterationId(ITERATION_ID);


    // --------------------------------------------------------------------------------------------------------------
    // 2. EDGES
    // --------------------------------------------------------------------------------------------------------------
    JobGraphUtils.connect(input, head, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(head, tail, ChannelType.IN_MEMORY, DistributionPattern.BIPARTITE);
    tailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);


    JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);


    // --------------------------------------------------------------------------------------------------------------
    // 3. INSTANCE SHARING
    // --------------------------------------------------------------------------------------------------------------
    
    SlotSharingGroup sharingGroup = new SlotSharingGroup();
    
    input.setSlotSharingGroup(sharingGroup);
    head.setSlotSharingGroup(sharingGroup);
    tail.setSlotSharingGroup(sharingGroup);
    output.setSlotSharingGroup(sharingGroup);
    sync.setSlotSharingGroup(sharingGroup);

View Full Code Here

0 1

TOP

Related Classes of org.apache.flink.runtime.jobgraph.InputFormatVertex

org.apache.flink.compiler.plantranslate.NepheleJobGraphGenerator

org.apache.flink.runtime.operators.util.TaskConfig

org.apache.flink.test.broadcastvars.BroadcastVarsNepheleITCase

org.apache.flink.test.broadcastvars.KMeansIterativeNepheleITCase

org.apache.flink.test.iterative.nephele.ConnectedComponentsNepheleITCase

org.apache.flink.test.iterative.nephele.customdanglingpagerank.CustomCompensatableDanglingPageRank

org.apache.flink.test.iterative.nephele.customdanglingpagerank.CustomCompensatableDanglingPageRankWithCombiner

org.apache.flink.test.iterative.nephele.danglingpagerank.CompensatableDanglingPageRank

org.apache.flink.test.iterative.nephele.IterationWithChainingNepheleITCase

org.apache.flink.test.iterative.nephele.JobGraphUtils

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.