Examples of org.apache.flink.api.java.ExecutionEnvironment

org.apache.flink.api.java.ExecutionEnvironment
The ExecutionEnviroment is the context in which a program is executed. A {@link LocalEnvironment} will cause execution in the current JVM, a{@link RemoteEnvironment} will cause execution on a remote setup.
The environment provides methods to control the job execution (such as setting the parallelism) and to interact with the outside world (data access).
Please note that the execution environment needs strong type information for the input and return types of all operations that are executed. This means that the environments needs to know that the return value of an operation is for example a Tuple of String and Integer. Because the Java compiler throws much of the generic type information away, most methods attempt to re- obtain that information using reflection. In certain cases, it may be necessary to manually supply that information to some of the methods. @see LocalEnvironment @see RemoteEnvironment

  
  @Override
  protected void testProgram() throws Exception {
      
    // set up execution environment
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    
    // read vertex and edge data
    DataSet<Long> vertices = env.fromElements(ConnectedComponentsData.getEnumeratingVertices(NUM_VERTICES).split("\n"))
        .map(new VertexParser());
    
    DataSet<Tuple2<Long, Long>> edges = env.fromElements(ConnectedComponentsData.getRandomOddEvenEdges(NUM_EDGES, NUM_VERTICES, SEED).split("\n"))
        .flatMap(new EdgeParser());
    
    // assign the initial components (equal to the vertex id)
    DataSet<Tuple2<Long, Long>> verticesWithInitialId = vertices.map(new DuplicateValue<Long>());
        
    // open a delta iteration
    DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration =
        verticesWithInitialId.iterateDelta(verticesWithInitialId, 100, 0);
    
    // apply the step logic: join with the edges, select the minimum neighbor, update if the component of the candidate is smaller
    DataSet<Tuple2<Long, Long>> changes = iteration
        .getWorkset().join(edges).where(0).equalTo(0).with(new NeighborWithComponentIDJoin())
        .coGroup(iteration.getSolutionSet()).where(0).equalTo(0)
        .with(new MinIdAndUpdate());


    // close the delta iteration (delta and new workset are identical)
    DataSet<Tuple2<Long, Long>> result = iteration.closeWith(changes, changes);
    
    
    // emit result
    List<Tuple2<Long,Long>> resutTuples = new ArrayList<Tuple2<Long,Long>>();
    result.output(new LocalCollectionOutputFormat<Tuple2<Long,Long>>(resutTuples));
    
    env.execute();
  }

View Full Code Here

        
        /*
         * check correctness of cross on two tuple inputs 
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.getSmall5TupleDataSet(env);
        DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
        DataSet<Tuple2<Integer, String>> crossDs = ds.cross(ds2).with(new Tuple5Cross());
        
        crossDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return "0,HalloHallo\n" +
            "1,HalloHallo Welt\n" +
            "2,HalloHallo Welt wie\n" +
            "1,Hallo WeltHallo\n" +
            "2,Hallo WeltHallo Welt\n" +
            "3,Hallo WeltHallo Welt wie\n" +
            "2,Hallo Welt wieHallo\n" +
            "3,Hallo Welt wieHallo Welt\n" +
            "4,Hallo Welt wieHallo Welt wie\n";
      }
      case 2: {
        
        /*
         * check correctness of cross if UDF returns left input object
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.getSmall3TupleDataSet(env);
        DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
        DataSet<Tuple3<Integer, Long, String>> crossDs = ds.cross(ds2).with(new Tuple3ReturnLeft());
        
        crossDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return "1,1,Hi\n" +
            "1,1,Hi\n" +
            "1,1,Hi\n" +
            "2,2,Hello\n" +
            "2,2,Hello\n" +
            "2,2,Hello\n" +
            "3,2,Hello world\n" +
            "3,2,Hello world\n" +
            "3,2,Hello world\n";
        
      }
      case 3: {
        
        /*
         * check correctness of cross if UDF returns right input object
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.getSmall3TupleDataSet(env);
        DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
        DataSet<Tuple5<Integer, Long, Integer, String, Long>> crossDs = ds.cross(ds2).with(new Tuple5ReturnRight());
        
        crossDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return "1,1,0,Hallo,1\n" +
            "1,1,0,Hallo,1\n" +
            "1,1,0,Hallo,1\n" +
            "2,2,1,Hallo Welt,2\n" +
            "2,2,1,Hallo Welt,2\n" +
            "2,2,1,Hallo Welt,2\n" +
            "2,3,2,Hallo Welt wie,1\n" +
            "2,3,2,Hallo Welt wie,1\n" +
            "2,3,2,Hallo Welt wie,1\n";
        
      }
      case 4: {
        
        /*
         * check correctness of cross with broadcast set
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Integer> intDs = CollectionDataSets.getIntegerDataSet(env);
        
        DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.getSmall5TupleDataSet(env);
        DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
        DataSet<Tuple3<Integer, Integer, Integer>> crossDs = ds.cross(ds2).with(new Tuple5CrossBC()).withBroadcastSet(intDs, "ints");
        
        crossDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return "2,0,55\n" +
            "3,0,55\n" +
            "3,0,55\n" +
            "3,0,55\n" +
            "4,1,55\n" +
            "4,2,55\n" +
            "3,0,55\n" +
            "4,2,55\n" +
            "4,4,55\n";
      }
      case 5: {
        
        /*
         * check correctness of crossWithHuge (only correctness of result -> should be the same as with normal cross)
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.getSmall5TupleDataSet(env);
        DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
        DataSet<Tuple2<Integer, String>> crossDs = ds.crossWithHuge(ds2).with(new Tuple5Cross());
        
        crossDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return "0,HalloHallo\n" +
            "1,HalloHallo Welt\n" +
            "2,HalloHallo Welt wie\n" +
            "1,Hallo WeltHallo\n" +
            "2,Hallo WeltHallo Welt\n" +
            "3,Hallo WeltHallo Welt wie\n" +
            "2,Hallo Welt wieHallo\n" +
            "3,Hallo Welt wieHallo Welt\n" +
            "4,Hallo Welt wieHallo Welt wie\n";
        
      }
      case 6: {
        
        /*
         * check correctness of crossWithTiny (only correctness of result -> should be the same as with normal cross)
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.getSmall5TupleDataSet(env);
        DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
        DataSet<Tuple2<Integer, String>> crossDs = ds.crossWithTiny(ds2).with(new Tuple5Cross());
        
        crossDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return "0,HalloHallo\n" +
            "1,HalloHallo Welt\n" +
            "2,HalloHallo Welt wie\n" +
            "1,Hallo WeltHallo\n" +
            "2,Hallo WeltHallo Welt\n" +
            "3,Hallo WeltHallo Welt wie\n" +
            "2,Hallo Welt wieHallo\n" +
            "3,Hallo Welt wieHallo Welt\n" +
            "4,Hallo Welt wieHallo Welt wie\n";
        
      }
      case 7: {


      /*
       * project cross on a tuple input 1
       */


        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();


        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.getSmall3TupleDataSet(env);
        DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
        DataSet<Tuple6<String, Long, String, Integer, Long, Long>> crossDs = ds.cross(ds2)
          .projectFirst(2, 1)
          .projectSecond(3)
          .projectFirst(0)
          .projectSecond(4,1)
          .types(String.class, Long.class, String.class, Integer.class, Long.class, Long.class);


        crossDs.writeAsCsv(resultPath);
        env.execute();


        // return expected result
        return "Hi,1,Hallo,1,1,1\n" +
          "Hi,1,Hallo Welt,1,2,2\n" +
          "Hi,1,Hallo Welt wie,1,1,3\n" +
          "Hello,2,Hallo,2,1,1\n" +
          "Hello,2,Hallo Welt,2,2,2\n" +
          "Hello,2,Hallo Welt wie,2,1,3\n" +
          "Hello world,2,Hallo,3,1,1\n" +
          "Hello world,2,Hallo Welt,3,2,2\n" +
          "Hello world,2,Hallo Welt wie,3,1,3\n";


      }
      case 8: {


      /*
       * project cross on a tuple input 2
       */


          final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();


          DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.getSmall3TupleDataSet(env);
          DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
          DataSet<Tuple6<String, String, Long, Long, Long,Integer>> crossDs = ds.cross(ds2)
            .projectSecond(3)
            .projectFirst(2, 1)
            .projectSecond(4,1)
            .projectFirst(0)
            .types(String.class, String.class, Long.class, Long.class, Long.class, Integer.class);


          crossDs.writeAsCsv(resultPath);
          env.execute();


          // return expected result
          return "Hallo,Hi,1,1,1,1\n" +
            "Hallo Welt,Hi,1,2,2,1\n" +
            "Hallo Welt wie,Hi,1,1,3,1\n" +
            "Hallo,Hello,2,1,1,2\n" +
            "Hallo Welt,Hello,2,2,2,2\n" +
            "Hallo Welt wie,Hello,2,1,3,2\n" +
            "Hallo,Hello world,2,1,1,3\n" +
            "Hallo Welt,Hello world,2,2,2,3\n" +
            "Hallo Welt wie,Hello world,2,1,3,3\n";


      }
      case 9: {
        /*
         * check correctness of default cross
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.getSmall3TupleDataSet(env);
        DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
        DataSet<Tuple2<Tuple3<Integer, Long, String>, Tuple5<Integer, Long, Integer, String, Long>>> crossDs = ds.cross(ds2);
        
        crossDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return "(1,1,Hi),(2,2,1,Hallo Welt,2)\n" +
            "(1,1,Hi),(1,1,0,Hallo,1)\n" +
            "(1,1,Hi),(2,3,2,Hallo Welt wie,1)\n" +
            "(2,2,Hello),(2,2,1,Hallo Welt,2)\n" +
            "(2,2,Hello),(1,1,0,Hallo,1)\n" +
            "(2,2,Hello),(2,3,2,Hallo Welt wie,1)\n" +
            "(3,2,Hello world),(2,2,1,Hallo Welt,2)\n" +
            "(3,2,Hello world),(1,1,0,Hallo,1)\n" +
            "(3,2,Hello world),(2,3,2,Hallo Welt wie,1)\n";
        
      }


      case 10: {
        
        /*
         * check correctness of cross on two custom type inputs
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<CustomType> ds = CollectionDataSets.getSmallCustomTypeDataSet(env);
        DataSet<CustomType> ds2 = CollectionDataSets.getSmallCustomTypeDataSet(env);
        DataSet<CustomType> crossDs = ds.cross(ds2).with(new CustomTypeCross());
        
        crossDs.writeAsText(resultPath);
        env.execute();
        
        // return expected result
        return "1,0,HiHi\n"
            + "2,1,HiHello\n"
            + "2,2,HiHello world\n"
            + "2,1,HelloHi\n"
            + "4,2,HelloHello\n"
            + "4,3,HelloHello world\n"
            + "2,2,Hello worldHi\n"
            + "4,3,Hello worldHello\n"
            + "4,4,Hello worldHello world";
      }
      
      case 11: {
        
        /*
         * check correctness of cross a tuple input and a custom type input
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.getSmall5TupleDataSet(env);
        DataSet<CustomType> ds2 = CollectionDataSets.getSmallCustomTypeDataSet(env);
        DataSet<Tuple3<Integer, Long, String>> crossDs = ds.cross(ds2).with(new MixedCross());
        
        crossDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return "2,0,HalloHi\n" +
            "3,0,HalloHello\n" +
            "3,0,HalloHello world\n" +

View Full Code Here


  private List<Long> result = new ArrayList<Long>();
  
  @Override
  protected void testProgram() throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    
    IterativeDataSet<Long> iteration = env.generateSequence(1, 10).iterate(100);
    iteration.closeWith(iteration)
      .output(new LocalCollectionOutputFormat<Long>(result));
    
    env.execute();
  }

View Full Code Here


  
  @Override
  protected void testProgram() throws Exception {
    // set up execution environment
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
    // read vertex and edge data
    DataSet<Tuple1<Long>> vertices = env.readCsvFile(verticesPath).types(Long.class);
    
    DataSet<Tuple2<Long, Long>> edges = env.readCsvFile(edgesPath).fieldDelimiter(' ').types(Long.class, Long.class)
                        .flatMap(new UndirectEdge());
        
    // assign the initial components (equal to the vertex id)
    DataSet<Tuple2<Long, Long>> verticesWithInitialId = vertices.map(new DuplicateValue<Long>());
            
    // open a delta iteration
    DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration =
        verticesWithInitialId.iterateDelta(verticesWithInitialId, 100, 0);
    iteration.setSolutionSetUnManaged(true);
        
    // apply the step logic: join with the edges, select the minimum neighbor, update if the component of the candidate is smaller
    DataSet<Tuple2<Long, Long>> changes = iteration.getWorkset().join(edges).where(0).equalTo(0).with(new NeighborWithComponentIDJoin())
        .groupBy(0).aggregate(Aggregations.MIN, 1)
        .join(iteration.getSolutionSet()).where(0).equalTo(0)
        .with(new ComponentIdFilter());


    // close the delta iteration (delta and new workset are identical)
    DataSet<Tuple2<Long, Long>> result = iteration.closeWith(changes, changes);
        
    result.writeAsCsv(resultPath, "\n", " ");
    
    // execute program
    env.execute("Connected Components Example");
  }

View Full Code Here

      case 1: {
        /*
         * First-n on ungrouped data set
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
        DataSet<Tuple1<Integer>> seven = ds.first(7).map(new OneMapper()).sum(0);
        
        seven.writeAsText(resultPath);
        env.execute();
        
        // return expected result
        return "(7)\n";
      }
      case 2: {
        /*
         * First-n on grouped data set
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
        DataSet<Tuple2<Long, Integer>> first = ds.groupBy(1).first(4)
                              .map(new OneMapper2()).groupBy(0).sum(1);
        
        first.writeAsText(resultPath);
        env.execute();
        
        // return expected result
        return "(1,1)\n(2,2)\n(3,3)\n(4,4)\n(5,4)\n(6,4)\n";
      }
      case 3: {
        /*
         * First-n on grouped and sorted data set
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
        DataSet<Tuple2<Long, Integer>> first = ds.groupBy(1).sortGroup(0, Order.DESCENDING).first(3)
                              .project(1,0).types(Long.class, Integer.class);
        
        first.writeAsText(resultPath);
        env.execute();
        
        // return expected result
        return "(1,1)\n"
            + "(2,3)\n(2,2)\n"
            + "(3,6)\n(3,5)\n(3,4)\n"

View Full Code Here

    Assert.assertEquals(distinctWords, res.getAccumulatorResult("distinct-words"));
  }


  @Override
  protected void testProgram() throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    
    DataSet<String> input = env.readTextFile(dataPath); 
    
    input.flatMap(new TokenizeLine())
      .groupBy(0)
      .reduceGroup(new CountWords())
      .writeAsCsv(resultPath, "\n", " ");
    
    this.result = env.execute();
  }

View Full Code Here

      case 1: {
        /*
         * Projection with tuple fields indexes
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.get5TupleDataSet(env);
        DataSet<Tuple3<String, Long, Integer>> projDs = ds.
            project(3,4,2).types(String.class, Long.class, Integer.class);
        projDs.writeAsCsv(resultPath);
        
        env.execute();
        return "Hallo,1,0\n" +
            "Hallo Welt,2,1\n" +
            "Hallo Welt wie,1,2\n" +
            "Hallo Welt wie gehts?,2,3\n" +
            "ABC,2,4\n" +

View Full Code Here

//  }
  
  public static void main(String[] args) throws Exception {
    String inputPath = args[0];
    
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    
    DataSet<MyUser> input = env.createInput(new AvroInputFormat<MyUser>(new Path(inputPath), MyUser.class));
  
    DataSet<Tuple2<String, MyUser>> result = input.map(new NameExtractor()).groupBy(0).reduce(new NameGrouper());
    
    result.output(new DiscardingOuputFormat<Tuple2<String,MyUser>>());
    env.execute();
  }

View Full Code Here

public class BroadcastVariablePipelinebreakerTest extends CompilerTestBase {


  @Test
  public void testNoBreakerForIndependentVariable() {
    try {
      ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
      
      DataSet<String> source1 = env.fromElements("test");
      DataSet<String> source2 = env.fromElements("test");
      
      source1.map(new IdentityMapper<String>()).withBroadcastSet(source2, "some name").print();
      
      Plan p = env.createProgramPlan();
      OptimizedPlan op = compileNoStats(p);
      
      SinkPlanNode sink = op.getDataSinks().iterator().next();
      SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();

View Full Code Here

  }
  
   @Test
  public void testBreakerForDependentVariable() {
      try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<String> source1 = env.fromElements("test");
        
        source1.map(new IdentityMapper<String>()).map(new IdentityMapper<String>()).withBroadcastSet(source1, "some name").print();
        
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
        
        SinkPlanNode sink = op.getDataSinks().iterator().next();
        SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.flink.api.java.ExecutionEnvironment

org.apache.flink.api.avro.AvroOutputFormatTest

org.apache.flink.api.common.cache.DistributedCache.DistributedCacheEntry

org.apache.flink.api.common.operators.CollectionExecutionIterationTest

org.apache.flink.api.common.operators.CollectionExecutionWithBroadcastVariableTest

org.apache.flink.api.java.functions.SemanticPropertiesProjectionTest

org.apache.flink.api.java.functions.SemanticPropertiesTranslationTest

org.apache.flink.api.java.io.AvroInputFormatTypeExtractionTest

org.apache.flink.api.java.io.CsvReader

org.apache.flink.api.java.io.jdbc.example.JDBCExample

org.apache.flink.api.java.io.TextInputFormat

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.