Package org.apache.flink.api.java.record.operators

Examples of org.apache.flink.api.java.record.operators.MapOperator


    FileDataSource edges = new FileDataSource(new EdgeInputFormat(), edgeInput, "Input Edges");
    edges.setParameter(EdgeInputFormat.ID_DELIMITER_CHAR, delimiter);

    // =========================== Vertex Degree ============================
   
    MapOperator projectEdge = MapOperator.builder(new ProjectEdge())
        .input(edges).name("Project Edge").build();
   
    ReduceOperator edgeCounter = ReduceOperator.builder(new CountEdges(), IntValue.class, 0)
        .input(projectEdge).name("Count Edges for Vertex").build();
   
    ReduceOperator countJoiner = ReduceOperator.builder(new JoinCountsAndUniquify(), IntValue.class, 0)
        .keyField(IntValue.class, 1)
        .input(edgeCounter).name("Join Counts").build();
   
   
    // =========================== Triangle Enumeration ============================
   
    MapOperator toLowerDegreeEdge = MapOperator.builder(new ProjectToLowerDegreeVertex())
        .input(countJoiner).name("Select lower-degree Edge").build();
   
    MapOperator projectOutCounts = MapOperator.builder(new ProjectOutCounts())
        .input(countJoiner).name("Project out Counts").build();

    ReduceOperator buildTriads = ReduceOperator.builder(new BuildTriads(), IntValue.class, 0)
        .input(toLowerDegreeEdge).name("Build Triads").build();
View Full Code Here


      new IntTupleDataInFormat(), this.nationInputPath, "\"nation\" source");
    //nationInput.setOutputContract(UniqueKey.class);
//    nationInput.getCompilerHints().setAvgNumValuesPerKey(1);

    /* Filter on part's name, project values to NULL: */
    MapOperator filterPart = MapOperator.builder(PartFilter.class)
      .name("filterParts")
      .build();

    /* Map to change the key element of partsupp, project value to (supplycost, suppkey): */
    MapOperator mapPartsupp = MapOperator.builder(PartsuppMap.class)
      .name("mapPartsupp")
      .build();

    /* Map to extract the year from order: */
    MapOperator mapOrder = MapOperator.builder(OrderMap.class)
      .name("mapOrder")
      .build();

    /* Project value to (partkey, suppkey, quantity, price = extendedprice*(1-discount)): */
    MapOperator mapLineItem = MapOperator.builder(LineItemMap.class)
      .name("proj.Partsupp")
      .build();

    /* - change the key of supplier to nationkey, project value to suppkey */
    MapOperator mapSupplier = MapOperator.builder(SupplierMap.class)
      .name("proj.Partsupp")
      .build();

    /* Equijoin on partkey of part and partsupp: */
    JoinOperator partsJoin = JoinOperator.builder(PartJoin.class, IntValue.class, 0, 0)
      .name("partsJoin")
      .build();

    /* Equijoin on orderkey of orders and lineitem: */
    JoinOperator orderedPartsJoin =
      JoinOperator.builder(OrderedPartsJoin.class, IntValue.class, 0, 0)
      .name("orderedPartsJoin")
      .build();

    /* Equijoin on nationkey of supplier and nation: */
    JoinOperator suppliersJoin =
      JoinOperator.builder(SuppliersJoin.class, IntValue.class, 0, 0)
      .name("suppliersJoin")
      .build();

    /* Equijoin on (partkey,suppkey) of parts and orderedParts: */
    JoinOperator filteredPartsJoin =
      JoinOperator.builder(FilteredPartsJoin.class, IntPair.class, 0, 0)
      .name("filteredPartsJoin")
      .build();

    /* Equijoin on suppkey of filteredParts and suppliers: */
    JoinOperator partListJoin =
      JoinOperator.builder(PartListJoin.class, IntValue.class , 0, 0)
      .name("partlistJoin")
      .build();

    /* Aggregate sum(amount) by (nation,year): */
    ReduceOperator sumAmountAggregate =
      ReduceOperator.builder(AmountAggregate.class, StringIntPair.class, 0)
      .name("groupyBy")
      .build();

    /* Connect input filters: */
    filterPart.setInput(partInput);
    mapPartsupp.setInput(partSuppInput);
    mapOrder.setInput(ordersInput);
    mapLineItem.setInput(lineItemInput);
    mapSupplier.setInput(supplierInput);

    /* Connect equijoins: */
    partsJoin.setFirstInput(filterPart);
    partsJoin.setSecondInput(mapPartsupp);
    orderedPartsJoin.setFirstInput(mapOrder);
 
View Full Code Here

    // input is {word, count} pair
    FileDataSource source = new FileDataSource(new TextInputFormat(), dataInput, "Input Lines");

    //do a selection using cached file
    MapOperator mapper = MapOperator.builder(new TokenizeLine())
      .input(source)
      .name("Tokenize Lines")
      .build();

    FileDataSink out = new FileDataSink(new CsvOutputFormat(), output, mapper, "Selection");
View Full Code Here

      .fieldDelimiter('|')
      .field(LongValue.class, 0)
      .field(DoubleValue.class, 5);

    // create MapOperator for filtering Orders tuples
    MapOperator filterO1 = MapOperator.builder(new FilterO())
      .name("FilterO")
      .input(orders1)
      .build();
    // filter configuration
    filterO1.setParameter(TPCHQuery3.YEAR_FILTER, 1993);
    filterO1.setParameter(TPCHQuery3.PRIO_FILTER, "5");
    filterO1.getCompilerHints().setFilterFactor(0.05f);
   
    // create MapOperator for filtering Orders tuples
    MapOperator filterO2 = MapOperator.builder(new FilterO())
      .name("FilterO")
      .input(orders2)
      .build();
    // filter configuration
    filterO2.setParameter(TPCHQuery3.YEAR_FILTER, 1993);
    filterO2.setParameter(TPCHQuery3.PRIO_FILTER, "5");

    // create JoinOperator for joining Orders and LineItems
    @SuppressWarnings("unchecked")
    JoinOperator joinLiO = JoinOperator.builder(new JoinLiO(), LongValue.class, 0, 0)
      .input1(filterO2, filterO1)
View Full Code Here

    FileDataSource customers = new FileDataSource(new IntTupleDataInFormat(), customersPath, "Customers");

    FileDataSource nations = new FileDataSource(new IntTupleDataInFormat(), nationsPath, "Nations");


    MapOperator mapO = MapOperator.builder(FilterO.class)
      .name("FilterO")
      .build();

    MapOperator mapLi = MapOperator.builder(FilterLI.class)
      .name("FilterLi")
      .build();

    MapOperator projectC = MapOperator.builder(ProjectC.class)
      .name("ProjectC")
      .build();

    MapOperator projectN = MapOperator.builder(ProjectN.class)
      .name("ProjectN")
      .build();

    JoinOperator joinOL = JoinOperator.builder(JoinOL.class, IntValue.class, 0, 0)
      .name("JoinOL")
      .build();

    JoinOperator joinCOL = JoinOperator.builder(JoinCOL.class, IntValue.class, 0, 0)
      .name("JoinCOL")
      .build();

    JoinOperator joinNCOL = JoinOperator.builder(JoinNCOL.class, IntValue.class, 4, 0)
      .name("JoinNCOL")
      .build();

    ReduceOperator reduce = ReduceOperator.builder(Sum.class)
      .keyField(IntValue.class, 0)
      .keyField(StringValue.class, 1)
      .keyField(StringValue.class, 3)
      .keyField(StringValue.class, 4)
      .keyField(StringValue.class, 5)
      .keyField(StringValue.class, 6)
      .keyField(StringValue.class, 7)
      .name("Reduce")
      .build();

    FileDataSink result = new FileDataSink(new TupleOutputFormat(), resultPath, "Output");

    result.setInput(reduce);
   
    reduce.setInput(joinNCOL);
   
    joinNCOL.setFirstInput(joinCOL);
    joinNCOL.setSecondInput(projectN);
   
    joinCOL.setFirstInput(projectC);
    joinCOL.setSecondInput(joinOL);
   
    joinOL.setFirstInput(mapO);
    joinOL.setSecondInput(mapLi);
   
    projectC.setInput(customers);
    projectN.setInput(nations);
    mapLi.setInput(lineitems);
    mapO.setInput(orders);

    // return the PACT plan
    Plan p = new Plan(result, "TPCH Q10");
View Full Code Here

    edges.setParameter(EdgeWithDegreesInputFormat.VERTEX_DELIMITER_CHAR, '|');
    edges.setParameter(EdgeWithDegreesInputFormat.DEGREE_DELIMITER_CHAR, ',');

    // =========================== Triangle Enumeration ============================
   
    MapOperator toLowerDegreeEdge = MapOperator.builder(new ProjectToLowerDegreeVertex())
        .input(edges)
        .name("Select lower-degree Edge")
        .build();
   
    MapOperator projectOutCounts = MapOperator.builder(new ProjectOutCounts())
        .input(edges)
        .name("Project to vertex Ids only")
        .build();

    ReduceOperator buildTriads = ReduceOperator.builder(new BuildTriads(), IntValue.class, 0)
View Full Code Here

  }
 
  private void checkWordCountWithSortedSink(boolean estimates) {
    try {
      FileDataSource sourceNode = new FileDataSource(new TextInputFormat(), IN_FILE, "Input Lines");
      MapOperator mapNode = MapOperator.builder(new TokenizeLine())
        .input(sourceNode)
        .name("Tokenize Lines")
        .build();
      ReduceOperator reduceNode = ReduceOperator.builder(new CountWords(), StringValue.class, 0)
        .input(mapNode)
View Full Code Here

    try {
      // set statistics
      OperatorResolver cr = getContractResolver(p);
      FileDataSource ordersSource = cr.getNode(ORDERS);
      FileDataSource lineItemSource = cr.getNode(LINEITEM);
      MapOperator mapper = cr.getNode(MAPPER_NAME);
      JoinOperator joiner = cr.getNode(JOIN_NAME);
      setSourceStatistics(ordersSource, orderSize, 100f);
      setSourceStatistics(lineItemSource, lineitemSize, 140f);
      mapper.getCompilerHints().setAvgOutputRecordSize(16f);
      mapper.getCompilerHints().setFilterFactor(orderSelectivity);
      joiner.getCompilerHints().setFilterFactor(joinSelectivity);
     
      // compile
      final OptimizedPlan plan = compileWithStats(p);
      final OptimizerPlanNodeResolver or = getOptimizerPlanNodeResolver(plan);
View Full Code Here

    // Jn2 matches polynomial and arguments by id, computes p = min(P(x),P(y)) and emits (id, p) tuples
    JoinOperator jn2 = JoinOperator.builder(Jn2.class, StringValue.class, 0, 0).input1(jn1).input2(sc1).build();

    // Mp1 selects (id, x, y) triples where x = y and broadcasts z (=x=y) to Mp2
    MapOperator mp1 = MapOperator.builder(Mp1.class).input(jn1).build();

    // Mp2 filters out all p values which can be divided by z
    MapOperator mp2 = MapOperator.builder(Mp2.class).setBroadcastVariable("z", mp1).input(jn2).build();

    FileDataSink output = new FileDataSink(new ContractITCaseOutputFormat(), resultPath);
    output.setDegreeOfParallelism(1);
    output.setInput(mp2);
View Full Code Here

      int numSubTasks, String verticesInput, String edgeInput, String output, int maxIterations)
  {
    // create DataSourceContract for the vertices
    FileDataSource initialVertices = new FileDataSource(new CsvInputFormat(' ', LongValue.class), verticesInput, "Vertices");
   
    MapOperator verticesWithId = MapOperator.builder(DuplicateLongMap.class).input(initialVertices).name("Assign Vertex Ids").build();
   
    DeltaIteration iteration = new DeltaIteration(0, "Connected Components Iteration");
    iteration.setInitialSolutionSet(verticesWithId);
    iteration.setInitialWorkset(verticesWithId);
    iteration.setMaximumNumberOfIterations(maxIterations);
View Full Code Here

TOP

Related Classes of org.apache.flink.api.java.record.operators.MapOperator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.