Examples of eu.stratosphere.api.java.record.operators.JoinOperator

Package eu.stratosphere.api.java.record.operators

Examples of eu.stratosphere.api.java.record.operators.JoinOperator

eu.stratosphere.api.java.record.operators.JoinOperator
JoinOperator that applies a {@link JoinFunction} to each pair of records from both inputsthat have matching keys.

    // Sc3 generates N y values to be evaluated with the polynomial identified by id
    FileDataSource sc3 = new FileDataSource(new CsvInputFormat(), sc3Path);
    CsvInputFormat.configureRecordFormat(sc3).fieldDelimiter(' ').field(StringValue.class, 0).field(IntValue.class, 1);


    // Jn1 matches x and y values on id and emits (id, x, y) triples
    JoinOperator jn1 = JoinOperator.builder(Jn1.class, StringValue.class, 0, 0).input1(sc2).input2(sc3).build();


    // Jn2 matches polynomial and arguments by id, computes p = min(P(x),P(y)) and emits (id, p) tuples
    JoinOperator jn2 = JoinOperator.builder(Jn2.class, StringValue.class, 0, 0).input1(jn1).input2(sc1).build();


    // Mp1 selects (id, x, y) triples where x = y and broadcasts z (=x=y) to Mp2
    MapOperator mp1 = MapOperator.builder(Mp1.class).input(jn1).build();


    // Mp2 filters out all p values which can be divided by z

View Full Code Here

        new ContractITCaseInputFormat(), rightInPath);
    DelimitedInputFormat.configureDelimitedFormat(input_right)
      .recordDelimiter('\n');
    input_right.setDegreeOfParallelism(config.getInteger("MatchTest#NoSubtasks", 1));


    JoinOperator testMatcher = JoinOperator.builder(new TestMatcher(), StringValue.class, 0, 0)
      .build();
    testMatcher.setDegreeOfParallelism(config.getInteger("MatchTest#NoSubtasks", 1));
    testMatcher.getParameters().setString(PactCompiler.HINT_LOCAL_STRATEGY,
        config.getString("MatchTest#LocalStrategy", ""));
    if (config.getString("MatchTest#ShipStrategy", "").equals("BROADCAST_FIRST")) {
      testMatcher.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY_FIRST_INPUT,
          PactCompiler.HINT_SHIP_STRATEGY_BROADCAST);
      testMatcher.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY_SECOND_INPUT,
          PactCompiler.HINT_SHIP_STRATEGY_FORWARD);
    } else if (config.getString("MatchTest#ShipStrategy", "").equals("BROADCAST_SECOND")) {
      testMatcher.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY_FIRST_INPUT,
          PactCompiler.HINT_SHIP_STRATEGY_FORWARD);
      testMatcher.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY_SECOND_INPUT,
          PactCompiler.HINT_SHIP_STRATEGY_BROADCAST);
    } else {
      testMatcher.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY,
          config.getString("MatchTest#ShipStrategy", ""));
    }


    FileDataSink output = new FileDataSink(
        new ContractITCaseOutputFormat(), resultPath);
    output.setDegreeOfParallelism(1);


    output.setInput(testMatcher);
    testMatcher.setFirstInput(input_left);
    testMatcher.setSecondInput(input_right);


    return new Plan(output);
  }

View Full Code Here

    TPCHQuery3 query = new TPCHQuery3();
    Plan p = query.getPlan(DEFAULT_PARALLELISM_STRING, IN_FILE, IN_FILE, OUT_FILE);
    
    // set compiler hints
    OperatorResolver cr = getContractResolver(p);
    JoinOperator match = cr.getNode("JoinLiO");
    match.getCompilerHints().setFilterFactor(100f);
    
    testQueryGeneric(100l*1024*1024*1024*1024, 100l*1024*1024*1024*1024, 0.05f, 100f, false, true, false, false, true);
  }

View Full Code Here

      // set statistics
      OperatorResolver cr = getContractResolver(p);
      FileDataSource ordersSource = cr.getNode(ORDERS);
      FileDataSource lineItemSource = cr.getNode(LINEITEM);
      MapOperator mapper = cr.getNode(MAPPER_NAME);
      JoinOperator joiner = cr.getNode(JOIN_NAME);
      setSourceStatistics(ordersSource, orderSize, 100f);
      setSourceStatistics(lineItemSource, lineitemSize, 140f);
      mapper.getCompilerHints().setAvgOutputRecordSize(16f);
      mapper.getCompilerHints().setFilterFactor(orderSelectivity);
      joiner.getCompilerHints().setFilterFactor(joinSelectivity);
      
      // compile
      final OptimizedPlan plan = compileWithStats(p);
      final OptimizerPlanNodeResolver or = getOptimizerPlanNodeResolver(plan);

View Full Code Here

    
    // create DataSourceContract for the edges
    FileDataSource edges = new FileDataSource(new CsvInputFormat(' ', LongValue.class, LongValue.class), edgeInput, "Edges");


    // create CrossOperator for distance computation
    JoinOperator joinWithNeighbors = JoinOperator.builder(new NeighborWithComponentIDJoin(), LongValue.class, 0, 0)
        .input1(iteration.getWorkset())
        .input2(edges)
        .name("Join Candidate Id With Neighbor")
        .build();


    // create ReduceOperator for finding the nearest cluster centers
    ReduceOperator minCandidateId = ReduceOperator.builder(new MinimumComponentIDReduce(), LongValue.class, 0)
        .input(joinWithNeighbors)
        .name("Find Minimum Candidate Id")
        .build();
    
    // create CrossOperator for distance computation
    JoinOperator updateComponentId = JoinOperator.builder(new UpdateComponentIdMatchMirrored(), LongValue.class, 0, 0)
        .input1(iteration.getSolutionSet())
        .input2(minCandidateId)
        .name("Update Component Id")
        .build();

View Full Code Here

  public void testBranchingSourceMultipleTimes() {
    try {
      // construct the plan
      FileDataSource sourceA = new FileDataSource(new DummyInputFormat(), IN_FILE);
      
      JoinOperator mat1 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(sourceA)
        .input2(sourceA)
        .build();
      JoinOperator mat2 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(sourceA)
        .input2(mat1)
        .build();
      JoinOperator mat3 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(sourceA)
        .input2(mat2)
        .build();
      JoinOperator mat4 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(sourceA)
        .input2(mat3)
        .build();
      JoinOperator mat5 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(sourceA)
        .input2(mat4)
        .build();
      
      MapOperator ma = MapOperator.builder(new IdentityMap()).input(sourceA).build();
      
      JoinOperator mat6 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(ma)
        .input2(ma)
        .build();
      JoinOperator mat7 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(ma)
        .input2(mat6)
        .build();
      JoinOperator mat8 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(ma)
        .input2(mat7)
        .build();
      JoinOperator mat9 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(ma)
        .input2(mat8)
        .build();
      JoinOperator mat10 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(ma)
        .input2(mat9)
        .build();
      
      CoGroupOperator co = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0, 0)

View Full Code Here

      CoGroupOperator co = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
        .input1(sourceA)
        .input2(sourceB)
        .build();
      MapOperator ma = MapOperator.builder(new IdentityMap()).input(co).build();
      JoinOperator mat1 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(sourceB)
        .input2(sourceC)
        .build();
      JoinOperator mat2 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(ma)
        .input2(mat1)
        .build();
      ReduceOperator r = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
        .input(ma)

View Full Code Here

      ReduceOperator reduce1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
        .input(map1)
        .name("Reduce 1")
        .build();
      
      @SuppressWarnings("unchecked")
      JoinOperator match1 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(sourceB, sourceB, sourceC)
        .input2(sourceC)
        .name("Match 1")
        .build();

View Full Code Here

    try {
      // construct the plan
      FileDataSource source1 = new FileDataSource(new DummyInputFormat(), IN_FILE);
      FileDataSource source2 = new FileDataSource(new DummyInputFormat(), IN_FILE);
      
      JoinOperator mat1 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(source1)
        .input2(source2)
        .name("Match 1")
        .build();
      
      MapOperator ma1 = MapOperator.builder(new IdentityMap()).input(mat1).name("Map1").build();
      
      ReduceOperator r1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
        .input(ma1)
        .name("Reduce 1")
        .build();
      
      ReduceOperator r2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
        .input(mat1)
        .name("Reduce 2")
        .build();
      
      MapOperator ma2 = MapOperator.builder(new IdentityMap()).input(mat1).name("Map 2").build();
      
      MapOperator ma3 = MapOperator.builder(new IdentityMap()).input(ma2).name("Map 3").build();
      
      @SuppressWarnings("unchecked")
      JoinOperator mat2 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(r1, r2, ma2, ma3)
        .input2(ma2)
        .name("Match 2")
        .build();
      mat2.setParameter(PactCompiler.HINT_LOCAL_STRATEGY, PactCompiler.HINT_LOCAL_STRATEGY_MERGE);
      
      FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, mat2);
      
      
      // return the PACT plan

View Full Code Here

    CrossOperator nextWorkset = CrossOperator.builder(DummyCrossStub.class).name("Next workset").
        input1(iteration.getWorkset()).
        input2(sourceC).
        build();


    JoinOperator solutionSetDelta = JoinOperator.builder(DummyMatchStub.class, LongValue.class,0,0).
        name("Next solution set.").
        input1(nextWorkset).
        input2(iteration.getSolutionSet()).
        build();

View Full Code Here

0 1 2 3 4

TOP

Related Classes of eu.stratosphere.api.java.record.operators.JoinOperator

eu.stratosphere.pact.compiler.BranchingPlansCompilerTest

eu.stratosphere.pact.compiler.DOPChangeTest

eu.stratosphere.pact.compiler.WorksetIterationsRecordApiCompilerTest

eu.stratosphere.test.broadcastvars.BroadcastBranchingITCase

eu.stratosphere.test.cancelling.MatchJoinCancelingITCase

eu.stratosphere.test.compiler.examples.RelationalQueryCompilerTest

eu.stratosphere.test.compiler.iterations.ConnectedComponentsTest

eu.stratosphere.test.iterative.CoGroupConnectedComponentsITCase

eu.stratosphere.test.iterative.ConnectedComponentsWithDeferredUpdateITCase

eu.stratosphere.test.iterative.ConnectedComponentsWithSolutionSetFirstITCase

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.