Package org.apache.pig.newplan.logical.relational

Examples of org.apache.pig.newplan.logical.relational.LOCogroup


  }

    }

    LOCogroup createGroupOp() {
        return new LOCogroup( plan );
    }
View Full Code Here


            "D = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);" +
            "C = cogroup A by id, B by id, D by id using 'merge';" +
            "store C into 'output';";
            LogicalPlan lp = Util.buildLp(pigServer, query);
            Operator op = lp.getSinks().get(0);
            LOCogroup cogrp = (LOCogroup)lp.getPredecessors(op).get(0);
            assertEquals(LOCogroup.GROUPTYPE.MERGE, cogrp.getGroupType());

            PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
            pc.connect();
            PhysicalPlan phyP = Util.buildPp(pigServer, query);
            PhysicalOperator phyOp = phyP.getLeaves().get(0);
View Full Code Here

        "D = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);" +
        "C = cogroup A by id inner, B by id, D by id inner using 'merge';" +
        "store C into 'output';";
        LogicalPlan lp = Util.buildLp(pigServer, query);
        Operator op = lp.getSinks().get(0);
        LOCogroup cogrp = (LOCogroup)lp.getPredecessors(op).get(0);
        assertEquals(LOCogroup.GROUPTYPE.MERGE, cogrp.getGroupType());

        PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
        pc.connect();
        boolean exceptionCaught = false;
        try{
View Full Code Here

        return expectedSch;
    }

    private LOCogroup checkNumExpressionPlansForGroup(LogicalPlan lp, int numPlans) {
        Class<?> groupClass = org.apache.pig.newplan.logical.relational.LOCogroup.class;
        LOCogroup grp = (LOCogroup) NewLogicalPlanUtil.getRelOpFromPlan(lp, groupClass);

        for( int inp : grp.getExpressionPlans().keySet()){
            List<LogicalExpressionPlan> plans = grp.getExpressionPlans().get(inp);
            assertEquals("number of group-by plans", numPlans, plans.size());
        }

        return grp;
    }
View Full Code Here

            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1))) ;
            load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema2))) ;
            load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema2))) ;
   
            LOCogroup cogroup1 = new LOCogroup(plan);
           
            // Create expression inner plan #1 of input #1
            LogicalExpressionPlan innerPlan11 = new LogicalExpressionPlan() ;
            ProjectExpression project111 = new ProjectExpression(innerPlan11, 0, 0, cogroup1) ;
            ConstantExpression const111 = new ConstantExpression(innerPlan11, 26F) ;
            SubtractExpression subtract111 = new SubtractExpression(innerPlan11, project111, const111) ;
   

   
            // Create expression inner plan #2 of input #1
            LogicalExpressionPlan innerPlan21 = new LogicalExpressionPlan() ;
            ProjectExpression project211 = new ProjectExpression(innerPlan21, 0, 0, cogroup1) ;
            ProjectExpression project212 = new ProjectExpression(innerPlan21, 0, 1, cogroup1) ;
   
            AddExpression add211 = new AddExpression(innerPlan21, project211, project212) ;

  
            // Create expression inner plan #1 of input #2
            LogicalExpressionPlan innerPlan12 = new LogicalExpressionPlan() ;
            ProjectExpression project121 = new ProjectExpression(innerPlan12, 1, 0, cogroup1) ;
            ConstantExpression const121 = new ConstantExpression(innerPlan12, 26) ;
            SubtractExpression subtract121 = new SubtractExpression(innerPlan12, project121, const121) ;
   

            // Create expression inner plan #2 of input #2
            LogicalExpressionPlan innerPlan22 = new LogicalExpressionPlan() ;
            ConstantExpression const122 = new ConstantExpression(innerPlan22, 26) ;
//            innerPlan22.add(const122) ;
   
            // Create Cogroup
            ArrayList<LogicalRelationalOperator> inputs = new ArrayList<LogicalRelationalOperator>() ;
            inputs.add(load1) ;
            inputs.add(load2) ;
   
            MultiMap<Integer, LogicalExpressionPlan> maps
                                = new MultiMap<Integer, LogicalExpressionPlan>() ;
            maps.put(0, innerPlan11);
            maps.put(0, innerPlan21);
            maps.put(1, innerPlan12);
            maps.put(1, innerPlan22);
   
            boolean[] isInner = new boolean[inputs.size()] ;
            for (int i=0; i < isInner.length ; i++) {
                isInner[i] = false ;
            }
   
            cogroup1.setInnerFlags(isInner);
            cogroup1.setExpressionPlans(maps);
   
            // construct the main plan
            plan.add(load1) ;
            plan.add(load2) ;
            plan.add(cogroup1) ;
   
            plan.connect(load1, cogroup1);
            plan.connect(load2, cogroup1);
   
            CompilationMessageCollector collector = new CompilationMessageCollector() ;
            TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
            typeChecker.visit()
            printMessageCollector(collector) ;
            //printTypeGraph(plan) ;
   
            if (collector.hasError()) {
                throw new AssertionError("Expect no error") ;
            }
           
            // check outer schema
            LogicalSchema endResultSchema = cogroup1.getSchema() ;
   
            // Tuple group column
            assertEquals(endResultSchema.getField(0).type, DataType.TUPLE) ;
            assertEquals(endResultSchema.getField(0).schema.getField(0).type, DataType.DOUBLE) ;
            assertEquals(endResultSchema.getField(0).schema.getField(1).type, DataType.LONG);
View Full Code Here

   
            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1))) ; ;
            load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema2))) ;
   
            LOCogroup cogroup1 = new LOCogroup(plan);
                   
            // Create expression inner plan #1 of input #1
            LogicalExpressionPlan innerPlan11 = new LogicalExpressionPlan() ;
            ProjectExpression project111 = new ProjectExpression(innerPlan11, 0, 0, cogroup1) ;
            ConstantExpression const111 = new ConstantExpression(innerPlan11, 26F) ;
            SubtractExpression subtract111 = new SubtractExpression(innerPlan11, project111, const111) ;

   
            // Create expression inner plan #1 of input #2
            LogicalExpressionPlan innerPlan12 = new LogicalExpressionPlan() ;
            ProjectExpression project121 = new ProjectExpression(innerPlan12, 1, 0, cogroup1) ;
            ConstantExpression const121 = new ConstantExpression(innerPlan12, 26) ;
            SubtractExpression subtract121 = new SubtractExpression(innerPlan12, project121, const121) ;
   

   
            // Create Cogroup
            ArrayList<LogicalRelationalOperator> inputs = new ArrayList<LogicalRelationalOperator>() ;
            inputs.add(load1) ;
            inputs.add(load2) ;
   
            MultiMap<Integer, LogicalExpressionPlan> maps
                                = new MultiMap<Integer, LogicalExpressionPlan>() ;
            maps.put(0, innerPlan11);
            maps.put(1, innerPlan12);
   
            boolean[] isInner = new boolean[inputs.size()] ;
            for (int i=0; i < isInner.length ; i++) {
                isInner[i] = false ;
            }
   
           
            cogroup1.setInnerFlags(isInner);
            cogroup1.setExpressionPlans(maps);
   
            // construct the main plan
            plan.add(load1) ;
            plan.add(load2) ;
            plan.add(cogroup1) ;
   
            plan.connect(load1, cogroup1);
            plan.connect(load2, cogroup1);
   
            CompilationMessageCollector collector = new CompilationMessageCollector() ;
            TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
            typeChecker.visit()
            printMessageCollector(collector) ;
            //printTypeGraph(plan) ;
   
            if (collector.hasError()) {
                throw new AssertionError("Expect no error") ;
            }
   
            // check outer schema
            LogicalSchema endResultSchema = cogroup1.getSchema() ;
   
            // Tuple group column
            assertEquals(endResultSchema.getField(0).type, DataType.DOUBLE) ;
   
            assertEquals(endResultSchema.getField(1).type, DataType.BAG) ;
View Full Code Here

   
            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1))) ;
            load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema2))) ;
   
            LOCogroup cogroup1 = new LOCogroup(plan);
            // Create expression inner plan #1
            LogicalExpressionPlan innerPlan11 = new LogicalExpressionPlan() ;
            ProjectExpression project111 = new ProjectExpression(innerPlan11, 0, 0, cogroup1) ;
            ConstantExpression const111 = new ConstantExpression(innerPlan11, 26F) ;
            SubtractExpression subtract111 = new SubtractExpression(innerPlan11, project111, const111) ;

   
            // Create expression inner plan #2
            LogicalExpressionPlan innerPlan12 = new LogicalExpressionPlan() ;
            ConstantExpression const121 = new ConstantExpression(innerPlan12, 26) ;
//            innerPlan12.add(const121) ;
   
            // Create Cogroup
            ArrayList<LogicalRelationalOperator> inputs = new ArrayList<LogicalRelationalOperator>() ;
            inputs.add(load1) ;
            inputs.add(load2) ;
   
            MultiMap<Integer, LogicalExpressionPlan> maps
                                = new MultiMap<Integer, LogicalExpressionPlan>() ;
            maps.put(0, innerPlan11);
            maps.put(1, innerPlan12);
   
            boolean[] isInner = new boolean[inputs.size()] ;
            for (int i=0; i < isInner.length ; i++) {
                isInner[i] = false ;
            }
   
           
            cogroup1.setInnerFlags(isInner);
            cogroup1.setExpressionPlans(maps);
   
            // construct the main plan
            plan.add(load1) ;
            plan.add(load2) ;
            plan.add(cogroup1) ;
   
            plan.connect(load1, cogroup1);
            plan.connect(load2, cogroup1);
   
            CompilationMessageCollector collector = new CompilationMessageCollector() ;
            TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
            typeChecker.visit()
            printMessageCollector(collector) ;
            //printTypeGraph(plan) ;
   
            if (collector.hasError()) {
                throw new AssertionError("Expect no error") ;
            }
   
            // check outer schema
            LogicalSchema endResultSchema = cogroup1.getSchema() ;
   
            // Tuple group column
            assertEquals(endResultSchema.getField(0).type, DataType.FLOAT) ;
   
            assertEquals(endResultSchema.getField(1).type, DataType.BAG) ;
View Full Code Here

      String query = "A = LOAD '" + INPUT_FILE + "' as (id, name, grade);" +
                     "B = group A by id using 'collected';" +
                     "Store B into 'y';";
        LogicalPlan lp = Util.buildLp(pigServer, query );
        LOStore store = (LOStore)lp.getSinks().get(0);
        LOCogroup grp = (LOCogroup)lp.getPredecessors( store ).get(0);
        Assert.assertEquals( LOCogroup.GROUPTYPE.COLLECTED, grp.getGroupType() );
    }
View Full Code Here

        String query = "A = LOAD '" + INPUT_FILE + "' as (id, name, grade);" +
                       "B = group A all using 'regular';" +
                       "Store B into 'y';";
        LogicalPlan lp = Util.buildLp(pigServer, query );
        LOStore store = (LOStore)lp.getSinks().get(0);
        LOCogroup grp = (LOCogroup)lp.getPredecessors( store ).get(0);
        Assert.assertEquals(LOCogroup.GROUPTYPE.REGULAR, grp.getGroupType());
    }
View Full Code Here

            "D = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);" +
            "C = cogroup A by id, B by id, D by id using 'merge';" +
            "store C into 'output';";
            LogicalPlan lp = Util.buildLp(pigServer, query);
            Operator op = lp.getSinks().get(0);
            LOCogroup cogrp = (LOCogroup)lp.getPredecessors(op).get(0);
            assertEquals(LOCogroup.GROUPTYPE.MERGE, cogrp.getGroupType());

            PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
            pc.connect();
            PhysicalPlan phyP = Util.buildPp(pigServer, query);
            PhysicalOperator phyOp = phyP.getLeaves().get(0);
View Full Code Here

TOP

Related Classes of org.apache.pig.newplan.logical.relational.LOCogroup

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.