Examples of org.apache.pig.newplan.logical.relational.LOCogroup

Package org.apache.pig.newplan.logical.relational

Examples of org.apache.pig.newplan.logical.relational.LOCogroup

org.apache.pig.newplan.logical.relational.LOCogroup

        "D = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);" +
        "C = cogroup A by id inner, B by id, D by id inner using 'merge';" +
        "store C into 'output';";
        LogicalPlan lp = Util.buildLp(pigServer, query);
        Operator op = lp.getSinks().get(0);
        LOCogroup cogrp = (LOCogroup)lp.getPredecessors(op).get(0);
        assertEquals(LOCogroup.GROUPTYPE.MERGE, cogrp.getGroupType());


        PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
        pc.connect();
        boolean exceptionCaught = false;
        try{

View Full Code Here

            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1)));
            load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema2)));
            load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema2)));


            LOCogroup cogroup1 = new LOCogroup(plan);


            // Create expression inner plan #1 of input #1
            LogicalExpressionPlan innerPlan11 = new LogicalExpressionPlan();
            ProjectExpression project111 = new ProjectExpression(innerPlan11, 0, 0, cogroup1);
            ConstantExpression const111 = new ConstantExpression(innerPlan11, 26F);
            SubtractExpression subtract111 = new SubtractExpression(innerPlan11, project111, const111);


            // Create expression inner plan #2 of input #1
            LogicalExpressionPlan innerPlan21 = new LogicalExpressionPlan();
            ProjectExpression project211 = new ProjectExpression(innerPlan21, 0, 0, cogroup1);
            ProjectExpression project212 = new ProjectExpression(innerPlan21, 0, 1, cogroup1);


            AddExpression add211 = new AddExpression(innerPlan21, project211, project212);


            // Create expression inner plan #1 of input #2
            LogicalExpressionPlan innerPlan12 = new LogicalExpressionPlan();
            ProjectExpression project121 = new ProjectExpression(innerPlan12, 1, 0, cogroup1);
            ConstantExpression const121 = new ConstantExpression(innerPlan12, 26);
            SubtractExpression subtract121 = new SubtractExpression(innerPlan12, project121, const121);


            // Create expression inner plan #2 of input #2
            LogicalExpressionPlan innerPlan22 = new LogicalExpressionPlan();
            ConstantExpression const122 = new ConstantExpression(innerPlan22, 26);
//            innerPlan22.add(const122);


            // Create Cogroup
            ArrayList<LogicalRelationalOperator> inputs = new ArrayList<LogicalRelationalOperator>();
            inputs.add(load1);
            inputs.add(load2);


            MultiMap<Integer, LogicalExpressionPlan> maps
                                = new MultiMap<Integer, LogicalExpressionPlan>();
            maps.put(0, innerPlan11);
            maps.put(0, innerPlan21);
            maps.put(1, innerPlan12);
            maps.put(1, innerPlan22);


            boolean[] isInner = new boolean[inputs.size()];
            for (int i=0; i < isInner.length; i++) {
                isInner[i] = false;
            }


            cogroup1.setInnerFlags(isInner);
            cogroup1.setExpressionPlans(maps);


            // construct the main plan
            plan.add(load1);
            plan.add(load2);
            plan.add(cogroup1);


            plan.connect(load1, cogroup1);
            plan.connect(load2, cogroup1);


            CompilationMessageCollector collector = new CompilationMessageCollector();
            TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
            typeChecker.visit();
            printMessageCollector(collector);
            //printTypeGraph(plan);


            if (collector.hasError()) {
                throw new AssertionError("Expect no error");
            }


            // check outer schema
            LogicalSchema endResultSchema = cogroup1.getSchema();


            // Tuple group column
            assertEquals(DataType.TUPLE, endResultSchema.getField(0).type);
            assertEquals(DataType.DOUBLE,endResultSchema.getField(0).schema.getField(0).type);
            assertEquals(DataType.LONG, endResultSchema.getField(0).schema.getField(1).type);

View Full Code Here


            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1)));;
            load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema2)));


            LOCogroup cogroup1 = new LOCogroup(plan);


            // Create expression inner plan #1 of input #1
            LogicalExpressionPlan innerPlan11 = new LogicalExpressionPlan();
            ProjectExpression project111 = new ProjectExpression(innerPlan11, 0, 0, cogroup1);
            ConstantExpression const111 = new ConstantExpression(innerPlan11, 26F);
            SubtractExpression subtract111 = new SubtractExpression(innerPlan11, project111, const111);


            // Create expression inner plan #1 of input #2
            LogicalExpressionPlan innerPlan12 = new LogicalExpressionPlan();
            ProjectExpression project121 = new ProjectExpression(innerPlan12, 1, 0, cogroup1);
            ConstantExpression const121 = new ConstantExpression(innerPlan12, 26);
            SubtractExpression subtract121 = new SubtractExpression(innerPlan12, project121, const121);


            // Create Cogroup
            ArrayList<LogicalRelationalOperator> inputs = new ArrayList<LogicalRelationalOperator>();
            inputs.add(load1);
            inputs.add(load2);


            MultiMap<Integer, LogicalExpressionPlan> maps
                                = new MultiMap<Integer, LogicalExpressionPlan>();
            maps.put(0, innerPlan11);
            maps.put(1, innerPlan12);


            boolean[] isInner = new boolean[inputs.size()];
            for (int i=0; i < isInner.length; i++) {
                isInner[i] = false;
            }




            cogroup1.setInnerFlags(isInner);
            cogroup1.setExpressionPlans(maps);


            // construct the main plan
            plan.add(load1);
            plan.add(load2);
            plan.add(cogroup1);


            plan.connect(load1, cogroup1);
            plan.connect(load2, cogroup1);


            CompilationMessageCollector collector = new CompilationMessageCollector();
            TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
            typeChecker.visit();
            printMessageCollector(collector);
            //printTypeGraph(plan);


            if (collector.hasError()) {
                throw new AssertionError("Expect no error");
            }


            // check outer schema
            LogicalSchema endResultSchema = cogroup1.getSchema();


            // Tuple group column
            assertEquals(DataType.DOUBLE, endResultSchema.getField(0).type);


            assertEquals(DataType.BAG, endResultSchema.getField(1).type);

View Full Code Here


            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1)));
            load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema2)));


            LOCogroup cogroup1 = new LOCogroup(plan);
            // Create expression inner plan #1
            LogicalExpressionPlan innerPlan11 = new LogicalExpressionPlan();
            ProjectExpression project111 = new ProjectExpression(innerPlan11, 0, 0, cogroup1);
            ConstantExpression const111 = new ConstantExpression(innerPlan11, 26F);
            SubtractExpression subtract111 = new SubtractExpression(innerPlan11, project111, const111);




            // Create expression inner plan #2
            LogicalExpressionPlan innerPlan12 = new LogicalExpressionPlan();
            ConstantExpression const121 = new ConstantExpression(innerPlan12, 26);
//            innerPlan12.add(const121);


            // Create Cogroup
            ArrayList<LogicalRelationalOperator> inputs = new ArrayList<LogicalRelationalOperator>();
            inputs.add(load1);
            inputs.add(load2);


            MultiMap<Integer, LogicalExpressionPlan> maps
                                = new MultiMap<Integer, LogicalExpressionPlan>();
            maps.put(0, innerPlan11);
            maps.put(1, innerPlan12);


            boolean[] isInner = new boolean[inputs.size()];
            for (int i=0; i < isInner.length; i++) {
                isInner[i] = false;
            }




            cogroup1.setInnerFlags(isInner);
            cogroup1.setExpressionPlans(maps);


            // construct the main plan
            plan.add(load1);
            plan.add(load2);
            plan.add(cogroup1);


            plan.connect(load1, cogroup1);
            plan.connect(load2, cogroup1);


            CompilationMessageCollector collector = new CompilationMessageCollector();
            TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
            typeChecker.visit();
            printMessageCollector(collector);
            //printTypeGraph(plan);


            if (collector.hasError()) {
                throw new AssertionError("Expect no error");
            }


            // check outer schema
            LogicalSchema endResultSchema = cogroup1.getSchema();


            // Tuple group column
            assertEquals(DataType.FLOAT, endResultSchema.getField(0).type);


            assertEquals(DataType.BAG, endResultSchema.getField(1).type);

View Full Code Here

        LogicalPlan lp;
        String query = "a = load 'myfile' as (name, age, gpa);" +
                   "b = group a by (name, age);";
        lp = buildPlan( query + "store b into 'output';");
        Operator store = lp.getSinks().get(0);
        LOCogroup cogroup = (LOCogroup) lp.getPredecessors(store).get(0);


        LogicalSchema actual = cogroup.getSchema();
        System.out.println( actual.toString( false ) );


        Assert.assertTrue(  actual.toString( false ).equals( "group:tuple(name:bytearray,age:bytearray),a:bag{:tuple(name:bytearray,age:bytearray,gpa:bytearray)}" ) );


        lp = buildPlan(query +

View Full Code Here

        String query = "a = load 'myfile' as (name:Chararray, age:Int, gpa:Float);" +
                       "b = group a by (name, age);" +
                       "store b into 'output';";
        lp = buildPlan( query );
        Operator store = lp.getSinks().get(0);
        LOCogroup cogroup = (LOCogroup) lp.getPredecessors(store).get(0);


        Schema.FieldSchema nameFs = new Schema.FieldSchema("name", DataType.CHARARRAY);
        Schema.FieldSchema ageFs = new Schema.FieldSchema("age", DataType.INTEGER);
        Schema.FieldSchema gpaFs = new Schema.FieldSchema("gpa", DataType.FLOAT);


        Schema groupSchema = new Schema(nameFs);
        groupSchema.add(ageFs);
        Schema.FieldSchema groupFs = new Schema.FieldSchema("group", groupSchema, DataType.TUPLE);


        Schema loadSchema = new Schema(nameFs);
        loadSchema.add(ageFs);
        loadSchema.add(gpaFs);


        Schema.FieldSchema bagFs = new Schema.FieldSchema("a", loadSchema, DataType.BAG);


        Schema cogroupExpectedSchema = new Schema(groupFs);
        cogroupExpectedSchema.add(bagFs);
        Assert.assertTrue(cogroup.getSchema().toString(false).equals("group:tuple(name:chararray,age:int),a:bag{:tuple(name:chararray,age:int,gpa:float)}"));
    }

View Full Code Here

                       "c = foreach b {d = order a by $1; generate flatten(d), MAX(a.age) as max_age;};" +
                       "store c into 'output';";
        LogicalPlan lp = buildPlan(query);
        Operator store = lp.getSinks().get(0);
        LOForEach foreach = (LOForEach)lp.getPredecessors(store).get(0);
        LOCogroup cogroup = (LOCogroup) lp.getPredecessors(foreach).get(0);
        String s = cogroup.getSchema().toString(false);
        Assert.assertTrue( s.equals("group:bytearray,a:bag{:tuple(name:bytearray,age:bytearray,gpa:bytearray)}"));
        s = foreach.getSchema().toString(false);
        Assert.assertTrue( s.equals("d::name:bytearray,d::age:bytearray,d::gpa:bytearray,max_age:double"));
    }

View Full Code Here

        String query = "a = load 'one' as (name, age, gpa);" + 
        "b = group a by *;" +
        "store b into 'output';";
        LogicalPlan lp = buildPlan(query);
        Operator store = lp.getSinks().get(0);
        LOCogroup cogroup = (LOCogroup)lp.getPredecessors(store).get(0);
        String s = cogroup.getSchema().toString(false);
        Assert.assertTrue(s.equals("group:tuple(name:bytearray,age:bytearray,gpa:bytearray),a:bag{:tuple(name:bytearray,age:bytearray,gpa:bytearray)}"));
    }

View Full Code Here

                       "b = load 'two' as (first_name, enrol_age, high_school_gpa);" +
                       "c = group a by *, b by *;" +
                       "store c into 'output';";
        LogicalPlan lp = buildPlan(query);
        Operator store = lp.getSinks().get(0);
        LOCogroup cogroup = (LOCogroup)lp.getPredecessors(store).get(0);
        String s = cogroup.getSchema().toString(false);
        Assert.assertTrue(s.equals("group:tuple(name:bytearray,age:bytearray,gpa:bytearray),a:bag{:tuple(name:bytearray,age:bytearray,gpa:bytearray)},b:bag{:tuple(first_name:bytearray,enrol_age:bytearray,high_school_gpa:bytearray)}"));
    }

View Full Code Here

        return expectedSch;
    }


    private LOCogroup checkNumExpressionPlansForGroup(LogicalPlan lp, int numPlans) {
        Class<?> groupClass = org.apache.pig.newplan.logical.relational.LOCogroup.class;
        LOCogroup grp = (LOCogroup) NewLogicalPlanUtil.getRelOpFromPlan(lp, groupClass);


        for( int inp : grp.getExpressionPlans().keySet()){
            List<LogicalExpressionPlan> plans = grp.getExpressionPlans().get(inp);
            assertEquals("number of group-by plans", numPlans, plans.size());
        }


        return grp;
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.pig.newplan.logical.relational.LOCogroup

org.apache.pig.newplan.logical.expression.LogicalExpression

org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema

org.apache.pig.newplan.logical.rules.GroupByConstParallelSetter

org.apache.pig.newplan.logical.rules.GroupByConstParallelSetter$GroupAllParallelSetterTransformer

org.apache.pig.newplan.logical.rules.PushUpFilter$PushUpFilterTransformer

org.apache.pig.parser.LogicalPlanBuilder

org.apache.pig.pen.LineageTrimmingVisitor

org.apache.pig.test.TestCollectedGroup

org.apache.pig.test.TestLogicalPlanBuilder

org.apache.pig.test.TestLogicalPlanMigrationVisitor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.