Package org.apache.pig.newplan.logical.relational

Examples of org.apache.pig.newplan.logical.relational.LOCogroup


                return;

            Operator childOp = plan.getPredecessors(op).get(0);
            if (op instanceof LOForEach && childOp instanceof LOCogroup)
            {
                LOCogroup cg = (LOCogroup) childOp;
                for (Operator input : cg.getInputs(plan)) {
                    AffinityGroups.put(input, eg.getEqClasses());
                    Lineage.put(input, eg.getLineage());
                }
            } else {
                List<Operator> childOps = plan.getPredecessors(op);
View Full Code Here


        pigServer.setBatchOn();
        pigServer.registerQuery("a = load 'tmp1' as (foo, bar);");
        pigServer.registerQuery("b = group a by foo;");
        pigServer.registerQuery("store b into 'tmp2';");
       
        LOCogroup op = (LOCogroup)TestPigStats.getLogicalPlan(pigServer).findByAlias("b");
        assertFalse(op.isPinnedOption(LOCogroup.OPTION_GROUPTYPE));
        pigServer.discardBatch();
       
        pigServer.setBatchOn();
        pigServer.registerQuery("a = load 'tmp' as (foo, bar);");
        pigServer.registerQuery("b = group a by foo using 'collected';");
        pigServer.registerQuery("store b into 'tmp2';");
        op = (LOCogroup)TestPigStats.getLogicalPlan(pigServer).findByAlias("b");
        assertTrue(op.isPinnedOption(LOCogroup.OPTION_GROUPTYPE));
        pigServer.discardBatch();
    }
View Full Code Here

            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1))) ;
            load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema2))) ;
            load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema2))) ;
   
            LOCogroup cogroup1 = new LOCogroup(plan);
           
            // Create expression inner plan #1 of input #1
            LogicalExpressionPlan innerPlan11 = new LogicalExpressionPlan() ;
            ProjectExpression project111 = new ProjectExpression(innerPlan11, 0, 0, cogroup1) ;
            ConstantExpression const111 = new ConstantExpression(innerPlan11, 26F) ;
            SubtractExpression subtract111 = new SubtractExpression(innerPlan11, project111, const111) ;
   

   
            // Create expression inner plan #2 of input #1
            LogicalExpressionPlan innerPlan21 = new LogicalExpressionPlan() ;
            ProjectExpression project211 = new ProjectExpression(innerPlan21, 0, 0, cogroup1) ;
            ProjectExpression project212 = new ProjectExpression(innerPlan21, 0, 1, cogroup1) ;
   
            AddExpression add211 = new AddExpression(innerPlan21, project211, project212) ;

  
            // Create expression inner plan #1 of input #2
            LogicalExpressionPlan innerPlan12 = new LogicalExpressionPlan() ;
            ProjectExpression project121 = new ProjectExpression(innerPlan12, 1, 0, cogroup1) ;
            ConstantExpression const121 = new ConstantExpression(innerPlan12, 26) ;
            SubtractExpression subtract121 = new SubtractExpression(innerPlan12, project121, const121) ;
   

            // Create expression inner plan #2 of input #2
            LogicalExpressionPlan innerPlan22 = new LogicalExpressionPlan() ;
            ConstantExpression const122 = new ConstantExpression(innerPlan22, 26) ;
//            innerPlan22.add(const122) ;
   
            // Create Cogroup
            ArrayList<LogicalRelationalOperator> inputs = new ArrayList<LogicalRelationalOperator>() ;
            inputs.add(load1) ;
            inputs.add(load2) ;
   
            MultiMap<Integer, LogicalExpressionPlan> maps
                                = new MultiMap<Integer, LogicalExpressionPlan>() ;
            maps.put(0, innerPlan11);
            maps.put(0, innerPlan21);
            maps.put(1, innerPlan12);
            maps.put(1, innerPlan22);
   
            boolean[] isInner = new boolean[inputs.size()] ;
            for (int i=0; i < isInner.length ; i++) {
                isInner[i] = false ;
            }
   
            cogroup1.setInnerFlags(isInner);
            cogroup1.setExpressionPlans(maps);
   
            // construct the main plan
            plan.add(load1) ;
            plan.add(load2) ;
            plan.add(cogroup1) ;
   
            plan.connect(load1, cogroup1);
            plan.connect(load2, cogroup1);
   
            CompilationMessageCollector collector = new CompilationMessageCollector() ;
            TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
            typeChecker.visit()
            printMessageCollector(collector) ;
            //printTypeGraph(plan) ;
   
            if (collector.hasError()) {
                throw new AssertionError("Expect no error") ;
            }
           
            // check outer schema
            LogicalSchema endResultSchema = cogroup1.getSchema() ;
   
            // Tuple group column
            assertEquals(endResultSchema.getField(0).type, DataType.TUPLE) ;
            assertEquals(endResultSchema.getField(0).schema.getField(0).type, DataType.DOUBLE) ;
            assertEquals(endResultSchema.getField(0).schema.getField(1).type, DataType.LONG);
View Full Code Here

   
            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1))) ; ;
            load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema2))) ;
   
            LOCogroup cogroup1 = new LOCogroup(plan);
                   
            // Create expression inner plan #1 of input #1
            LogicalExpressionPlan innerPlan11 = new LogicalExpressionPlan() ;
            ProjectExpression project111 = new ProjectExpression(innerPlan11, 0, 0, cogroup1) ;
            ConstantExpression const111 = new ConstantExpression(innerPlan11, 26F) ;
            SubtractExpression subtract111 = new SubtractExpression(innerPlan11, project111, const111) ;

   
            // Create expression inner plan #1 of input #2
            LogicalExpressionPlan innerPlan12 = new LogicalExpressionPlan() ;
            ProjectExpression project121 = new ProjectExpression(innerPlan12, 1, 0, cogroup1) ;
            ConstantExpression const121 = new ConstantExpression(innerPlan12, 26) ;
            SubtractExpression subtract121 = new SubtractExpression(innerPlan12, project121, const121) ;
   

   
            // Create Cogroup
            ArrayList<LogicalRelationalOperator> inputs = new ArrayList<LogicalRelationalOperator>() ;
            inputs.add(load1) ;
            inputs.add(load2) ;
   
            MultiMap<Integer, LogicalExpressionPlan> maps
                                = new MultiMap<Integer, LogicalExpressionPlan>() ;
            maps.put(0, innerPlan11);
            maps.put(1, innerPlan12);
   
            boolean[] isInner = new boolean[inputs.size()] ;
            for (int i=0; i < isInner.length ; i++) {
                isInner[i] = false ;
            }
   
           
            cogroup1.setInnerFlags(isInner);
            cogroup1.setExpressionPlans(maps);
   
            // construct the main plan
            plan.add(load1) ;
            plan.add(load2) ;
            plan.add(cogroup1) ;
   
            plan.connect(load1, cogroup1);
            plan.connect(load2, cogroup1);
   
            CompilationMessageCollector collector = new CompilationMessageCollector() ;
            TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
            typeChecker.visit()
            printMessageCollector(collector) ;
            //printTypeGraph(plan) ;
   
            if (collector.hasError()) {
                throw new AssertionError("Expect no error") ;
            }
   
            // check outer schema
            LogicalSchema endResultSchema = cogroup1.getSchema() ;
   
            // Tuple group column
            assertEquals(endResultSchema.getField(0).type, DataType.DOUBLE) ;
   
            assertEquals(endResultSchema.getField(1).type, DataType.BAG) ;
View Full Code Here

   
            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1))) ;
            load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema2))) ;
   
            LOCogroup cogroup1 = new LOCogroup(plan);
            // Create expression inner plan #1
            LogicalExpressionPlan innerPlan11 = new LogicalExpressionPlan() ;
            ProjectExpression project111 = new ProjectExpression(innerPlan11, 0, 0, cogroup1) ;
            ConstantExpression const111 = new ConstantExpression(innerPlan11, 26F) ;
            SubtractExpression subtract111 = new SubtractExpression(innerPlan11, project111, const111) ;

   
            // Create expression inner plan #2
            LogicalExpressionPlan innerPlan12 = new LogicalExpressionPlan() ;
            ConstantExpression const121 = new ConstantExpression(innerPlan12, 26) ;
//            innerPlan12.add(const121) ;
   
            // Create Cogroup
            ArrayList<LogicalRelationalOperator> inputs = new ArrayList<LogicalRelationalOperator>() ;
            inputs.add(load1) ;
            inputs.add(load2) ;
   
            MultiMap<Integer, LogicalExpressionPlan> maps
                                = new MultiMap<Integer, LogicalExpressionPlan>() ;
            maps.put(0, innerPlan11);
            maps.put(1, innerPlan12);
   
            boolean[] isInner = new boolean[inputs.size()] ;
            for (int i=0; i < isInner.length ; i++) {
                isInner[i] = false ;
            }
   
           
            cogroup1.setInnerFlags(isInner);
            cogroup1.setExpressionPlans(maps);
   
            // construct the main plan
            plan.add(load1) ;
            plan.add(load2) ;
            plan.add(cogroup1) ;
   
            plan.connect(load1, cogroup1);
            plan.connect(load2, cogroup1);
   
            CompilationMessageCollector collector = new CompilationMessageCollector() ;
            TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
            typeChecker.visit()
            printMessageCollector(collector) ;
            //printTypeGraph(plan) ;
   
            if (collector.hasError()) {
                throw new AssertionError("Expect no error") ;
            }
   
            // check outer schema
            LogicalSchema endResultSchema = cogroup1.getSchema() ;
   
            // Tuple group column
            assertEquals(endResultSchema.getField(0).type, DataType.FLOAT) ;
   
            assertEquals(endResultSchema.getField(1).type, DataType.BAG) ;
View Full Code Here

        return expectedSch;
    }

    private LOCogroup checkNumExpressionPlansForGroup(LogicalPlan lp, int numPlans) {
        Class<?> groupClass = org.apache.pig.newplan.logical.relational.LOCogroup.class;
        LOCogroup grp = (LOCogroup) NewLogicalPlanUtil.getRelOpFromPlan(lp, groupClass);
       
        for( int inp : grp.getExpressionPlans().keySet()){
            List<LogicalExpressionPlan> plans = grp.getExpressionPlans().get(inp);
            assertEquals("number of group-by plans", numPlans, plans.size());
        }
       
        return grp;
    }
View Full Code Here

        "b = group a by a0;" +
        "store b into 'empty';")
       
        LogicalPlan newLogicalPlan = buildPlan(query);
        Operator store = newLogicalPlan.getSinks().get(0);
        LOCogroup cogroup = (LOCogroup)newLogicalPlan.getPredecessors(store).get(0);
       
        LogicalSchema cogroupSchema = cogroup.getSchema();
        assertEquals(cogroupSchema.getField(1).type, DataType.BAG);
        assertTrue(cogroupSchema.getField(1).alias.equals("a"));
        LogicalSchema bagSchema = cogroupSchema.getField(1).schema;
        assertEquals(bagSchema.getField(0).type, DataType.TUPLE);
        assertEquals(bagSchema.getField(0).alias, null);
View Full Code Here

        }
        return alias;
    }

    LOCogroup createGroupOp() {
        return new LOCogroup( plan );
    }
View Full Code Here

        LogicalPlan lp;
        String query = "a = load 'myfile' as (name, age, gpa);" +
                   "b = group a by (name, age);";
        lp = buildPlan( query + "store b into 'output';");
        Operator store = lp.getSinks().get(0);
        LOCogroup cogroup = (LOCogroup) lp.getPredecessors(store).get(0);

        LogicalSchema actual = cogroup.getSchema();
        System.out.println( actual.toString( false ) );

        Assert.assertTrueactual.toString( false ).equals( "group:tuple(name:bytearray,age:bytearray),a:bag{:tuple(name:bytearray,age:bytearray,gpa:bytearray)}" ) );

        lp = buildPlan(query +
View Full Code Here

        String query = "a = load 'myfile' as (name:Chararray, age:Int, gpa:Float);" +
                       "b = group a by (name, age);" +
                       "store b into 'output';";
        lp = buildPlan( query );
        Operator store = lp.getSinks().get(0);
        LOCogroup cogroup = (LOCogroup) lp.getPredecessors(store).get(0);

        Schema.FieldSchema nameFs = new Schema.FieldSchema("name", DataType.CHARARRAY);
        Schema.FieldSchema ageFs = new Schema.FieldSchema("age", DataType.INTEGER);
        Schema.FieldSchema gpaFs = new Schema.FieldSchema("gpa", DataType.FLOAT);

        Schema groupSchema = new Schema(nameFs);
        groupSchema.add(ageFs);
        Schema.FieldSchema groupFs = new Schema.FieldSchema("group", groupSchema, DataType.TUPLE);

        Schema loadSchema = new Schema(nameFs);
        loadSchema.add(ageFs);
        loadSchema.add(gpaFs);

        Schema.FieldSchema bagFs = new Schema.FieldSchema("a", loadSchema, DataType.BAG);

        Schema cogroupExpectedSchema = new Schema(groupFs);
        cogroupExpectedSchema.add(bagFs);
        Assert.assertTrue(cogroup.getSchema().toString(false).equals("group:tuple(name:chararray,age:int),a:bag{:tuple(name:chararray,age:int,gpa:float)}"));
    }
View Full Code Here

TOP

Related Classes of org.apache.pig.newplan.logical.relational.LOCogroup

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.