Package org.apache.pig.newplan.logical.relational

Examples of org.apache.pig.newplan.logical.relational.LOCogroup


            "D = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);" +
            "C = cogroup A by id, B by id, D by id using 'merge';" +
            "store C into 'output';";
            LogicalPlan lp = Util.buildLp(pigServer, query);
            Operator op = lp.getSinks().get(0);
            LOCogroup cogrp = (LOCogroup)lp.getPredecessors(op).get(0);
            assertEquals(LOCogroup.GROUPTYPE.MERGE, cogrp.getGroupType());

            PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
            pc.connect();
            PhysicalPlan phyP = Util.buildPp(pigServer, query);
            PhysicalOperator phyOp = phyP.getLeaves().get(0);
View Full Code Here


        "D = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);" +
        "C = cogroup A by id inner, B by id, D by id inner using 'merge';" +
        "store C into 'output';";
        LogicalPlan lp = Util.buildLp(pigServer, query);
        Operator op = lp.getSinks().get(0);
        LOCogroup cogrp = (LOCogroup)lp.getPredecessors(op).get(0);
        assertEquals(LOCogroup.GROUPTYPE.MERGE, cogrp.getGroupType());

        PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
        pc.connect();
        boolean exceptionCaught = false;
        try{
View Full Code Here

                "store B into '111';";

        LogicalPlan lp = Util.parseAndPreprocess(query, pc);
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOCogroup loCoGroup = (LOCogroup)lp.getPredecessors(loStore).get(0);
        LogicalFieldSchema groupFieldSchema = loCoGroup.getSchema().getField(0);
        assertEquals(DataType.TUPLE, groupFieldSchema.type);
        assertNull(groupFieldSchema.schema);
    }
View Full Code Here

                return;

            Operator childOp = plan.getPredecessors(op).get(0);
            if (op instanceof LOForEach && childOp instanceof LOCogroup)
            {
                LOCogroup cg = (LOCogroup) childOp;
                for (Operator input : cg.getInputs(plan)) {
                    AffinityGroups.put(input, eg.getEqClasses());
                    Lineage.put(input, eg.getLineage());
                }
            } else {
                List<Operator> childOps = plan.getPredecessors(op);
View Full Code Here

        pigServer.setBatchOn();
        pigServer.registerQuery("a = load 'tmp1' as (foo, bar);");
        pigServer.registerQuery("b = group a by foo;");
        pigServer.registerQuery("store b into 'tmp2';");

        LOCogroup op = (LOCogroup)TestPigStats.getLogicalPlan(pigServer).findByAlias("b");
        assertFalse(op.isPinnedOption(LOCogroup.OPTION_GROUPTYPE));
        pigServer.discardBatch();

        pigServer.setBatchOn();
        pigServer.registerQuery("a = load 'tmp' as (foo, bar);");
        pigServer.registerQuery("b = group a by foo using 'collected';");
        pigServer.registerQuery("store b into 'tmp2';");
        op = (LOCogroup)TestPigStats.getLogicalPlan(pigServer).findByAlias("b");
        assertTrue(op.isPinnedOption(LOCogroup.OPTION_GROUPTYPE));
        pigServer.discardBatch();
    }
View Full Code Here

            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1)));
            load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema2)));
            load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema2)));

            LOCogroup cogroup1 = new LOCogroup(plan);

            // Create expression inner plan #1 of input #1
            LogicalExpressionPlan innerPlan11 = new LogicalExpressionPlan();
            ProjectExpression project111 = new ProjectExpression(innerPlan11, 0, 0, cogroup1);
            ConstantExpression const111 = new ConstantExpression(innerPlan11, 26F);
            SubtractExpression subtract111 = new SubtractExpression(innerPlan11, project111, const111);

            // Create expression inner plan #2 of input #1
            LogicalExpressionPlan innerPlan21 = new LogicalExpressionPlan();
            ProjectExpression project211 = new ProjectExpression(innerPlan21, 0, 0, cogroup1);
            ProjectExpression project212 = new ProjectExpression(innerPlan21, 0, 1, cogroup1);

            AddExpression add211 = new AddExpression(innerPlan21, project211, project212);

            // Create expression inner plan #1 of input #2
            LogicalExpressionPlan innerPlan12 = new LogicalExpressionPlan();
            ProjectExpression project121 = new ProjectExpression(innerPlan12, 1, 0, cogroup1);
            ConstantExpression const121 = new ConstantExpression(innerPlan12, 26);
            SubtractExpression subtract121 = new SubtractExpression(innerPlan12, project121, const121);

            // Create expression inner plan #2 of input #2
            LogicalExpressionPlan innerPlan22 = new LogicalExpressionPlan();
            ConstantExpression const122 = new ConstantExpression(innerPlan22, 26);
//            innerPlan22.add(const122);

            // Create Cogroup
            ArrayList<LogicalRelationalOperator> inputs = new ArrayList<LogicalRelationalOperator>();
            inputs.add(load1);
            inputs.add(load2);

            MultiMap<Integer, LogicalExpressionPlan> maps
                                = new MultiMap<Integer, LogicalExpressionPlan>();
            maps.put(0, innerPlan11);
            maps.put(0, innerPlan21);
            maps.put(1, innerPlan12);
            maps.put(1, innerPlan22);

            boolean[] isInner = new boolean[inputs.size()];
            for (int i=0; i < isInner.length; i++) {
                isInner[i] = false;
            }

            cogroup1.setInnerFlags(isInner);
            cogroup1.setExpressionPlans(maps);

            // construct the main plan
            plan.add(load1);
            plan.add(load2);
            plan.add(cogroup1);

            plan.connect(load1, cogroup1);
            plan.connect(load2, cogroup1);

            CompilationMessageCollector collector = new CompilationMessageCollector();
            TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
            typeChecker.visit();
            printMessageCollector(collector);
            //printTypeGraph(plan);

            if (collector.hasError()) {
                throw new AssertionError("Expect no error");
            }

            // check outer schema
            LogicalSchema endResultSchema = cogroup1.getSchema();

            // Tuple group column
            assertEquals(DataType.TUPLE, endResultSchema.getField(0).type);
            assertEquals(DataType.DOUBLE,endResultSchema.getField(0).schema.getField(0).type);
            assertEquals(DataType.LONG, endResultSchema.getField(0).schema.getField(1).type);
View Full Code Here

            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1)));;
            load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema2)));

            LOCogroup cogroup1 = new LOCogroup(plan);

            // Create expression inner plan #1 of input #1
            LogicalExpressionPlan innerPlan11 = new LogicalExpressionPlan();
            ProjectExpression project111 = new ProjectExpression(innerPlan11, 0, 0, cogroup1);
            ConstantExpression const111 = new ConstantExpression(innerPlan11, 26F);
            SubtractExpression subtract111 = new SubtractExpression(innerPlan11, project111, const111);

            // Create expression inner plan #1 of input #2
            LogicalExpressionPlan innerPlan12 = new LogicalExpressionPlan();
            ProjectExpression project121 = new ProjectExpression(innerPlan12, 1, 0, cogroup1);
            ConstantExpression const121 = new ConstantExpression(innerPlan12, 26);
            SubtractExpression subtract121 = new SubtractExpression(innerPlan12, project121, const121);

            // Create Cogroup
            ArrayList<LogicalRelationalOperator> inputs = new ArrayList<LogicalRelationalOperator>();
            inputs.add(load1);
            inputs.add(load2);

            MultiMap<Integer, LogicalExpressionPlan> maps
                                = new MultiMap<Integer, LogicalExpressionPlan>();
            maps.put(0, innerPlan11);
            maps.put(1, innerPlan12);

            boolean[] isInner = new boolean[inputs.size()];
            for (int i=0; i < isInner.length; i++) {
                isInner[i] = false;
            }


            cogroup1.setInnerFlags(isInner);
            cogroup1.setExpressionPlans(maps);

            // construct the main plan
            plan.add(load1);
            plan.add(load2);
            plan.add(cogroup1);

            plan.connect(load1, cogroup1);
            plan.connect(load2, cogroup1);

            CompilationMessageCollector collector = new CompilationMessageCollector();
            TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
            typeChecker.visit();
            printMessageCollector(collector);
            //printTypeGraph(plan);

            if (collector.hasError()) {
                throw new AssertionError("Expect no error");
            }

            // check outer schema
            LogicalSchema endResultSchema = cogroup1.getSchema();

            // Tuple group column
            assertEquals(DataType.DOUBLE, endResultSchema.getField(0).type);

            assertEquals(DataType.BAG, endResultSchema.getField(1).type);
View Full Code Here

            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1)));
            load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema2)));

            LOCogroup cogroup1 = new LOCogroup(plan);
            // Create expression inner plan #1
            LogicalExpressionPlan innerPlan11 = new LogicalExpressionPlan();
            ProjectExpression project111 = new ProjectExpression(innerPlan11, 0, 0, cogroup1);
            ConstantExpression const111 = new ConstantExpression(innerPlan11, 26F);
            SubtractExpression subtract111 = new SubtractExpression(innerPlan11, project111, const111);


            // Create expression inner plan #2
            LogicalExpressionPlan innerPlan12 = new LogicalExpressionPlan();
            ConstantExpression const121 = new ConstantExpression(innerPlan12, 26);
//            innerPlan12.add(const121);

            // Create Cogroup
            ArrayList<LogicalRelationalOperator> inputs = new ArrayList<LogicalRelationalOperator>();
            inputs.add(load1);
            inputs.add(load2);

            MultiMap<Integer, LogicalExpressionPlan> maps
                                = new MultiMap<Integer, LogicalExpressionPlan>();
            maps.put(0, innerPlan11);
            maps.put(1, innerPlan12);

            boolean[] isInner = new boolean[inputs.size()];
            for (int i=0; i < isInner.length; i++) {
                isInner[i] = false;
            }


            cogroup1.setInnerFlags(isInner);
            cogroup1.setExpressionPlans(maps);

            // construct the main plan
            plan.add(load1);
            plan.add(load2);
            plan.add(cogroup1);

            plan.connect(load1, cogroup1);
            plan.connect(load2, cogroup1);

            CompilationMessageCollector collector = new CompilationMessageCollector();
            TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
            typeChecker.visit();
            printMessageCollector(collector);
            //printTypeGraph(plan);

            if (collector.hasError()) {
                throw new AssertionError("Expect no error");
            }

            // check outer schema
            LogicalSchema endResultSchema = cogroup1.getSchema();

            // Tuple group column
            assertEquals(DataType.FLOAT, endResultSchema.getField(0).type);

            assertEquals(DataType.BAG, endResultSchema.getField(1).type);
View Full Code Here

        "b = group a by a0;" +
        "store b into 'empty';")
       
        LogicalPlan newLogicalPlan = buildPlan(query);
        Operator store = newLogicalPlan.getSinks().get(0);
        LOCogroup cogroup = (LOCogroup)newLogicalPlan.getPredecessors(store).get(0);
       
        LogicalSchema cogroupSchema = cogroup.getSchema();
        assertEquals(cogroupSchema.getField(1).type, DataType.BAG);
        assertTrue(cogroupSchema.getField(1).alias.equals("a"));
        LogicalSchema bagSchema = cogroupSchema.getField(1).schema;
        assertEquals(bagSchema.getField(0).type, DataType.TUPLE);
        assertEquals(bagSchema.getField(0).alias, null);
View Full Code Here

    private String convertCubeToFGPlan(SourceLocation loc, LOCube op, String inputAlias,
      List<String> operations, MultiMap<Integer, LogicalExpressionPlan> expressionPlans)
      throws FrontendException {

  LOForEach foreach = new LOForEach(plan);
  LOCogroup groupby = new LOCogroup(plan);
  LogicalPlan innerPlan = new LogicalPlan();
  LogicalRelationalOperator gen = new LOGenerate(innerPlan);

  injectForeachOperator(loc, op, foreach);
View Full Code Here

TOP

Related Classes of org.apache.pig.newplan.logical.relational.LOCogroup

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.