Package org.apache.pig.impl.logicalLayer

Examples of org.apache.pig.impl.logicalLayer.LogicalPlan


   
    public void visit(LOSplitOutput splitOutput) throws VisitorException {
        org.apache.pig.newplan.logical.relational.LOSplitOutput newSplitOutput =
            new org.apache.pig.newplan.logical.relational.LOSplitOutput(logicalPlan);
       
        LogicalPlan filterPlan = splitOutput.getConditionPlan();
        LogicalExpressionPlan newFilterPlan = translateExpressionPlan(filterPlan, splitOutput, newSplitOutput);
     
        newSplitOutput.setFilterPlan(newFilterPlan);
        newSplitOutput.setAlias(splitOutput.getAlias());
        newSplitOutput.setRequestedParallelism(splitOutput.getRequestedParallelism());
View Full Code Here


        try{
            LogicalPlanTester lpt = new LogicalPlanTester();
            lpt.buildPlan("A = LOAD 'data1' using "+ DummyCollectableLoader.class.getName() +"() as (id, name, grade);");
            lpt.buildPlan("B = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);");
            lpt.buildPlan("D = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);");
            LogicalPlan lp = lpt.buildPlan("C = cogroup A by id, B by id, D by id using 'merge';");
            assertEquals(LOCogroup.GROUPTYPE.MERGE, ((LOCogroup)lp.getLeaves().get(0)).getGroupType());

            PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
            pc.connect();
            PhysicalPlan phyP = Util.buildPhysicalPlan(lp, pc);
            PhysicalOperator phyOp = phyP.getLeaves().get(0);
View Full Code Here

        LogicalPlanTester lpt = new LogicalPlanTester();
        lpt.buildPlan("A = LOAD 'data1' using "+ DummyCollectableLoader.class.getName() +"() as (id, name, grade);");
        lpt.buildPlan("E = group A by id;");
        lpt.buildPlan("B = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);");
        lpt.buildPlan("D = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);");
        LogicalPlan lp = lpt.buildPlan("C = cogroup E by A.id, B by id, D by id using 'merge';");
        assertEquals(LOCogroup.GROUPTYPE.MERGE, ((LOCogroup)lp.getLeaves().get(0)).getGroupType());

        PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
        pc.connect();
        boolean exceptionCaught = false;
        try{
View Full Code Here

    public void testFailure2() throws Exception{
        LogicalPlanTester lpt = new LogicalPlanTester();
        lpt.buildPlan("A = LOAD 'data1' using "+ DummyCollectableLoader.class.getName() +"() as (id, name, grade);");
        lpt.buildPlan("B = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);");
        lpt.buildPlan("D = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);");
        LogicalPlan lp = lpt.buildPlan("C = cogroup A by id inner, B by id, D by id inner using 'merge';");
        assertEquals(LOCogroup.GROUPTYPE.MERGE, ((LOCogroup)lp.getLeaves().get(0)).getGroupType());

        PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
        pc.connect();
        boolean exceptionCaught = false;
        try{
View Full Code Here

        boolean[] flattens = new boolean[foreach.getForEachPlans().size()];
        LOGenerate generate = new LOGenerate(newForEachInnerPlan, expPlans, flattens);
        newForEachInnerPlan.add(generate);
       
        for (int i=0;i<foreach.getForEachPlans().size();i++) {
            LogicalPlan innerPlan = foreach.getForEachPlans().get(i);
            // Assume only one project is allowed in this level of foreach
            LOProject project = (LOProject)innerPlan.iterator().next();

            LOInnerLoad innerLoad = new LOInnerLoad(newForEachInnerPlan,
                    newForEach, project.isStar()?-1:project.getCol());
            newForEachInnerPlan.add(innerLoad);
            newForEachInnerPlan.connect(innerLoad, generate);
View Full Code Here

                        }
                    }
                    else if (rlo instanceof LOForEach)
                    {
                        // Relay map keys from output to input
                        LogicalPlan forEachPlan = ((LOForEach)rlo).getRelevantPlan(requiredOutputField.second);
                        if (relevantFields.getFields()!=null && relevantFields.getFields().size()!=0)
                        {
                            int index = ((LOForEach)rlo).getForEachPlans().indexOf(forEachPlan);
                            // We check if the field get flattened, if it does, then we do not relay output map keys to input map keys.
                            // There are two situations:
                            // 1. input column is tuple, bag, or other simple type, there is no concept of map key, so we do not relay
                            // 2. input column is map, flatten does not do anything, we can still relay
                            boolean nonflatten = false;
                            if (!((LOForEach)rlo).getFlatten().get(index))
                            {
                                nonflatten = true;
                            }
                            else
                            {
                                // Foreach plan is flattened, check if there is only one input for this foreach plan
                                // and input schema for that input is not map, if so, it is a dummy flatten
                                if (forEachPlan.getRoots().size()==1 && forEachPlan.getRoots().get(0) instanceof LOProject)
                                {
                                    LOProject loProj = (LOProject)forEachPlan.getRoots().get(0);
                                    if (loProj.getExpression().getSchema()!=null &&
                                            loProj.getExpression().getSchema().getField(loProj.getCol()).type!=DataType.BAG)
                                        nonflatten = true;
                                }
                            }
View Full Code Here

            pig.registerQuery(query);
            System.out.print("Current aliases: ");
            Map<String, LogicalPlan> aliasPlan = pig.getAliases();
            for (Iterator<Map.Entry<String,LogicalPlan>> it = aliasPlan.entrySet().iterator(); it.hasNext(); ) {
                Map.Entry<String, LogicalPlan> e = it.next();
                LogicalPlan lp = e.getValue();
                System.out.print(e.getKey() + "->" + lp.getLeaves().get(0).getSchema());
                if (it.hasNext()) System.out.print(", \n");
                else System.out.print("\n");
            }
        } catch (IOException e) {
            log.error(e);
View Full Code Here

                "};";*/
      String query = "foreach (load 'a') {" +
        "B = FILTER $0 BY ($1 == $2);" +
        "generate B;" +
        "};";
        LogicalPlan plan = buildPlan(query);
      PhysicalPlan pp = buildPhysicalPlan(plan);
      int MAX_SIZE = 100000;
      ByteArrayOutputStream baos = new ByteArrayOutputStream();
        pp.explain(baos);
        baos.write((int)'\n');
View Full Code Here

        assertEquals(compiledPlan, goldenPlan);
    }
       
    public void testSort() throws VisitorException, IOException {
      String query = "order (load 'a') by $0;";
      LogicalPlan plan = buildPlan(query);
      PhysicalPlan pp = buildPhysicalPlan(plan);

      int MAX_SIZE = 100000;
      ByteArrayOutputStream baos = new ByteArrayOutputStream();
        pp.explain(baos);
View Full Code Here

        assertEquals(compiledPlan, goldenPlan);
    }
       
    public void testDistinct() throws VisitorException, IOException {
      String query = "distinct (load 'a');";
      LogicalPlan plan = buildPlan(query);
      PhysicalPlan pp = buildPhysicalPlan(plan);
     
      int MAX_SIZE = 100000;
      ByteArrayOutputStream baos = new ByteArrayOutputStream();
        pp.explain(baos);
View Full Code Here

TOP

Related Classes of org.apache.pig.impl.logicalLayer.LogicalPlan

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.