Package org.apache.pig.impl.logicalLayer

Examples of org.apache.pig.impl.logicalLayer.LogicalPlan


            String alias,
            boolean optimize) throws FrontendException {
       
        // create a clone of the logical plan and give it
        // to the operations below
        LogicalPlan lpClone;
        Graph g;
        try {
            g = getClonedGraph();
            lpClone = g.getPlan(alias);
View Full Code Here


            for(Map.Entry<LOUserFunc, Pair<LogicalPlan, LogicalOperator>> scalarEntry: scalarMap.entrySet()) {
                FileSpec fileSpec;
                String alias = scalarEntry.getKey().getImplicitReferencedOperator().getAlias();
                LogicalOperator store;

                LogicalPlan referredPlan = g.getAliases().get(g.getAliasOp().get(alias));

                // If referredPlan already has a store,
                // we just use it instead of adding one from our pocket
                store = referredPlan.getLeaves().get(0);
                if(store instanceof LOStore
                        &&
                        ((LOStore)store).getOutputFile().getFuncName().equals(
                                InterStorage.class.getName())                                           
                ) {
                        // use this store
                        fileSpec = ((LOStore)store).getOutputFile();
                }
                else {
                    // add new store
                    FuncSpec funcSpec = new FuncSpec(InterStorage.class.getName());
                    fileSpec = new FileSpec(FileLocalizer.getTemporaryPath(pigContext).toString(), funcSpec);
                    store = new LOStore(referredPlan, new OperatorKey(scope, NodeIdGenerator.getGenerator().getNextNodeId(scope)),
                            fileSpec, alias);
                    referredPlan.addAsLeaf(store);
                    ((LOStore)store).setTmpStore(true);
                    scalarEntry.getKey().setImplicitReferencedOperator(store);
                }
                lp.mergeSharedPlan(referredPlan);

                // Attach a constant operator to the ReadScalar func
                LogicalPlan innerPlan = scalarEntry.getValue().first;
                LOConst rconst = new LOConst(innerPlan, new OperatorKey(scope, NodeIdGenerator.getGenerator().getNextNodeId(scope)), fileSpec.getFileName());
                rconst.setType(DataType.CHARARRAY);

                innerPlan.add(rconst);
                innerPlan.connect(rconst, scalarEntry.getKey());
               
                if (lp.getSoftLinkSuccessors(store)==null || !lp.getSoftLinkSuccessors(store).contains(scalarEntry.getValue().second))
                    lp.createSoftLink(store, scalarEntry.getValue().second);
            }
        } catch (IOException ioe) {
View Full Code Here

        if (lo == null) {
            int errCode = 1004;
            String msg = "No alias " + alias + " to " + operation;
            throw new FrontendException(msg, errCode, PigException.INPUT, false, null);
        }
        LogicalPlan lp = currDAG.getAliases().get(lo);
        if (lp == null) {
            int errCode = 1005;
            String msg = "No plan for " + alias + " to " + operation;
            throw new FrontendException(msg, errCode, PigException.INPUT, false, null);
        }       
View Full Code Here

   
    @Test
    public void testJobControlCompilerErr() throws Exception {
      LogicalPlanTester planTester = new LogicalPlanTester() ;
      planTester.buildPlan("a = load 'input';");
      LogicalPlan lp = planTester.buildPlan("b = order a by $0;");
      PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
      POStore store = GenPhyOp.dummyPigStorageOp();
      pp.addAsLeaf(store);
      MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
     
View Full Code Here

    public void testDefaultParallel() throws Throwable {
        pc.defaultParallel = 100;
       
        LogicalPlanTester planTester = new LogicalPlanTester() ;
        planTester.buildPlan("a = load 'input';");
        LogicalPlan lp = planTester.buildPlan("b = group a by $0;");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        POStore store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
View Full Code Here

    public void testDefaultParallelInSort() throws Throwable {
        pc.defaultParallel = 100;
       
        LogicalPlanTester planTester = new LogicalPlanTester() ;
        planTester.buildPlan("a = load 'input';");
        LogicalPlan lp = planTester.buildPlan("b = order a by $0;");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        POStore store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
View Full Code Here

        pc.defaultParallel = 100;
       
        LogicalPlanTester planTester = new LogicalPlanTester() ;
        planTester.buildPlan("a = load 'input';");
        planTester.buildPlan("b = load 'input';");
        LogicalPlan lp = planTester.buildPlan("c = join a by $0, b by $0 using \"skewed\";");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        POStore store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
View Full Code Here

    public void testReducerNumEstimation() throws Exception{
       // use the estimation
        LogicalPlanTester planTester = new LogicalPlanTester(pc) ;
        Util.copyFromLocalToCluster(cluster, "test/org/apache/pig/test/data/passwd", "/passwd");
        planTester.buildPlan("a = load '/passwd';");
        LogicalPlan lp = planTester.buildPlan("b = group a by $0;");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        POStore store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
              
View Full Code Here

        pc.getProperties().setProperty("pig.exec.reducers.max", "10");
       
        LogicalPlanTester planTester = new LogicalPlanTester(pc) ;
        Util.copyFromLocalToCluster(cluster, "test/org/apache/pig/test/data/passwd", "/passwd");
        planTester.buildPlan("a = load '/passwd';");
        LogicalPlan lp = planTester.buildPlan("b = order a by $0;");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        POStore store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);

        MROperPlan mrPlan = Util.buildMRPlanWithOptimizer(pp, pc);
View Full Code Here

        LogicalPlanTester planTester = new LogicalPlanTester();
        planTester.buildPlan("A = LOAD 'input1' AS (a0, a1, a2);");
        planTester.buildPlan("B = group A by $0;");
        planTester.buildPlan("C = foreach B { D = A.a0; E = distinct D; generate group, E;};");

        LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);

        SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
        so.visit();
View Full Code Here

TOP

Related Classes of org.apache.pig.impl.logicalLayer.LogicalPlan

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.