Examples of LogicalExpressionPlan

org.apache.pig.experimental.logical.expression.LogicalExpressionPlan
A plan containing LogicalExpressionOperators.
org.apache.pig.newplan.logical.expression.LogicalExpressionPlan
A plan containing LogicalExpressionOperators.

Examples of org.apache.pig.experimental.logical.expression.LogicalExpressionPlan

              "b", null, DataType.INTEGER));
          cschema.getField(0).uid = 1;
          cschema.getField(1).uid = 2;
          cschema.getField(2).uid = 3;
          cschema.getField(3).uid = 4;
          LogicalExpressionPlan aprojplan = new LogicalExpressionPlan();
          ProjectExpression x = new ProjectExpression(aprojplan, DataType.INTEGER, 0, 0);
          x.neverUseForRealSetUid(1);
          LogicalExpressionPlan bprojplan = new LogicalExpressionPlan();
          ProjectExpression y = new ProjectExpression(bprojplan, DataType.INTEGER, 1, 0);
          y.neverUseForRealSetUid(3);
          MultiMap<Integer, LogicalExpressionPlan> mm = 
              new MultiMap<Integer, LogicalExpressionPlan>();
          mm.put(0, aprojplan);
          mm.put(1, bprojplan);
          LOJoin C = new LOJoin(lp, mm, JOINTYPE.HASH, new boolean[] {true, true});
          C.neverUseForRealSetSchema(cschema);
          lp.add(C);
          lp.connect(A, C);
          lp.connect(B, C);
        
          // D = filter
          LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
          ProjectExpression fx = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 0);
          fx.neverUseForRealSetUid(1);
          ConstantExpression fc0 = new ConstantExpression(filterPlan, DataType.INTEGER, new Integer(0));
          EqualExpression eq1 = new EqualExpression(filterPlan, fx, fc0);
          ProjectExpression fanotherx = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 0);
          fanotherx.neverUseForRealSetUid(1);
          ProjectExpression fa = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 2);
          fa.neverUseForRealSetUid(3);
          EqualExpression eq2 = new EqualExpression(filterPlan, fanotherx, fa);
          AndExpression and1 = new AndExpression(filterPlan, eq1, eq2);
          ProjectExpression fb = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 3);
          fb.neverUseForRealSetUid(4);
          ConstantExpression fc1 = new ConstantExpression(filterPlan, DataType.INTEGER, new Integer(1));
          EqualExpression eq3 = new EqualExpression(filterPlan, fb, fc1);
          AndExpression and2 = new AndExpression(filterPlan, and1, eq3);
          ProjectExpression fanotherb = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 3);
          fanotherb.neverUseForRealSetUid(4);
          ProjectExpression fy = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 1);
          fy.neverUseForRealSetUid(2);
          EqualExpression eq4 = new EqualExpression(filterPlan, fy, fanotherb);
          new AndExpression(filterPlan, and2, eq4);
        
          LOFilter D = new LOFilter(lp, filterPlan);
          D.neverUseForRealSetSchema(cschema);
          // Connect D to B, since the transform has happened.
          lp.add(D);
          lp.connect(C, D);
        }
        
        LogicalPlanOptimizer optimizer = new LogicalPlanOptimizer(lp, 500);
        optimizer.optimize();
        
        LogicalPlan expected = new LogicalPlan();
        {
            // A = load
          LogicalSchema aschema = new LogicalSchema();
          aschema.addField(new LogicalSchema.LogicalFieldSchema(
              "x", null, DataType.INTEGER));
          aschema.addField(new LogicalSchema.LogicalFieldSchema(
              "y", null, DataType.INTEGER));
          aschema.getField(0).uid = 1;
          aschema.getField(1).uid = 2;
          LOLoad A = new LOLoad(new FileSpec("bla", new FuncSpec("PigStorage", "\t")), aschema, expected);
          expected.add(A);
          
          // DA = filter
          LogicalExpressionPlan DAfilterPlan = new LogicalExpressionPlan();
          ProjectExpression fx = new ProjectExpression(DAfilterPlan, DataType.INTEGER, 0, 0);
          fx.neverUseForRealSetUid(1);
          ConstantExpression fc0 = new ConstantExpression(DAfilterPlan, DataType.INTEGER, new Integer(0));
          new EqualExpression(DAfilterPlan, fx, fc0);
          
          LOFilter DA = new LOFilter(expected, DAfilterPlan);
          DA.neverUseForRealSetSchema(aschema);
          expected.add(DA);
          expected.connect(A, DA);
          
          // B = load
          LogicalSchema bschema = new LogicalSchema();
          bschema.addField(new LogicalSchema.LogicalFieldSchema(
              "a", null, DataType.INTEGER));
          bschema.addField(new LogicalSchema.LogicalFieldSchema(
              "b", null, DataType.INTEGER));
          bschema.getField(0).uid = 3;
          bschema.getField(1).uid = 4;
          LOLoad B = new LOLoad(null, bschema, expected);
          expected.add(B);
          
          // DB = filter
          LogicalExpressionPlan DBfilterPlan = new LogicalExpressionPlan();
          ProjectExpression fb = new ProjectExpression(DBfilterPlan, DataType.INTEGER, 0, 1);
          fb.neverUseForRealSetUid(4);
          ConstantExpression fc1 = new ConstantExpression(DBfilterPlan, DataType.INTEGER, new Integer(1));
          new EqualExpression(DBfilterPlan, fb, fc1);
          
          LOFilter DB = new LOFilter(expected, DBfilterPlan);
          DB.neverUseForRealSetSchema(bschema);
          expected.add(DB);
          expected.connect(B, DB);
          
          // C = join
          LogicalSchema cschema = new LogicalSchema();
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "x", null, DataType.INTEGER));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "y", null, DataType.INTEGER));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "a", null, DataType.INTEGER));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "b", null, DataType.INTEGER));
          cschema.getField(0).uid = 1;
          cschema.getField(1).uid = 2;
          cschema.getField(2).uid = 3;
          cschema.getField(3).uid = 4;
          LogicalExpressionPlan aprojplan = new LogicalExpressionPlan();
          ProjectExpression x = new ProjectExpression(aprojplan, DataType.INTEGER, 0, 0);
          x.neverUseForRealSetUid(1);
          LogicalExpressionPlan bprojplan = new LogicalExpressionPlan();
          ProjectExpression y = new ProjectExpression(bprojplan, DataType.INTEGER, 1, 0);
          y.neverUseForRealSetUid(3);
          MultiMap<Integer, LogicalExpressionPlan> mm = 
              new MultiMap<Integer, LogicalExpressionPlan>();
          mm.put(0, aprojplan);
          mm.put(1, bprojplan);
          LOJoin C = new LOJoin(expected, mm, JOINTYPE.HASH, new boolean[] {true, true});
          C.neverUseForRealSetSchema(cschema);
          expected.add(C);
          expected.connect(DA, C);
          expected.connect(DB, C);
          
          // D = filter
          LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
          ProjectExpression fanotherx = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 0);
          fanotherx.neverUseForRealSetUid(1);
          ProjectExpression fa = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 2);
          fa.neverUseForRealSetUid(3);
          EqualExpression eq2 = new EqualExpression(filterPlan, fanotherx, fa);

View Full Code Here

Examples of org.apache.pig.newplan.logical.expression.LogicalExpressionPlan

    
    public SortInfo getSortInfo() throws FrontendException {
        LogicalSchema schema = this.getSchema();
        List<SortColInfo> sortColInfoList = new ArrayList<SortColInfo>();
        for (int i = 0; i < mSortColPlans.size(); i++) {
            LogicalExpressionPlan lp = mSortColPlans.get(i);
            Iterator<Operator> opsIterator = lp.getOperators();
            List<Operator> opsList = new ArrayList<Operator>();
            while(opsIterator.hasNext()) {
                opsList.add(opsIterator.next());
            }
            if(opsList.size() != 1 || !(opsList.get(0) instanceof ProjectExpression)) {

View Full Code Here

Examples of org.apache.pig.newplan.logical.expression.LogicalExpressionPlan

    public LOInnerLoad(OperatorPlan plan, LOForEach foreach, int colNum) {
        super("LOInnerLoad", plan);        
        
        // store column number as a ProjectExpression in a plan 
        // to be able to dynamically adjust column number during optimization
        LogicalExpressionPlan exp = new LogicalExpressionPlan();
        
        // we don't care about type, so set to -1
        prj = new ProjectExpression(exp, 0, colNum, foreach);
        this.foreach = foreach;
    }

View Full Code Here

Examples of org.apache.pig.newplan.logical.expression.LogicalExpressionPlan

                    LoadPushDown.RequiredField rf = requiredFields.getFields().get(i);
                    LOInnerLoad innerLoad = new LOInnerLoad(innerPlan, foreach, rf.getIndex());                    
                    innerPlan.add(innerLoad);          
                    innerPlan.connect(innerLoad, gen);
                    
                    LogicalExpressionPlan exp = new LogicalExpressionPlan();
                    ProjectExpression prj = new ProjectExpression(exp, i, -1, gen);
                    exp.add(prj);
                    exps.add(exp);
                }                
               
            } else {
                // columns are pruned, reset schema for LOLoader

View Full Code Here

Examples of org.apache.pig.newplan.logical.expression.LogicalExpressionPlan

        LOGenerate gen = (LOGenerate)innerPlan.getSinks().get(0);
        List<LogicalExpressionPlan> genPlansToRemove = new ArrayList<LogicalExpressionPlan>();
        
        List<LogicalExpressionPlan> genPlans = gen.getOutputPlans();
        for (int i=0;i<genPlans.size();i++) {
            LogicalExpressionPlan expPlan = genPlans.get(i);
            List<Operator> expSources = expPlan.getSinks();
            
            for (Operator expSrc : expSources) {
                if (expSrc instanceof ProjectExpression) {
                    LogicalRelationalOperator reference = ((ProjectExpression)expSrc).findReferent();
                    if (branchHeadToRemove.contains(reference)) {
                        genPlansToRemove.add(expPlan);
                    }
                }
            }
        }
        
        // Build the temporary structure based on genPlansToRemove, which include:
        // * flattenList
        // * outputPlanSchemas
        // * uidOnlySchemas
        // * inputsRemoved
        //     We first construct inputsNeeded, and inputsRemoved = (all inputs) - inputsNeeded.
        //     We cannot figure out inputsRemoved directly since the inputs may be used by other output plan.
        //     We can only get inputsRemoved after visiting all output plans.
        List<Boolean> flattenList = new ArrayList<Boolean>();
        Set<Integer> inputsNeeded = new HashSet<Integer>();
        Set<Integer> inputsRemoved = new HashSet<Integer>();
        List<LogicalSchema> outputPlanSchemas = new ArrayList<LogicalSchema>();
        List<LogicalSchema> uidOnlySchemas = new ArrayList<LogicalSchema>();
        List<LogicalSchema> userDefinedSchemas = null;
        
        if (gen.getUserDefinedSchema()!=null)
            userDefinedSchemas = new ArrayList<LogicalSchema>();
        
        for (int i=0;i<genPlans.size();i++) {
            LogicalExpressionPlan genPlan = genPlans.get(i);
            if (!genPlansToRemove.contains(genPlan)) {
                flattenList.add(gen.getFlattenFlags()[i]);
                outputPlanSchemas.add(gen.getOutputPlanSchemas().get(i));
                uidOnlySchemas.add(gen.getUidOnlySchemas().get(i));
                if (gen.getUserDefinedSchema()!=null) {
                    userDefinedSchemas.add(gen.getUserDefinedSchema().get(i));
                }
                List<Operator> sinks = genPlan.getSinks();
                for(Operator s: sinks) {
                    if (s instanceof ProjectExpression) {
                        inputsNeeded.add(((ProjectExpression)s).getInputNum());
                    }
                }
            }
        }
        
        List<Operator> preds = innerPlan.getPredecessors(gen);
        
        if (preds!=null) {  // otherwise, all gen plan are based on constant, no need to adjust
            for (int i=0;i<preds.size();i++) {
                if (!inputsNeeded.contains(i))
                    inputsRemoved.add(i);
            }
        }
        
        
        // Change LOGenerate: remove unneeded output expression plan
        // change flatten flag, outputPlanSchema, uidOnlySchemas
        boolean[] flatten = new boolean[flattenList.size()];
        for (int i=0;i<flattenList.size();i++)
            flatten[i] = flattenList.get(i);


        gen.setFlattenFlags(flatten);
        gen.setOutputPlanSchemas(outputPlanSchemas);
        gen.setUidOnlySchemas(uidOnlySchemas);
        gen.setUserDefinedSchema(userDefinedSchemas);
        
        for (LogicalExpressionPlan genPlanToRemove : genPlansToRemove) {
            genPlans.remove(genPlanToRemove);
        }
        
        // shift project input
        if (!inputsRemoved.isEmpty()) {
            for (LogicalExpressionPlan genPlan : genPlans) {
                List<Operator> sinks = genPlan.getSinks();
                for(Operator s: sinks) {
                    if (s instanceof ProjectExpression) {
                        int input = ((ProjectExpression)s).getInputNum();
                        int numToShift = 0;
                        for (int i :inputsRemoved) {

View Full Code Here

Examples of org.apache.pig.newplan.logical.expression.LogicalExpressionPlan

                
                LOInnerLoad innerLoad = new LOInnerLoad(innerPlan, foreach, i);
                innerPlan.add(innerLoad);          
                innerPlan.connect(innerLoad, gen);
                
                LogicalExpressionPlan exp = new LogicalExpressionPlan();
                
                ProjectExpression prj = new ProjectExpression(exp, i, 0, gen);
                exp.add(prj);
                
                if (fs.type != DataType.BYTEARRAY && (determinedSchema == null || (!fs.isEqual(determinedSchema.getField(i))))) {
                    // Either no schema was determined by loader OR the type 
                    // from the "determinedSchema" is different
                    // from the type specified - so we need to cast
                    CastExpression cast = new CastExpression(exp, prj, new LogicalSchema.LogicalFieldSchema(fs));
                    exp.add(cast);
                    FuncSpec loadFuncSpec = null;
                    if(op instanceof LOLoad) {
                        loadFuncSpec = ((LOLoad)op).getFileSpec().getFuncSpec();
                    } else if (op instanceof LOStream) {
                        StreamingCommand command = ((LOStream)op).getStreamingCommand();

View Full Code Here

Examples of org.apache.pig.newplan.logical.expression.LogicalExpressionPlan

            LOGenerate newGen = new LOGenerate(newForEachInnerPlan, newExpList, gen2.getFlattenFlags());
            newGen.setUserDefinedSchema(gen2.getUserDefinedSchema());
            newForEachInnerPlan.add(newGen);
            
            for (LogicalExpressionPlan exp2 : gen2.getOutputPlans()) {
                LogicalExpressionPlan newExpPlan = new LogicalExpressionPlan();
                LogicalExpressionPlan exp2Copy = exp2.deepCopy();
                newExpPlan.merge(exp2Copy);
                
                // Add expression plan in 2nd ForEach
                List<Operator> exp2Sinks = new ArrayList<Operator>();
                exp2Sinks.addAll(newExpPlan.getSinks());
                for (Operator exp2Sink : exp2Sinks) {
                    if (exp2Sink instanceof ProjectExpression) {
                        // Find referred expression plan in 1st ForEach
                        ProjectExpression proj = (ProjectExpression)exp2Sink;
                        LOInnerLoad innerLoad = (LOInnerLoad)foreach2.getInnerPlan().getPredecessors(gen2).get(proj.getInputNum());
                        int exp1Pos = innerLoad.getProjection().getColNum();
                        LogicalExpressionPlan exp1 = gen1.getOutputPlans().get(exp1Pos);
                        LogicalExpressionPlan exp1Copy = exp1.deepCopy();
                        List<Operator> exp1Sources = newExpPlan.merge(exp1Copy);
                        
                        // Copy expression plan to the new ForEach, connect to the expression plan of 2nd ForEach
                        Operator exp1Source = exp1Sources.get(0);
                        if (newExpPlan.getPredecessors(exp2Sink)!=null) {

View Full Code Here

Examples of org.apache.pig.newplan.logical.expression.LogicalExpressionPlan

            
            // the input uids contains all the output uids and
            // projections in filter conditions
            Set<Long> input = new HashSet<Long>(output);
            
            LogicalExpressionPlan exp = filter.getFilterPlan();
            collectUids(filter, exp, input);
            
            filter.annotate(INPUTUIDS, input);
        }

View Full Code Here

Examples of org.apache.pig.newplan.logical.expression.LogicalExpressionPlan

            Set<Long> input = new HashSet<Long>(output);
            
            Collection<LogicalExpressionPlan> exps = join.getExpressionPlanValues();
            Iterator<LogicalExpressionPlan> iter = exps.iterator();
            while(iter.hasNext()) {
                LogicalExpressionPlan exp = iter.next();
                collectUids(join, exp, input);
            }
            
            join.annotate(INPUTUIDS, input);
        }

View Full Code Here

Examples of org.apache.pig.newplan.logical.expression.LogicalExpressionPlan

            
            for (long uid : output) {
                input.add(splitOutput.getInputUids(uid));
            }
            
            LogicalExpressionPlan exp = splitOutput.getFilterPlan();
            collectUids(splitOutput, exp, input);
            
            splitOutput.annotate(INPUTUIDS, input);
        }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.