Package org.apache.pig.newplan.logical.expression

Examples of org.apache.pig.newplan.logical.expression.ProjectExpression


            new HashMap<ProjectExpression, LogicalRelationalOperator>();
       
       
        for(int i=0; i<expPlans.size(); i++){
            LogicalExpressionPlan expPlan = expPlans.get(i);
            ProjectExpression projStar = getProjectLonelyStar(expPlan, oldPos2Rel);

            boolean foundExpandableProject = false;
            if(projStar != null){             
                //there is a project-star to be expanded

                LogicalSchema userStarSch = null;
                if(userSchema != null && userSchema.get(i) != null){
                    userStarSch = userSchema.get(i);
                }


                //the range values are set in the project in LOInnerLoad
                ProjectExpression loInnerProj = ((LOInnerLoad)oldPos2Rel.get(projStar.getInputNum())).getProjection();

                int firstProjCol = 0;
                int lastProjCol = 0;
               
                if(loInnerProj.isRangeProject()){
                    loInnerProj.setColumnNumberFromAlias();
                    firstProjCol = loInnerProj.getStartCol();
                    lastProjCol = loInnerProj.getEndCol();
                }

               
                boolean isProjectToEnd = loInnerProj.isProjectStar() ||
                    (loInnerProj.isRangeProject() && lastProjCol == -1);
               
                //can't expand if there is no input schema, and this is
                // as project star or project-range-to-end
                if( !(inpSch == null && isProjectToEnd) ){
                   
                    foundExpandableProject = true;

                    if(isProjectToEnd)
                        lastProjCol = inpSch.size() - 1;

                    //replacing the existing project star with new ones
                    expPlan.remove(projStar);

                    //remove the LOInnerLoad with star
                    LOInnerLoad oldLOInnerLoad = (LOInnerLoad)oldPos2Rel.get(projStar.getInputNum());
                    innerPlan.disconnect(oldLOInnerLoad, gen);
                    innerPlan.remove(oldLOInnerLoad);


                    //generate new exp plan, inner load for each field in schema
                    for(int j = firstProjCol; j <= lastProjCol; j++){

                        //add new LOInnerLoad
                        LOInnerLoad newInLoad = new LOInnerLoad(innerPlan, foreach, j);
                        innerPlan.add(newInLoad);
                        innerPlan.connect(newInLoad, gen);


                        // new expression plan and proj
                        LogicalExpressionPlan newExpPlan = new LogicalExpressionPlan();
                        newExpPlans.add(newExpPlan);

                        ProjectExpression newProj =
                            new ProjectExpression(newExpPlan, -2, -1, gen);

                        proj2InpRel.put(newProj, newInLoad);

                        newFlattens.add(flattens[i]);
                        if(newUserSchema != null ){
                            //index into user specified schema
                            int schIdx = j - firstProjCol;
                            if(userStarSch != null
                                    && userStarSch.getFields().size() > schIdx
                                    && userStarSch.getField(schIdx) != null){

                                //if the project-star field has user specified schema, use the
                                // j'th field for this column
                                LogicalSchema sch = new LogicalSchema();
                                sch.addField(new LogicalFieldSchema(userStarSch.getField(schIdx)));
                                newUserSchema.add(sch);
                            }
                            else{
                                newUserSchema.add(null);
                            }
                        }
                    }
                }
            }

            if(!foundExpandableProject){ //no project-star that could be expanded

                //get all projects in here
                FindProjects findProjs = new FindProjects(expPlan);
                findProjs.visit();
                List<ProjectExpression> projs = findProjs.getProjs();

                //create a mapping of project expression to their inputs
                for(ProjectExpression proj : projs){
                    proj2InpRel.put(proj, oldPos2Rel.get(proj.getInputNum()));
                }

                newExpPlans.add(expPlan);

                newFlattens.add(flattens[i]);
                if(newUserSchema != null)
                    newUserSchema.add(userSchema.get(i));

            }
        }

        //get mapping of LoGenerate input relation to current position
        Map<LogicalRelationalOperator, Integer> rel2pos = new HashMap<LogicalRelationalOperator, Integer>();
        List<Operator> newGenPreds = innerPlan.getPredecessors(gen);
        int numNewGenPreds = 0;
        if(newGenPreds != null)
            numNewGenPreds = newGenPreds.size();
           
        for(int i=0; i<numNewGenPreds; i++){
            rel2pos.put((LogicalRelationalOperator) newGenPreds.get(i),i);
        }
       
        //correct the input num for projects
        for(Entry<ProjectExpression, LogicalRelationalOperator> projAndInp : proj2InpRel.entrySet()){
           ProjectExpression proj = projAndInp.getKey();
           LogicalRelationalOperator rel = projAndInp.getValue();
           proj.setInputNum(rel2pos.get(rel));
        }
       
        // set the new lists
        gen.setOutputPlans(newExpPlans);
        gen.setFlattenFlags(Booleans.toArray(newFlattens));
View Full Code Here


            return null;
        }

        Operator outputOp = expPlan.getOperators().next();
        if(outputOp instanceof ProjectExpression){
            ProjectExpression proj = (ProjectExpression)outputOp;
            //check if ProjectExpression is projectStar
            if(proj.isProjectStar()){
                //now check if its input is a LOInnerLoad and it is projectStar
                // or range project
                LogicalRelationalOperator inputRel = oldPos2Rel.get(proj.getInputNum());
                if(! (inputRel  instanceof LOInnerLoad)){
                    return null;
                }

                ProjectExpression innerProj = ((LOInnerLoad) inputRel).getProjection();
                if( innerProj.isRangeOrStarProject()){
                    return proj;
                }
            }
        }
        return null;
View Full Code Here

     */
    private LogicalExpressionPlan createExpPlanWithProj(
            LogicalRelationalOperator attachRel,
            int inputNum, int colNum) {
        LogicalExpressionPlan newExpPlan = new LogicalExpressionPlan();
        ProjectExpression newProj =
            new ProjectExpression(newExpPlan, inputNum, colNum, attachRel);
        newExpPlan.add(newProj);
        return newExpPlan;
    }
View Full Code Here

     * @throws FrontendException
     */
    private ProjectExpression getProjectStar(LogicalExpressionPlan expPlan)
    throws FrontendException {
        List<Operator> outputs = expPlan.getSources();
        ProjectExpression projStar = null;
        for(Operator outputOp : outputs){
            if(outputOp instanceof ProjectExpression){
                ProjectExpression proj = (ProjectExpression)outputOp;
                if(proj.isRangeOrStarProject()){
                    if(outputs.size() > 1){
                        String msg = "More than one operator in an expression plan" +
                        " containing project star(*)/project-range (..)";
                        throw new VisitorException(proj,
                                msg,
View Full Code Here

        // store column number as a ProjectExpression in a plan
        // to be able to dynamically adjust column number during optimization
        LogicalExpressionPlan exp = new LogicalExpressionPlan();
       
        // we don't care about type, so set to -1
        prj = new ProjectExpression(exp, 0, colNum, foreach);
        this.foreach = foreach;
    }
View Full Code Here

       
        // store column number as a ProjectExpression in a plan
        // to be able to dynamically adjust column number during optimization
        LogicalExpressionPlan exp = new LogicalExpressionPlan();
       
        this.prj = new ProjectExpression( exp, 0, colAlias, null, foreach );
        this.foreach = foreach;
    }
View Full Code Here

                opsList.add(opsIterator.next());
            }
            if(opsList.size() != 1 || !(opsList.get(0) instanceof ProjectExpression)) {
                throw new FrontendException(this, "Unsupported operator in inner plan: " + opsList.get(0), 2237);
            }
            ProjectExpression project = (ProjectExpression) opsList.get(0);
           
            //create SortColInfo from the project
            if(project.isProjectStar()){
                //there is no input schema, that is why project-star is still here
                // we don't know how many columns are represented by this
                //so don't add further columns to sort list
                return new SortInfo(sortColInfoList);
            }
            if(project.isRangeProject()){
                if(project.getEndCol() < 0){
                    //stop here for
                    // same reason as project-star condition above
                    //(unkown number of columns this represents)
                    return new SortInfo(sortColInfoList);
                }
                //expand the project-range into multiple SortColInfos
                for(int cnum = project.getStartCol(); cnum < project.getEndCol(); cnum++){
                    sortColInfoList.add(
                            new SortColInfo(null, cnum, getOrder(mAscCols,i))
                    );
                }
            }
            else{
                int sortColIndex = project.getColNum();
                String sortColName = (schema == null) ? null :
                    schema.getField(sortColIndex).alias;

                sortColInfoList.add(
                        new SortColInfo(sortColName, sortColIndex, getOrder(mAscCols,i))
View Full Code Here

                    LOInnerLoad innerLoad = new LOInnerLoad(innerPlan, foreach, rf.getIndex());
                    innerPlan.add(innerLoad);
                    innerPlan.connect(innerLoad, gen);

                    LogicalExpressionPlan exp = new LogicalExpressionPlan();
                    ProjectExpression prj = new ProjectExpression(exp, i, -1, gen);
                    exp.add(prj);
                    exps.add(exp);
                }

            } else {
View Full Code Here

        }

        // If project of the innerLoad is not in INPUTUIDS, remove this innerLoad
        Set<LOInnerLoad> innerLoadsToRemove = new HashSet<LOInnerLoad>();
        for (LOInnerLoad innerLoad: innerLoads) {
            ProjectExpression project = innerLoad.getProjection();
            if (project.isProjectStar()) {
                LogicalSchema.LogicalFieldSchema tupleFS = project.getFieldSchema();
                // Check the first component of the star projection
                long uid = tupleFS.schema.getField(0).uid;
                if (!inputUids.contains(uid))
                    innerLoadsToRemove.add(innerLoad);
            }
            else {
                if (!inputUids.contains(project.getFieldSchema().uid))
                    innerLoadsToRemove.add(innerLoad);
            }
        }

        // Find the logical operator immediate precede LOGenerate which should be removed (the whole branch)
View Full Code Here

                Iterator<Operator> iter = filterPlan.getOperators();
                Operator op = null;
                while( iter.hasNext() ) {
                    op = iter.next();
                    if( op instanceof ProjectExpression ) {
                        ProjectExpression proj = (ProjectExpression)op;
                        if( proj.isProjectStar() ) {
                            //project-range is always expanded when schema is
                            //available, so nothing to do here for it
                            LogicalRelationalOperator pred = (LogicalRelationalOperator)filter.getPlan().getPredecessors(filter).get(0);
                            LogicalSchema predSchema = pred.getSchema();
                            if (predSchema!=null) {
                                for (int i=0;i<predSchema.size();i++) {
                                    uids.add(predSchema.getField(i).uid);
                                    types.add(predSchema.getField(i).type);
                                }
                            }
                        } else {
                            uids.add(proj.getFieldSchema().uid);
                            types.add(proj.getFieldSchema().type);
                        }
                    }
                }

            }
View Full Code Here

TOP

Related Classes of org.apache.pig.newplan.logical.expression.ProjectExpression

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.