Package org.apache.pig.impl.logicalLayer

Examples of org.apache.pig.impl.logicalLayer.LogicalPlan


     */

    @Override
    protected void visit(LOFilter filter) throws VisitorException {
        filter.unsetSchema();
        LogicalPlan comparisonPlan = filter.getComparisonPlan() ;
       
        // Check that the inner plan has only 1 output port
        if (!comparisonPlan.isSingleLeafPlan()) {
            int errCode = 1057;
            String msg = "Filter's cond plan can only have one output (leaf)" ;
            msgCollector.collect(msg, MessageType.Error) ;
            throw new TypeCheckerException(msg, errCode, PigException.INPUT) ;
        }

        checkInnerPlan(filter.getAlias(), comparisonPlan) ;
             
        byte innerCondType = comparisonPlan.getLeaves().get(0).getType() ;
        if (innerCondType != DataType.BOOLEAN) {
            int errCode = 1058;
            String msg = "Filter's condition must evaluate to boolean. Found: " + DataType.findTypeName(innerCondType);
            msgCollector.collect(msg, MessageType.Error) ;
            throw new TypeCheckerException(msg, errCode, PigException.INPUT) ;
View Full Code Here


            List<LogicalPlan> innerPlans
                        = new ArrayList<LogicalPlan>(joinColPlans.get(input)) ;

            for(int j=0; j < innerPlans.size(); j++) {

                LogicalPlan innerPlan = innerPlans.get(j) ;
               
                // Check that the inner plan has only 1 output port
                if (!innerPlan.isSingleLeafPlan()) {
                    int errCode = 1057;
                    String msg = "Join's inner plans can only"
                                 + " have one output (leaf)" ;
                    msgCollector.collect(msg, MessageType.Error) ;
                    throw new TypeCheckerException(msg, errCode, PigException.INPUT) ;
                }

                checkInnerPlan(join.getAlias(), innerPlans.get(j)) ;
            }
        }
       
        try {

            if (!join.isTupleJoinCol()) {
                // merge all the inner plan outputs so we know what type
                // our group column should be

                // TODO: Don't recompute schema here
                //byte groupType = schema.getField(0).type ;
                byte groupType = join.getAtomicJoinColType() ;

                // go through all inputs again to add cast if necessary
                for(int i=0;i < inputs.size(); i++) {
                    LogicalOperator input = inputs.get(i) ;
                    List<LogicalPlan> innerPlans
                                = new ArrayList<LogicalPlan>(joinColPlans.get(input)) ;
                    // Checking innerPlan size already done above
                    byte innerType = innerPlans.get(0).getSingleLeafPlanOutputType() ;
                    if (innerType != groupType) {
                        insertAtomicCastForJoinInnerPlan(innerPlans.get(0),
                                                            join,
                                                            groupType) ;
                    }
                }
            }
            else {

                // TODO: Don't recompute schema here
                //Schema groupBySchema = schema.getField(0).schema ;
                Schema groupBySchema = join.getTupleJoinSchema() ;

                // go through all inputs again to add cast if necessary
                for(int i=0;i < inputs.size(); i++) {
                    LogicalOperator input = inputs.get(i) ;
                    List<LogicalPlan> innerPlans
                                = new ArrayList<LogicalPlan>(joinColPlans.get(input)) ;
                    for(int j=0;j < innerPlans.size(); j++) {
                        LogicalPlan innerPlan = innerPlans.get(j) ;
                        byte innerType = innerPlan.getSingleLeafPlanOutputType() ;
                        byte expectedType = DataType.BYTEARRAY ;

                        if (!DataType.isAtomic(innerType) && (DataType.TUPLE != innerType)) {
                            int errCode = 1057;
                            String msg = "Join's inner plans can only"
View Full Code Here

            List<LogicalPlan> innerPlans
                        = new ArrayList<LogicalPlan>(groupByPlans.get(input)) ;

            for(int j=0; j < innerPlans.size(); j++) {

                LogicalPlan innerPlan = innerPlans.get(j) ;
               
                // Check that the inner plan has only 1 output port
                if (!innerPlan.isSingleLeafPlan()) {
                    int errCode = 1057;
                    String msg = "COGroup's inner plans can only"
                                 + "have one output (leaf)" ;
                    msgCollector.collect(msg, MessageType.Error) ;
                    throw new TypeCheckerException(msg, errCode, PigException.INPUT) ;
                }

                checkInnerPlan(cg.getAlias(), innerPlans.get(j)) ;
            }

        }

        try {

            if (!cg.isTupleGroupCol()) {
                // merge all the inner plan outputs so we know what type
                // our group column should be

                // TODO: Don't recompute schema here
                //byte groupType = schema.getField(0).type ;
                byte groupType = cg.getAtomicGroupByType() ;

                // go through all inputs again to add cast if necessary
                for(int i=0;i < inputs.size(); i++) {
                    LogicalOperator input = inputs.get(i) ;
                    List<LogicalPlan> innerPlans
                                = new ArrayList<LogicalPlan>(groupByPlans.get(input)) ;
                    // Checking innerPlan size already done above
                    byte innerType = innerPlans.get(0).getSingleLeafPlanOutputType() ;
                    if (innerType != groupType) {
                        insertAtomicCastForCOGroupInnerPlan(innerPlans.get(0),
                                                            cg,
                                                            groupType) ;
                    }
                }
            }
            else {

                // TODO: Don't recompute schema here
                //Schema groupBySchema = schema.getField(0).schema ;
                Schema groupBySchema = cg.getTupleGroupBySchema() ;

                // go through all inputs again to add cast if necessary
                for(int i=0;i < inputs.size(); i++) {
                    LogicalOperator input = inputs.get(i) ;
                    List<LogicalPlan> innerPlans
                                = new ArrayList<LogicalPlan>(groupByPlans.get(input)) ;
                    for(int j=0;j < innerPlans.size(); j++) {
                        LogicalPlan innerPlan = innerPlans.get(j) ;
                        byte innerType = innerPlan.getSingleLeafPlanOutputType() ;
                        byte expectedType = DataType.BYTEARRAY ;

                        if (!DataType.isAtomic(innerType) && (DataType.TUPLE != innerType)) {
                            int errCode = 1061;
                            String msg = "Sorry, group by complex types"
View Full Code Here

            // Have to resolve all inner plans before calling getSchema
            int outputSchemaIdx = 0 ;
            for(int i=0;i < plans.size(); i++) {

                LogicalPlan plan = plans.get(i) ;

                // Check that the inner plan has only 1 output port
                if (!plan.isSingleLeafPlan()) {
                    int errCode = 1057;
                    String msg = "Generate's expression plan can "
                                 + " only have one output (leaf)" ;
                    msgCollector.collect(msg, MessageType.Error) ;
                    throw new TypeCheckerException(msg, errCode, PigException.INPUT) ;
                }

                List<LogicalOperator> rootList = plan.getRoots() ;
                for(int j=0; j<rootList.size(); j++) {
                    LogicalOperator innerRoot = rootList.get(j) ;
                    // TODO: Support MAP dereference
                    if (innerRoot instanceof LOProject) {
                        resolveLOProjectType((LOProject) innerRoot) ;
View Full Code Here

     */
    private LOForEach insertCastForEachInBetweenIfNecessary(LogicalOperator fromOp,
                                                       LogicalOperator toOp,
                                                       Schema targetSchema)
                                                    throws VisitorException {
        LogicalPlan currentPlan =  mCurrentWalker.getPlan() ;

        /*
        // Make sure that two operators are in the same plan
        if (fromOp.getPlan() != toOp.getPlan()) {
            throw new AssertionError("Two operators have toOp be in the same plan") ;
        }
        // Mare sure that they are in the plan we're looking at
        if (fromOp.getPlan() != toOp.getPlan()) {
            throw new AssertionError("Cannot manipulate any other plan"
                                    +" than the current one") ;
        }
        */

        // Make sure that they are adjacent and the direction
        // is from "fromOp" to "toOp"
        List<LogicalOperator> preList = currentPlan.getPredecessors(toOp) ;
        boolean found = false ;
        for(LogicalOperator tmpOp: preList) {
            // compare by reference
            if (tmpOp == fromOp) {
                found = true ;
                break ;
            }
        }

        if (!found) {
            int errCode = 1077;
            String msg = "Two operators that require a cast in between are not adjacent.";
            throw new TypeCheckerException(msg, errCode, PigException.INPUT);
        }

        // retrieve input schema to be casted
        // this will be used later
        Schema fromSchema = null ;
        try {
            fromSchema = fromOp.getSchema() ;
        }
        catch(FrontendException fe) {
            int errCode = 1055;
            String msg = "Problem while reading schema from input of " + fromOp.getClass().getSimpleName();
            throw new TypeCheckerException(msg, errCode, PigException.BUG, fe);
        }

        // make sure the supplied targetSchema has the same number of members
        // as number of output fields from "fromOp"
        if (fromSchema.size() != targetSchema.size()) {
            int errCode = 1078;
            String msg = "Schema size mismatch for casting. Input schema size: " + fromSchema.size() + ". Target schema size: " + targetSchema.size();
            throw new TypeCheckerException(msg, errCode, PigException.INPUT);
        }

        // Plans inside Generate. Fields that do not need casting will only
        // have Project. Fields that need casting will have Project + Cast
        ArrayList<LogicalPlan> generatePlans = new ArrayList<LogicalPlan>() ;

        int castNeededCounter = 0 ;
        for(int i=0;i < fromSchema.size(); i++) {

            LogicalPlan genPlan = new LogicalPlan() ;
            LOProject project = new LOProject(genPlan,
                                              genNewOperatorKey(fromOp),
                                              fromOp,
                                              i) ;
            project.setSentinel(true);
            genPlan.add(project);

            // add casting if necessary by comparing target types
            // to the input schema
            FieldSchema fs = null ;
            try {
                fs = fromSchema.getField(i) ;
            }
            catch(FrontendException fee) {
                int errCode = 1063;
                String msg = "Problem while reading"
                                + " field schema from input while"
                                + " inserting cast " ;
                msgCollector.collect(msg, MessageType.Error) ;
                throw new TypeCheckerException(msg, errCode, PigException.INPUT, fee) ;
            }

            // This only does "shallow checking"

            byte inputFieldType ;

            try {
                inputFieldType = targetSchema.getField(i).type ;
            }
            catch (FrontendException fee) {
                int errCode = 1064;
                String msg = "Problem reading column " + i + " from schema: " + targetSchema;
                throw new TypeCheckerException(msg, errCode, PigException.INPUT, fee) ;               
            }

            if (inputFieldType != fs.type) {
                castNeededCounter++ ;
                LOCast cast = new LOCast(genPlan,
                                         genNewOperatorKey(fromOp),
                                         inputFieldType) ;
                genPlan.add(cast) ;
                try {
                    genPlan.connect(project, cast);
                }
                catch (PlanException pe) {
                    // This should never happen
                    int errCode = 2059;
                    String msg = "Problem with inserting cast operator for project in plan.";
View Full Code Here

            Schema s = op.getSchema();
            if(null != s) {
              FieldSchema fieldSchema = s.findFieldSchema( parentCanonicalName );
                getLoadFuncSpec( fieldSchema, loadFuncSpecMap );
            } else {
                LogicalPlan lp = op.getPlan();
                for(LogicalOperator pred: lp.getPredecessors(op)) {
                    FuncSpec lfSpec = getLoadFuncSpec(pred, parentCanonicalName);
                    if(null != lfSpec) loadFuncSpecMap.put(lfSpec.getClassName(), lfSpec);
                }
            }
        }
View Full Code Here

            Schema determinedSchema = null;
            if(LOLoad.class.getName().equals(operatorClassName)) {
                determinedSchema = ((LOLoad)lo).getDeterminedSchema();
            }
            for (int i = 0; i < s.size(); i++) {
                LogicalPlan p = new LogicalPlan();
                genPlans.add(p);
                flattens.add(false);
                List<Integer> toProject = new ArrayList<Integer>(1);
                toProject.add(i);
                LOProject proj = new LOProject(p, OperatorKey.genOpKey(scope),
                    lo, toProject);
                p.add(proj);
                Schema.FieldSchema fs = s.getField(i);
                if (fs.type != DataType.BYTEARRAY) {
                    if(determinedSchema == null || (fs.type != determinedSchema.getField(i).type)) {
                            // Either no schema was determined by loader OR the type
                            // from the "determinedSchema" is different
                            // from the type specified - so we need to cast
                            LOCast cast = new LOCast(p,
                                        OperatorKey.genOpKey(scope), fs.type);
                            cast.setFieldSchema(fs);
                            p.add(cast);
                            p.connect(proj, cast);
                           
                            cast.setFieldSchema(fs.clone());
                            FuncSpec loadFuncSpec = null;
                            if(lo instanceof LOLoad) {
                                loadFuncSpec = ((LOLoad)lo).getInputFile().getFuncSpec();
View Full Code Here

                flattenList = new ArrayList<Boolean>();
               
                ArrayList<LogicalPlan> foreachInnerPlans = new ArrayList<LogicalPlan>();
               
                for(int i = 0; i < successorSchema.size(); ++i) {
                    LogicalPlan innerPlan = new LogicalPlan();
                    LOProject project = new LOProject(innerPlan, OperatorKey
                            .genOpKey(foreach.getOperatorKey().scope),
                            successor, i);
                    innerPlan.add(project);
                    foreachInnerPlans.add(innerPlan);
                    flattenList.add(false);
                }
               
                // set the flattened remapped column to true
View Full Code Here

        innerPlan.add(gen);               
       
        List<LogicalPlan> ll = forEach.getForEachPlans();
        try {
            for(int i=0; i<ll.size(); i++) {
                LogicalPlan lp = ll.get(i);
                ForeachInnerPlanVisitor v = new ForeachInnerPlanVisitor(newForeach, forEach, lp, mPlan, opsMap);
                v.visit();
               
                expPlans.add(v.exprPlan);
            }
View Full Code Here

    }
   
    public void visit(LOFilter filter) throws VisitorException {
        org.apache.pig.newplan.logical.relational.LOFilter newFilter = new org.apache.pig.newplan.logical.relational.LOFilter(logicalPlan);
       
        LogicalPlan filterPlan = filter.getComparisonPlan();
        LogicalExpressionPlan newFilterPlan = translateExpressionPlan(filterPlan, filter, newFilter);
     
        newFilter.setFilterPlan(newFilterPlan);
        newFilter.setAlias(filter.getAlias());
        newFilter.setRequestedParallelism(filter.getRequestedParallelism());
View Full Code Here

TOP

Related Classes of org.apache.pig.impl.logicalLayer.LogicalPlan

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.