Package org.apache.pig.impl.logicalLayer

Examples of org.apache.pig.impl.logicalLayer.LogicalOperator


        }

        @Override
        protected void visit(LOStore store) throws VisitorException {
           
            LogicalOperator storePred = store.getPlan().getPredecessors(store).get(0);
            if(storePred == null){
                int errCode = 2051;
                String msg = "Did not find a predecessor for Store." ;
                throw new VisitorException(msg, errCode, PigException.BUG);   
            }
           
            SortInfo sortInfo = null;
            if(storePred instanceof LOLimit) {
                storePred = store.getPlan().getPredecessors(storePred).get(0);
            } else if (storePred instanceof LOSplitOutput) {
                LOSplitOutput splitOutput = (LOSplitOutput)storePred;
                // We assume this is the LOSplitOutput we injected for this case:
                // b = order a by $0; store b into '1'; store b into '2';
                // In this case, we should mark both '1' and '2' as sorted
                LogicalPlan conditionPlan = splitOutput.getConditionPlan();
                if (conditionPlan.getRoots().size()==1) {
                    LogicalOperator root = conditionPlan.getRoots().get(0);
                    if (root instanceof LOConst) {
                        Object value = ((LOConst)root).getValue();
                        if (value instanceof Boolean && (Boolean)value==true) {
                            LogicalOperator split = splitOutput.getPlan().getPredecessors(splitOutput).get(0);
                            if (split instanceof LOSplit)
                                storePred = store.getPlan().getPredecessors(split).get(0);
                        }
                    }
                }
View Full Code Here


        LogicalPlan getPlan(String alias) throws IOException {
            LogicalPlan plan = lp;
               
            if (alias != null) {
                LogicalOperator op = aliasOp.get(alias);
                if(op == null) {
                    int errCode = 1003;
                    String msg = "Unable to find an operator for alias " + alias;
                    throw new FrontendException(msg, errCode, PigException.INPUT);
                }
View Full Code Here

            LogicalPlan tmpLp = parseQuery(query, startLine);
           
            // store away the query for use in cloning later
            scriptCache.add(query);
            if (tmpLp.getLeaves().size() == 1) {
                LogicalOperator op = tmpLp.getSingleLeafPlanOutputOp();
               
                // Check if we just processed a LOStore i.e. STORE
                if (op instanceof LOStore) {

                    if (!batchMode) {
                        lp = tmpLp;
                        try {
                            execute();
                        } catch (Exception e) {
                            int errCode = 1002;
                            String msg = "Unable to store alias "
                                    + op.getOperatorKey().getId();
                            throw new FrontendException(msg, errCode,
                                    PigException.INPUT, e);
                        }
                    } else {
                        if (0 == ignoreNumStores) {
View Full Code Here

            DerivedDataVisitor visitor = null;
            LineageTracer lineage = null;
            // create affinity groups
            if (cg.getInputs().size() == 1) {
                affinityGroups = new HashMap<IdentityHashSet<Tuple>, Integer>();
                LogicalOperator childOp = cg.getInputs().get(0);
                visitor = new DerivedDataVisitor(childOp, null, baseData,
                        LogToPhyMap, physPlan);
                try {
                    visitor.visit();
                } catch (VisitorException e) {
                    log.error(e.getMessage());
                }

                lineage = visitor.lineage;

                DataBag bag = visitor.evaluateIsolatedOperator(cg);
                for (Iterator<Tuple> it = bag.iterator(); it.hasNext();) {
                    DataBag field;
                    try {
                        field = (DataBag) it.next().get(1);
                    } catch (ExecException e) {
                        // TODO Auto-generated catch block
                        e.printStackTrace();
                        log.error(e.getMessage());
                        throw new VisitorException(
                                "Error trimming operator COGROUP operator "
                                        + cg.getAlias()
                                        + "in example generator");
                    }
                    IdentityHashSet<Tuple> set = new IdentityHashSet<Tuple>();
                    affinityGroups.put(set, 2);
                    for (Iterator<Tuple> it1 = field.iterator(); it1.hasNext();) {
                        set.add(it1.next());
                    }
                }

                // add the equivalence classes obtained from derived data
                // creation
                for (IdentityHashSet<Tuple> set : visitor.EqClasses) {
                    affinityGroups.put(set, 1);
                }
                AffinityGroups.put(cg.getInputs().get(0), affinityGroups);
                Lineage.put(cg.getInputs().get(0), lineage);

            } else {
                List<DataBag> inputs = new LinkedList<DataBag>();
                visitor = new DerivedDataVisitor(cg, null, baseData,
                        LogToPhyMap, physPlan);
                affinityGroups = new HashMap<IdentityHashSet<Tuple>, Integer>();
                for (int i = 0; i < cg.getInputs().size(); i++) {
                    // affinityGroups = new HashMap<IdentityHashSet<Tuple>,
                    // Integer>();
                    LogicalOperator childOp = cg.getInputs().get(i);
                    // visitor = new DerivedDataVisitor(cg.getInputs().get(i),
                    // null, baseData, LogToPhyMap, physPlan);
                    visitor.setOperatorToEvaluate(childOp);
                    try {
                        visitor.visit();
View Full Code Here

        continueTrimming = checkCompleteness(op);

        if (continueTrimming == false)
            return;

        LogicalOperator childOp = plan.getPredecessors(op).get(0);

        DerivedDataVisitor visitor = new DerivedDataVisitor(childOp, null,
                baseData, LogToPhyMap, physPlan);
        try {
            visitor.visit();
View Full Code Here

    @Override
    public Schema getSchema() throws FrontendException{
        if (!mIsSchemaComputed) {
            // get our parent's schema
            try {
                LogicalOperator input = mPlan.getPredecessors(this).get(0);
                if (null == input) {
                    int errCode = 1006;
                    String msg = "Could not find operator in plan";
                    throw new FrontendException(msg, errCode, PigException.INPUT, false, null);
                }
                if (input.getSchema()!=null) {
                    mSchema = new Schema(input.getSchema());
                    for (int i=0;i<input.getSchema().size();i++)
                        mSchema.getField(i).setParent(input.getSchema().getField(i).canonicalName, input);
                }
                else
                    mSchema = null;
                mIsSchemaComputed = true;
            } catch (FrontendException fe) {
View Full Code Here

     * @see org.apache.pig.impl.plan.Operator#rewire(org.apache.pig.impl.plan.Operator, org.apache.pig.impl.plan.Operator)
     */
    @Override
    public void rewire(Operator<LOVisitor> oldPred, int oldPredIndex, Operator<LOVisitor> newPred, boolean useOldPred) throws PlanException {
        super.rewire(oldPred, oldPredIndex, newPred, useOldPred);
        LogicalOperator previous = (LogicalOperator) oldPred;
        LogicalOperator current = (LogicalOperator) newPred;
        try {
            ProjectFixerUpper projectFixer = new ProjectFixerUpper(
                    mCondPlan, previous, oldPredIndex, current, useOldPred, this);
            projectFixer.visit();
        } catch (VisitorException ve) {
View Full Code Here

       }
       
        public void visit(LOProject project) throws VisitorException {
            int col = project.getCol();
           
            LogicalOperator lg = project.getExpression();
            LogicalOperator succed = oldLogicalPlan.getSuccessors(lg).get(0);
            int input = oldLogicalPlan.getPredecessors(succed).indexOf(lg);
                       
            // get data type of projection
            byte t = project.getType();           
            ProjectExpression pe = new ProjectExpression(exprPlan, t, input, project.isStar()?-1:col);         
View Full Code Here

                       
            innerOpsMap = new HashMap<LogicalOperator, LogicalRelationalOperator>();           
        }     
       
        public void visit(LOProject project) throws VisitorException {
            LogicalOperator op = project.getExpression();
           
            if (op == oldLogicalPlan.getPredecessors(oldForeach).get(0)) {
                // if this projection is to get a field from outer plan, change it
                // to LOInnerLoad
               
                LOInnerLoad innerLoad = new LOInnerLoad(newInnerPlan, foreach, project.isStar()?-1:project.getCol());   
               
                newInnerPlan.add(innerLoad);               
                innerOpsMap.put(project, innerLoad);               
           
           
                LogicalOperator succ = null;
                if (mPlan.getSuccessors(project) != null) {
                    succ = mPlan.getSuccessors(project).get(0);
                }
               
                // The logical plan part for this foreach plan is done, add ProjectExpression
View Full Code Here

     * @throws IOException
     */
    public Schema dumpSchemaNested(String alias, String nestedAlias) throws IOException{
        LogicalPlan lp = getPlanFromAlias(alias, "describe");
        lp = compileLp(alias, false);
        LogicalOperator op = lp.getLeaves().get(0);
        if(op instanceof LOForEach) {
            return ((LOForEach)op).dumpNestedSchema(alias, nestedAlias);
        }
        else {
            int errCode = 1001;
View Full Code Here

TOP

Related Classes of org.apache.pig.impl.logicalLayer.LogicalOperator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.