Package org.apache.pig.impl.logicalLayer

Examples of org.apache.pig.impl.logicalLayer.LogicalOperator


            DerivedDataVisitor visitor = null;
            LineageTracer lineage = null;
            // create affinity groups
            if (cg.getInputs().size() == 1) {
                affinityGroups = new HashMap<IdentityHashSet<Tuple>, Integer>();
                LogicalOperator childOp = cg.getInputs().get(0);
                visitor = new DerivedDataVisitor(childOp, null, baseData,
                        LogToPhyMap, physPlan);
                try {
                    visitor.visit();
                } catch (VisitorException e) {
                    log.error(e.getMessage());
                }

                lineage = visitor.lineage;

                DataBag bag = visitor.evaluateIsolatedOperator(cg);
                for (Iterator<Tuple> it = bag.iterator(); it.hasNext();) {
                    DataBag field;
                    try {
                        field = (DataBag) it.next().get(1);
                    } catch (ExecException e) {
                        // TODO Auto-generated catch block
                        e.printStackTrace();
                        log.error(e.getMessage());
                        throw new VisitorException(
                                "Error trimming operator COGROUP operator "
                                        + cg.getAlias()
                                        + "in example generator");
                    }
                    IdentityHashSet<Tuple> set = new IdentityHashSet<Tuple>();
                    affinityGroups.put(set, 2);
                    for (Iterator<Tuple> it1 = field.iterator(); it1.hasNext();) {
                        set.add(it1.next());
                    }
                }

                // add the equivalence classes obtained from derived data
                // creation
                for (IdentityHashSet<Tuple> set : visitor.EqClasses) {
                    affinityGroups.put(set, 1);
                }
                AffinityGroups.put(cg.getInputs().get(0), affinityGroups);
                Lineage.put(cg.getInputs().get(0), lineage);

            } else {
                List<DataBag> inputs = new LinkedList<DataBag>();
                visitor = new DerivedDataVisitor(cg, null, baseData,
                        LogToPhyMap, physPlan);
                affinityGroups = new HashMap<IdentityHashSet<Tuple>, Integer>();
                for (int i = 0; i < cg.getInputs().size(); i++) {
                    // affinityGroups = new HashMap<IdentityHashSet<Tuple>,
                    // Integer>();
                    LogicalOperator childOp = cg.getInputs().get(i);
                    // visitor = new DerivedDataVisitor(cg.getInputs().get(i),
                    // null, baseData, LogToPhyMap, physPlan);
                    visitor.setOperatorToEvaluate(childOp);
                    try {
                        visitor.visit();
View Full Code Here


        continueTrimming = checkCompleteness(op);

        if (continueTrimming == false)
            return;

        LogicalOperator childOp = plan.getPredecessors(op).get(0);

        DerivedDataVisitor visitor = new DerivedDataVisitor(childOp, null,
                baseData, LogToPhyMap, physPlan);
        try {
            visitor.visit();
View Full Code Here

    public static String printTabular(LogicalPlan lp,
            Map<LogicalOperator, DataBag> exampleData) {
        StringBuffer output = new StringBuffer();

        LogicalOperator currentOp = lp.getLeaves().get(0);
        printTabular(currentOp, exampleData, output);
        return output.toString();
    }
View Full Code Here

                List<LogicalOperator> listOp = lp.getSuccessors(op);

                if (null != listOp) {
                    Iterator<LogicalOperator> iter = listOp.iterator();
                    while (iter.hasNext()) {
                        LogicalOperator lop = iter.next();
                        System.err.println("Successor: "
                                + lop.getClass().getName() + " object " + lop);
                    }
                }
            }
            lp = refineLogicalPlan(lp);
            assertTrue(lp != null);
View Full Code Here

                           logicalOpTable,
                           aliasOp,
                           fileNameMap);

        Assert.assertTrue(lp.size()>1);
        LogicalOperator op = lp.getLeaves().get(0);
       
        Assert.assertTrue(op instanceof LOStore);
        LOStore store = (LOStore)op;

        String p = store.getOutputFile().getFileName();
View Full Code Here

                                           aliases,
                                           logicalOpTable,
                                           aliasOp,
                                           fileNameMap);
            Assert.assertTrue(lp.size()>0);
            LogicalOperator op = lp.getRoots().get(0);
           
            Assert.assertTrue(op instanceof LOLoad);
            LOLoad load = (LOLoad)op;
   
            String p = load.getInputFile().getFileName();
View Full Code Here

                           logicalOpTable,
                           aliasOp,
                           fileNameMap);

        Assert.assertTrue(lp.size()>1);
        LogicalOperator op = lp.getLeaves().get(0);
       
        Assert.assertTrue(op instanceof LOStore);
        LOStore store = (LOStore)op;

        String p = store.getOutputFile().getFileName();
View Full Code Here

                List<LogicalOperator> listOp = lp.getSuccessors(op);

                if (null != listOp) {
                    Iterator<LogicalOperator> iter = listOp.iterator();
                    while (iter.hasNext()) {
                        LogicalOperator lop = iter.next();
                        System.err.println("Successor: "
                                + lop.getClass().getName() + " object " + lop);
                    }
                }
            }
            lp = refineLogicalPlan(lp);
            assertTrue(lp != null);
View Full Code Here

     *
     */

    protected void visit(LOSort s) throws VisitorException {
        s.unsetSchema();
        LogicalOperator input = s.getInput() ;
       
        // Type checking internal plans.
        for(int i=0;i < s.getSortColPlans().size(); i++) {
           
            LogicalPlan sortColPlan = s.getSortColPlans().get(i) ;

            // Check that the inner plan has only 1 output port
            if (!sortColPlan.isSingleLeafPlan()) {
                int errCode = 1057;
                String msg = "Sort's inner plan can only have one output (leaf)" ;
                msgCollector.collect(msg, MessageType.Error) ;
                throw new TypeCheckerException(msg, errCode, PigException.INPUT) ;
            }

            checkInnerPlan(s.getAlias(), sortColPlan) ;
            // TODO: May have to check SortFunc compatibility here in the future
                      
        }
       
        s.setType(input.getType()) // This should be bag always.

        try {
            // Compute the schema
            s.getSchema() ;
        }
View Full Code Here

                                                    = join.getJoinPlans() ;
        List<LogicalOperator> inputs = join.getInputs() ;
       
        // Type checking internal plans.
        for(int i=0;i < inputs.size(); i++) {
            LogicalOperator input = inputs.get(i) ;
            List<LogicalPlan> innerPlans
                        = new ArrayList<LogicalPlan>(joinColPlans.get(input)) ;

            for(int j=0; j < innerPlans.size(); j++) {

                LogicalPlan innerPlan = innerPlans.get(j) ;
               
                // Check that the inner plan has only 1 output port
                if (!innerPlan.isSingleLeafPlan()) {
                    int errCode = 1057;
                    String msg = "Join's inner plans can only"
                                 + " have one output (leaf)" ;
                    msgCollector.collect(msg, MessageType.Error) ;
                    throw new TypeCheckerException(msg, errCode, PigException.INPUT) ;
                }

                checkInnerPlan(join.getAlias(), innerPlans.get(j)) ;
            }
        }
       
        try {

            if (!join.isTupleJoinCol()) {
                // merge all the inner plan outputs so we know what type
                // our group column should be

                // TODO: Don't recompute schema here
                //byte groupType = schema.getField(0).type ;
                byte groupType = join.getAtomicJoinColType() ;

                // go through all inputs again to add cast if necessary
                for(int i=0;i < inputs.size(); i++) {
                    LogicalOperator input = inputs.get(i) ;
                    List<LogicalPlan> innerPlans
                                = new ArrayList<LogicalPlan>(joinColPlans.get(input)) ;
                    // Checking innerPlan size already done above
                    byte innerType = innerPlans.get(0).getSingleLeafPlanOutputType() ;
                    if (innerType != groupType) {
                        insertAtomicCastForJoinInnerPlan(innerPlans.get(0),
                                                            join,
                                                            groupType) ;
                    }
                }
            }
            else {

                // TODO: Don't recompute schema here
                //Schema groupBySchema = schema.getField(0).schema ;
                Schema groupBySchema = join.getTupleJoinSchema() ;

                // go through all inputs again to add cast if necessary
                for(int i=0;i < inputs.size(); i++) {
                    LogicalOperator input = inputs.get(i) ;
                    List<LogicalPlan> innerPlans
                                = new ArrayList<LogicalPlan>(joinColPlans.get(input)) ;
                    for(int j=0;j < innerPlans.size(); j++) {
                        LogicalPlan innerPlan = innerPlans.get(j) ;
                        byte innerType = innerPlan.getSingleLeafPlanOutputType() ;
View Full Code Here

TOP

Related Classes of org.apache.pig.impl.logicalLayer.LogicalOperator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.