Examples of POPackage


Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage

        POProject prj2 = (POProject)cogroupPlan2.getLeaves().get(0);
        assertEquals( 0, prj2.getColumn() );
        assertEquals( DataType.CHARARRAY, prj2.getResultType() );
       
        assertEquals( POPackage.class, phyPlan.getSuccessors(globalR).get(0).getClass() );
        POPackage pack = (POPackage)phyPlan.getSuccessors(globalR).get(0);
        assertEquals( DataType.TUPLE, pack.getResultType() );
    }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage

        POProject prj4 = (POProject)cogroupPlan4.getLeaves().get(0);
        assertEquals( 1, prj4.getColumn() );
        assertEquals( DataType.INTEGER, prj4.getResultType() );
       
        assertEquals( POPackage.class, phyPlan.getSuccessors(globalR).get(0).getClass() );
        POPackage pack = (POPackage)phyPlan.getSuccessors(globalR).get(0);
        assertEquals( DataType.TUPLE, pack.getResultType() );
    }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage

        assertEquals( POGlobalRearrange.class, phyPlan.getSuccessors(localR).get(0).getClass() );
        POGlobalRearrange globalR = (POGlobalRearrange)phyPlan.getSuccessors(localR).get(0);
        assertEquals( DataType.TUPLE, globalR.getResultType() );
       
        assertEquals( POPackage.class, phyPlan.getSuccessors(globalR).get(0).getClass() );
        POPackage pack = (POPackage)phyPlan.getSuccessors(globalR).get(0);
        assertEquals( DataType.TUPLE, pack.getResultType() );
    }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage

        assertEquals( POGlobalRearrange.class, phyPlan.getSuccessors(localR).get(0).getClass() );
        POGlobalRearrange globalR = (POGlobalRearrange)phyPlan.getSuccessors(localR).get(0);
        assertEquals( DataType.TUPLE, globalR.getResultType() );
       
        assertEquals( POPackage.class, phyPlan.getSuccessors(globalR).get(0).getClass() );
        POPackage pack = (POPackage)phyPlan.getSuccessors(globalR).get(0);
        assertEquals( DataType.TUPLE, pack.getResultType() );
    }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage

        POProject prj2 = (POProject)cogroupPlan2.getLeaves().get(0);
        assertEquals( 0, prj2.getColumn() );
        assertEquals( DataType.CHARARRAY, prj2.getResultType() );
       
        assertEquals( POPackage.class, phyPlan.getSuccessors(globalR).get(0).getClass() );
        POPackage pack = (POPackage)phyPlan.getSuccessors(globalR).get(0);
        assertEquals( DataType.TUPLE, pack.getResultType() );
    }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage

        POProject prj4 = (POProject)cogroupPlan4.getLeaves().get(0);
        assertEquals( 1, prj4.getColumn() );
        assertEquals( DataType.INTEGER, prj4.getResultType() );
       
        assertEquals( POPackage.class, phyPlan.getSuccessors(globalR).get(0).getClass() );
        POPackage pack = (POPackage)phyPlan.getSuccessors(globalR).get(0);
        assertEquals( DataType.TUPLE, pack.getResultType() );
    }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage

            // this call modifies the ReplFiles names of POFRJoin operators
            // within the MR plans, must be called before the plans are
            // serialized
            setupDistributedCacheForJoin(mro, pigContext, conf);

            POPackage pack = null;
            if(mro.reducePlan.isEmpty()){
                //MapOnly Job
                nwJob.setMapperClass(PigMapOnly.Map.class);
                nwJob.setNumReduceTasks(0);
                conf.set("pig.mapPlan", ObjectSerializer.serialize(mro.mapPlan));
                if(mro.isEndOfAllInputSetInMap()) {
                    // this is used in Map.close() to decide whether the
                    // pipeline needs to be rerun one more time in the close()
                    // The pipeline is rerun if there either was a stream or POMergeJoin
                    conf.set(END_OF_INP_IN_MAP, "true");
                }
            }
            else{
                //Map Reduce Job
                //Process the POPackage operator and remove it from the reduce plan
                if(!mro.combinePlan.isEmpty()){
                    POPackage combPack = (POPackage)mro.combinePlan.getRoots().get(0);
                    mro.combinePlan.remove(combPack);
                    nwJob.setCombinerClass(PigCombiner.Combine.class);
                    conf.set("pig.combinePlan", ObjectSerializer.serialize(mro.combinePlan));
                    conf.set("pig.combine.package", ObjectSerializer.serialize(combPack));
                } else if (mro.needsDistinctCombiner()) {
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage

        return curIndex;
    }
       
    private void mergeOneReducePlanWithIndex(PhysicalPlan from,
            PhysicalPlan to, int initial, int current, byte mapKeyType) throws VisitorException {                   
        POPackage pk = (POPackage)from.getRoots().get(0);
        from.remove(pk);
        if(!(pk instanceof POMultiQueryPackage)){
            // XXX the index of the original keyInfo map is always 0,
            // we need to shift the index so that the lookups works
            // with the new indexed key
            addShiftedKeyInfoIndex(initial, pk);
        }
        
        int total = current - initial;
       
        POMultiQueryPackage pkg = (POMultiQueryPackage)to.getRoots().get(0);       
        int pkCount = 0;
        if (pk instanceof POMultiQueryPackage) {
            List<POPackage> pkgs = ((POMultiQueryPackage)pk).getPackages();
            for (POPackage p : pkgs) {
                pkg.addPackage(p);
                pkCount++;
            }
            pkg.addIsKeyWrappedList(((POMultiQueryPackage)pk).getIsKeyWrappedList());
            addShiftedKeyInfoIndex(initial, current, (POMultiQueryPackage)pk);
        } else {
            pkg.addPackage(pk, mapKeyType);
            pkCount = 1;
        }
       
        if (pkCount != total) {
            int errCode = 2146;
            String msg = "Internal Error. Inconsistency in key index found during optimization.";
            throw new OptimizerException(msg, errCode, PigException.BUG);
        }
       
        PODemux demux = (PODemux)to.getLeaves().get(0);
        int plCount = 0;
        PhysicalOperator root = from.getRoots().get(0);
        if (root instanceof PODemux) {
            // flattening the inner plans of the demux operator.
            // This is based on the fact that if a plan has a demux
            // operator, then it's the only operator in the plan.
            List<PhysicalPlan> pls = ((PODemux)root).getPlans();
            for (PhysicalPlan pl : pls) {
                demux.addPlan(pl);
                plCount++;
            }
        } else {
            demux.addPlan(from);
            plCount = 1;
        }
       
        if (plCount != total) {
            int errCode = 2146;
            String msg = "Internal Error. Inconsistency in key index found during optimization.";
            throw new OptimizerException(msg, errCode, PigException.BUG);
        }

        if (pkg.isSameMapKeyType()) {
            pkg.setKeyType(pk.getKeyType());
        } else {
            pkg.setKeyType(DataType.TUPLE);
        }           
    }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage

            throw new OptimizerException(msg, errCode, PigException.BUG);
        }
        int i = 0;
        int curIndex = initialIndex;
        while (i < end) {
            POPackage pkg = pkgs.get(i);
            addShiftedKeyInfoIndex(curIndex, pkg);
            curIndex++;
            i++;
        }
        return curIndex; // could be used in a caller who recursively called this function
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage

       
    }

    private void mergeOneCombinePlanWithIndex(PhysicalPlan from,
            PhysicalPlan to, int initial, int current, byte mapKeyType) throws VisitorException {
        POPackage cpk = (POPackage)from.getRoots().get(0);
        from.remove(cpk);
       
        PODemux demux = (PODemux)to.getLeaves().get(0);
               
        POMultiQueryPackage pkg = (POMultiQueryPackage)to.getRoots().get(0);
       
        boolean isSameKeyType = pkg.isSameMapKeyType();
       
        // if current > initial + 1, it means we had
        // a split in the map of the MROper we are trying to
        // merge. In that case we would have changed the indices
        // of the POLocalRearranges in the split to be in the
        // range initial to current. To handle key, value pairs
        // coming out of those POLocalRearranges, we add
        // the Packages in the 'from' POMultiQueryPackage (in this case,
        // it has to be a POMultiQueryPackage since we had
        // a POSplit in the map) to the 'to' POMultiQueryPackage.
        // These Packages would have correct positions in the package
        // list and would be able to handle the outputs from the different
        // POLocalRearranges.
        int total = current - initial;
        int pkCount = 0;
        if (cpk instanceof POMultiQueryPackage) {
            List<POPackage> pkgs = ((POMultiQueryPackage)cpk).getPackages();
            for (POPackage p : pkgs) {
                pkg.addPackage(p);
                if (!isSameKeyType) {
                    p.setKeyType(DataType.TUPLE);
                }
                pkCount++;
            }
        } else {
            pkg.addPackage(cpk);
            pkCount = 1;
        }

        pkg.setSameMapKeyType(isSameKeyType);
       
        if (pkCount != total) {
            int errCode = 2146;
            String msg = "Internal Error. Inconsistency in key index found during optimization.";
            throw new OptimizerException(msg, errCode, PigException.BUG);
        }

        // all packages should have the same key type
        if (!isSameKeyType) {
            cpk.setKeyType(DataType.TUPLE);         
        }
       
        pkg.setKeyType(cpk.getKeyType());
       
        // See comment above for why we flatten the Packages
        // in the from plan - for the same reason, we flatten
        // the inner plans of Demux operator now.
        int plCount = 0;
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.