Package org.apache.pig.impl.plan

Examples of org.apache.pig.impl.plan.NodeIdGenerator


    private int lastBagIndex;
   
    public POJoinPackage(OperatorKey k, int rp, POPackage p, POForEach f) {
        super(k, rp);
        String scope = getOperatorKey().getScope();
        NodeIdGenerator nig = NodeIdGenerator.getGenerator();
        forEach = new POOptimizedForEach(new OperatorKey(scope,nig.getNextNodeId(scope)));
        if (p!=null)
        {
            setKeyType(p.getKeyType());
            setNumInps(p.getNumInps());
            lastBagIndex = numInputs - 1;
View Full Code Here


        }
        try {
            // Insert a split and its corresponding SplitOutput nodes into the plan
            // between node 0 and 1 / 2.
            String scope = nodes.get(0).getOperatorKey().scope;
            NodeIdGenerator idGen = NodeIdGenerator.getGenerator();
            LOSplit splitOp = new LOSplit(mPlan, new OperatorKey(scope,
                    idGen.getNextNodeId(scope)), new ArrayList<LogicalOperator>());
            splitOp.setAlias(nodes.get(0).getAlias());
            mPlan.add(splitOp);
           
            // Find all the successors and connect appropriately with split
            // and splitoutput operators.  Keep our own copy
            // of the list, as we're changing the graph by doing these calls
            // and that will change the list of predecessors.
            List<LogicalOperator> succs =
                new ArrayList<LogicalOperator>(mPlan.getSuccessors(nodes.get(0)));
            int index = -1;
            // For two successors of nodes.get(0) here is a pictorial
            // representation of the change required:
            // BEFORE:
            // Succ1  Succ2
            //  \       /
            //  nodes.get(0)
           
            //  SHOULD BECOME:
           
            // AFTER:
            // Succ1          Succ2
            //   |              |
            // SplitOutput SplitOutput
            //      \       /
            //        Split
            //          |
            //        nodes.get(0)
           
            // Here is how this will be accomplished.
            // First (the same) Split Operator will be "inserted between" nodes.get(0)
            // and all its successors. The "insertBetween" API is used which makes sure
            // the ordering of operators in the graph is preserved. So we get the following:
            // Succ1        Succ2
            //    |          |
            //   Split     Split
            //      \      / 
            //      nodes.get(0)
           
            // Then all but the first connection between nodes.get(0) and the Split
            // Operator are removed using "disconnect" - so we get the following:
            // Succ1          Succ2
            //      \       /
            //        Split
            //          |
            //        nodes.get(0)
           
            // Now a new SplitOutputOperator is "inserted between" the Split operator
            // and the successors. So we get:
            // Succ1          Succ2
            //   |              |
            // SplitOutput SplitOutput
            //      \       /
            //        Split
            //          |
            //        nodes.get(0)
           
           
            for (LogicalOperator succ : succs) {
                mPlan.insertBetween(nodes.get(0), splitOp, succ);
            }
           
            for(int i = 1; i < succs.size(); i++) {
                mPlan.disconnect(nodes.get(0), splitOp);
            }

            for (LogicalOperator succ : succs) {
                LogicalPlan condPlan = new LogicalPlan();
                LOConst cnst = new LOConst(mPlan, new OperatorKey(scope,
                        idGen.getNextNodeId(scope)), Boolean.valueOf(true));
                cnst.setType(DataType.BOOLEAN);
                condPlan.add(cnst);
                LOSplitOutput splitOutput = new LOSplitOutput(mPlan,
                        new OperatorKey(scope, idGen.getNextNodeId(scope)), ++index, condPlan);
                splitOp.addOutput(splitOutput);
                mPlan.add(splitOutput);
                mPlan.insertBetween(splitOp, splitOutput, succ);
                splitOutput.setAlias(splitOp.getAlias());
            }
View Full Code Here

        }

        public static void replaceWithPOJoinPackage(PhysicalPlan plan,
                POPackage pack, POForEach forEach, String chunkSize) throws VisitorException {
            String scope = pack.getOperatorKey().scope;
            NodeIdGenerator nig = NodeIdGenerator.getGenerator();
            POJoinPackage joinPackage;
            joinPackage = new POJoinPackage(
                        new OperatorKey(scope, nig.getNextNodeId(scope)),
                        -1, pack, forEach);
            joinPackage.setChunkSize(Long.parseLong(chunkSize));
            List<PhysicalOperator> succs = plan.getSuccessors(forEach);
            if (succs!=null)
            {
View Full Code Here

        }

        public static void replaceWithPOJoinPackage(PhysicalPlan plan,
                POPackage pack, POForEach forEach, String chunkSize) throws VisitorException {
            String scope = pack.getOperatorKey().scope;
            NodeIdGenerator nig = NodeIdGenerator.getGenerator();
            POJoinPackage joinPackage;
            joinPackage = new POJoinPackage(
                        new OperatorKey(scope, nig.getNextNodeId(scope)),
                        -1, pack, forEach);
            joinPackage.setChunkSize(Long.parseLong(chunkSize));
            List<PhysicalOperator> succs = plan.getSuccessors(forEach);
            if (succs!=null)
            {
View Full Code Here

        }

        public static void replaceWithPOJoinPackage(PhysicalPlan plan,
                POPackage pack, POForEach forEach, String chunkSize) throws VisitorException {
            String scope = pack.getOperatorKey().scope;
            NodeIdGenerator nig = NodeIdGenerator.getGenerator();
            POJoinPackage joinPackage;
            joinPackage = new POJoinPackage(
                        new OperatorKey(scope, nig.getNextNodeId(scope)),
                        -1, pack, forEach);
            joinPackage.setChunkSize(Long.parseLong(chunkSize));
            List<PhysicalOperator> succs = plan.getSuccessors(forEach);
            if (succs!=null)
            {
View Full Code Here

        }

        public static void replaceWithPOJoinPackage(PhysicalPlan plan, MapReduceOper mr,
                POPackage pack, POForEach forEach, String chunkSize) throws VisitorException {
            String scope = pack.getOperatorKey().scope;
            NodeIdGenerator nig = NodeIdGenerator.getGenerator();
            POJoinPackage joinPackage;
            joinPackage = new POJoinPackage(
                        new OperatorKey(scope, nig.getNextNodeId(scope)),
                        -1, pack, forEach);
            joinPackage.setChunkSize(Long.parseLong(chunkSize));
            List<PhysicalOperator> succs = plan.getSuccessors(forEach);
            if (succs!=null)
            {
View Full Code Here

        }
        try {
            // Insert a split and its corresponding SplitOutput nodes into the plan
            // between node 0 and 1 / 2.
            String scope = nodes.get(0).getOperatorKey().scope;
            NodeIdGenerator idGen = NodeIdGenerator.getGenerator();
            LOSplit splitOp = new LOSplit(mPlan, new OperatorKey(scope,
                    idGen.getNextNodeId(scope)), new ArrayList<LogicalOperator>());
            splitOp.setAlias(nodes.get(0).getAlias());
            mPlan.add(splitOp);
           
            // Find all the successors and connect appropriately with split
            // and splitoutput operators.  Keep our own copy
            // of the list, as we're changing the graph by doing these calls
            // and that will change the list of predecessors.
            List<LogicalOperator> succs =
                new ArrayList<LogicalOperator>(mPlan.getSuccessors(nodes.get(0)));
            int index = -1;
            // For two successors of nodes.get(0) here is a pictorial
            // representation of the change required:
            // BEFORE:
            // Succ1  Succ2
            //  \       /
            //  nodes.get(0)
           
            //  SHOULD BECOME:
           
            // AFTER:
            // Succ1          Succ2
            //   |              |
            // SplitOutput SplitOutput
            //      \       /
            //        Split
            //          |
            //        nodes.get(0)
           
            // Here is how this will be accomplished.
            // First (the same) Split Operator will be "inserted between" nodes.get(0)
            // and all its successors. The "insertBetween" API is used which makes sure
            // the ordering of operators in the graph is preserved. So we get the following:
            // Succ1        Succ2
            //    |          |
            //   Split     Split
            //      \      / 
            //      nodes.get(0)
           
            // Then all but the first connection between nodes.get(0) and the Split
            // Operator are removed using "disconnect" - so we get the following:
            // Succ1          Succ2
            //      \       /
            //        Split
            //          |
            //        nodes.get(0)
           
            // Now a new SplitOutputOperator is "inserted between" the Split operator
            // and the successors. So we get:
            // Succ1          Succ2
            //   |              |
            // SplitOutput SplitOutput
            //      \       /
            //        Split
            //          |
            //        nodes.get(0)
           
           
            for (LogicalOperator succ : succs) {
                mPlan.insertBetween(nodes.get(0), splitOp, succ);
            }
           
            for(int i = 1; i < succs.size(); i++) {
                mPlan.disconnect(nodes.get(0), splitOp);
            }

            for (LogicalOperator succ : succs) {
                LogicalPlan condPlan = new LogicalPlan();
                LOConst cnst = new LOConst(mPlan, new OperatorKey(scope,
                        idGen.getNextNodeId(scope)), new Boolean(true));
                cnst.setType(DataType.BOOLEAN);
                condPlan.add(cnst);
                LOSplitOutput splitOutput = new LOSplitOutput(mPlan,
                        new OperatorKey(scope, idGen.getNextNodeId(scope)), ++index, condPlan);
                splitOp.addOutput(splitOutput);
                mPlan.add(splitOutput);
                mPlan.insertBetween(splitOp, splitOutput, succ);
                splitOutput.setAlias(splitOp.getAlias());
            }
View Full Code Here

        }

        public static void replaceWithPOJoinPackage(PhysicalPlan plan,
                POPackage pack, POForEach forEach, String chunkSize) throws VisitorException {
            String scope = pack.getOperatorKey().scope;
            NodeIdGenerator nig = NodeIdGenerator.getGenerator();
            POJoinPackage joinPackage;
            joinPackage = new POJoinPackage(
                        new OperatorKey(scope, nig.getNextNodeId(scope)),
                        -1, pack, forEach);
            joinPackage.setChunkSize(Long.parseLong(chunkSize));
            List<PhysicalOperator> succs = plan.getSuccessors(forEach);
            if (succs!=null)
            {
View Full Code Here

        }

        public static void replaceWithPOJoinPackage(PhysicalPlan plan,
                POPackage pack, POForEach forEach, String chunkSize) throws VisitorException {
            String scope = pack.getOperatorKey().scope;
            NodeIdGenerator nig = NodeIdGenerator.getGenerator();
            POJoinPackage joinPackage;
            joinPackage = new POJoinPackage(
                        new OperatorKey(scope, nig.getNextNodeId(scope)),
                        -1, pack, forEach);
            joinPackage.setChunkSize(Long.parseLong(chunkSize));
            List<PhysicalOperator> succs = plan.getSuccessors(forEach);
            if (succs!=null)
            {
View Full Code Here

    private Iterator<Tuple> lastBagIter;

    public JoinPackager(Packager p, POForEach f) {
        super();
        String scope = f.getOperatorKey().getScope();
        NodeIdGenerator nig = NodeIdGenerator.getGenerator();
        forEach = new POOptimizedForEach(new OperatorKey(scope,nig.getNextNodeId(scope)));
        if (p!=null)
        {
            setKeyType(p.getKeyType());
            setNumInputs(p.getNumInputs());
            lastBagIndex = numInputs - 1;
View Full Code Here

TOP

Related Classes of org.apache.pig.impl.plan.NodeIdGenerator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.