Examples of LOSplit


Examples of org.apache.pig.impl.logicalLayer.LOSplit

        try {
            // Insert a split and its corresponding SplitOutput nodes into the plan
            // between node 0 and 1 / 2.
            String scope = nodes.get(0).getOperatorKey().scope;
            NodeIdGenerator idGen = NodeIdGenerator.getGenerator();
            LOSplit splitOp = new LOSplit(mPlan, new OperatorKey(scope,
                    idGen.getNextNodeId(scope)), new ArrayList<LogicalOperator>());
            splitOp.setAlias(nodes.get(0).getAlias());
            mPlan.add(splitOp);
           
            // Find all the successors and connect appropriately with split
            // and splitoutput operators.  Keep our own copy
            // of the list, as we're changing the graph by doing these calls
            // and that will change the list of predecessors.
            List<LogicalOperator> succs =
                new ArrayList<LogicalOperator>(mPlan.getSuccessors(nodes.get(0)));
            int index = -1;
            // For two successors of nodes.get(0) here is a pictorial
            // representation of the change required:
            // BEFORE:
            // Succ1  Succ2
            //  \       /
            //  nodes.get(0)
           
            //  SHOULD BECOME:
           
            // AFTER:
            // Succ1          Succ2
            //   |              |
            // SplitOutput SplitOutput
            //      \       /
            //        Split
            //          |
            //        nodes.get(0)
           
            // Here is how this will be accomplished.
            // First (the same) Split Operator will be "inserted between" nodes.get(0)
            // and all its successors. The "insertBetween" API is used which makes sure
            // the ordering of operators in the graph is preserved. So we get the following:
            // Succ1        Succ2
            //    |          |
            //   Split     Split
            //      \      / 
            //      nodes.get(0)
           
            // Then all but the first connection between nodes.get(0) and the Split
            // Operator are removed using "disconnect" - so we get the following:
            // Succ1          Succ2
            //      \       /
            //        Split
            //          |
            //        nodes.get(0)
           
            // Now a new SplitOutputOperator is "inserted between" the Split operator
            // and the successors. So we get:
            // Succ1          Succ2
            //   |              |
            // SplitOutput SplitOutput
            //      \       /
            //        Split
            //          |
            //        nodes.get(0)
           
           
            for (LogicalOperator succ : succs) {
                mPlan.insertBetween(nodes.get(0), splitOp, succ);
            }
           
            for(int i = 1; i < succs.size(); i++) {
                mPlan.disconnect(nodes.get(0), splitOp);
            }

            for (LogicalOperator succ : succs) {
                LogicalPlan condPlan = new LogicalPlan();
                LOConst cnst = new LOConst(mPlan, new OperatorKey(scope,
                        idGen.getNextNodeId(scope)), Boolean.valueOf(true));
                cnst.setType(DataType.BOOLEAN);
                condPlan.add(cnst);
                LOSplitOutput splitOutput = new LOSplitOutput(mPlan,
                        new OperatorKey(scope, idGen.getNextNodeId(scope)), ++index, condPlan);
                splitOp.addOutput(splitOutput);
                mPlan.add(splitOutput);
                mPlan.insertBetween(splitOp, splitOutput, succ);
                splitOutput.setAlias(splitOp.getAlias());
            }
           
        } catch (Exception e) {
            int errCode = 2047;
            String msg = "Internal error. Unable to introduce split operators.";
View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOSplit

        }
        else {
            LogicalOperator lg = project.getExpression();
            int input;
            if (oldAttachedRelationalOp instanceof LOSplitOutput) {
                LOSplit split = (LOSplit)outerPlan.getPredecessors(oldAttachedRelationalOp).get(0);
                input = outerPlan.getPredecessors(split).indexOf(lg);
            }
            else {
                input = outerPlan.getPredecessors(oldAttachedRelationalOp).indexOf(lg);
            }
View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOSplit

    @Test
    public void testQuerySplitNoSchema() throws FrontendException {
        planTester.buildPlan("a = load 'a';");
        LogicalPlan lp = planTester.buildPlan("split a into b if $0 == '3', c if $1 == '3';");
       
        LOSplit split = (LOSplit)lp.getSuccessors(lp.getRoots().get(0)).get(0);
        RequiredFields splitRelevantFields = split.getRelevantInputs(0, 0).get(0);
        assertTrue(splitRelevantFields.needAllFields() == false);
        assertTrue(splitRelevantFields.needNoFields() == false);
        assertTrue(splitRelevantFields.getFields().size() == 1);

        LOSplitOutput splitb = (LOSplitOutput)lp.getSuccessors(split).get(0);
View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOSplit

    @Test
    public void testQuerySplitWithSchema() throws FrontendException {
        planTester.buildPlan("a = load 'a' as (url, hitCount);");
        LogicalPlan lp = planTester.buildPlan("split a into b if url == '3', c if hitCount == '3';");
       
        LOSplit split = (LOSplit)lp.getSuccessors(lp.getRoots().get(0)).get(0);
        RequiredFields splitRelevantFields0 = split.getRelevantInputs(0, 0).get(0);
        assertTrue(splitRelevantFields0.needAllFields() == false);
        assertTrue(splitRelevantFields0.needNoFields() == false);
        assertTrue(splitRelevantFields0.getFields().size() == 1);
        assertTrue(splitRelevantFields0.getFields().contains(new Pair<Integer, Integer>(0, 0)));
       
        RequiredFields splitRelevantFields1 = split.getRelevantInputs(0, 1).get(0);
        assertTrue(splitRelevantFields1.needAllFields() == false);
        assertTrue(splitRelevantFields1.needNoFields() == false);
        assertTrue(splitRelevantFields1.getFields().size() == 1);
        assertTrue(splitRelevantFields1.getFields().contains(new Pair<Integer, Integer>(0, 1)));

        assertTrue(split.getRelevantInputs(0, 2) == null);

        LOSplitOutput splitb = (LOSplitOutput)lp.getSuccessors(split).get(0);
        RequiredFields splitbRelevantFields = splitb.getRelevantInputs(0, 1).get(0);
        assertTrue(splitbRelevantFields.getNeedAllFields() == false);
        assertTrue(splitbRelevantFields.getNeedNoFields() == false);
View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOSplit

    @Test
    public void testQuerySplitWithStarSchema() throws FrontendException {
        planTester.buildPlan("a = load 'a' as (url, hitCount);");
        LogicalPlan lp = planTester.buildPlan("split a into b if url == '3', c if COUNT(*) == '3';");
       
        LOSplit split = (LOSplit)lp.getSuccessors(lp.getRoots().get(0)).get(0);
        RequiredFields splitRelevantFields0 = split.getRelevantInputs(0, 0).get(0);
        assertTrue(splitRelevantFields0.needAllFields() == false);
        assertTrue(splitRelevantFields0.needNoFields() == false);
        assertTrue(splitRelevantFields0.getFields().size() == 1);
        assertTrue(splitRelevantFields0.getFields().contains(new Pair<Integer, Integer>(0, 0)));
       
        RequiredFields splitRelevantFields1 = split.getRelevantInputs(0, 1).get(0);
        assertTrue(splitRelevantFields1.needAllFields() == false);
        assertTrue(splitRelevantFields1.needNoFields() == false);
        assertTrue(splitRelevantFields1.getFields().size() == 1);
        assertTrue(splitRelevantFields1.getFields().contains(new Pair<Integer, Integer>(0, 1)));
       
        assertTrue(split.getRelevantInputs(0, 2) == null);

        LOSplitOutput splitb = (LOSplitOutput)lp.getSuccessors(split).get(0);
        RequiredFields splitbRelevantFields0 = splitb.getRelevantInputs(0, 0).get(0);
        assertTrue(splitbRelevantFields0.getNeedAllFields() == false);
        assertTrue(splitbRelevantFields0.getNeedNoFields() == false);
View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOSplit

        try {
            // Insert a split and its corresponding SplitOutput nodes into the plan
            // between node 0 and 1 / 2.
            String scope = nodes.get(0).getOperatorKey().scope;
            NodeIdGenerator idGen = NodeIdGenerator.getGenerator();
            LOSplit splitOp = new LOSplit(mPlan, new OperatorKey(scope,
                    idGen.getNextNodeId(scope)), new ArrayList<LogicalOperator>());
            splitOp.setAlias(nodes.get(0).getAlias());
            mPlan.add(splitOp);
           
            // Find all the successors and connect appropriately with split
            // and splitoutput operators.  Keep our own copy
            // of the list, as we're changing the graph by doing these calls
            // and that will change the list of predecessors.
            List<LogicalOperator> succs =
                new ArrayList<LogicalOperator>(mPlan.getSuccessors(nodes.get(0)));
            int index = -1;
            // For two successors of nodes.get(0) here is a pictorial
            // representation of the change required:
            // BEFORE:
            // Succ1  Succ2
            //  \       /
            //  nodes.get(0)
           
            //  SHOULD BECOME:
           
            // AFTER:
            // Succ1          Succ2
            //   |              |
            // SplitOutput SplitOutput
            //      \       /
            //        Split
            //          |
            //        nodes.get(0)
           
            // Here is how this will be accomplished.
            // First (the same) Split Operator will be "inserted between" nodes.get(0)
            // and all its successors. The "insertBetween" API is used which makes sure
            // the ordering of operators in the graph is preserved. So we get the following:
            // Succ1        Succ2
            //    |          |
            //   Split     Split
            //      \      / 
            //      nodes.get(0)
           
            // Then all but the first connection between nodes.get(0) and the Split
            // Operator are removed using "disconnect" - so we get the following:
            // Succ1          Succ2
            //      \       /
            //        Split
            //          |
            //        nodes.get(0)
           
            // Now a new SplitOutputOperator is "inserted between" the Split operator
            // and the successors. So we get:
            // Succ1          Succ2
            //   |              |
            // SplitOutput SplitOutput
            //      \       /
            //        Split
            //          |
            //        nodes.get(0)
           
           
            for (LogicalOperator succ : succs) {
                mPlan.insertBetween(nodes.get(0), splitOp, succ);
            }
           
            for(int i = 1; i < succs.size(); i++) {
                mPlan.disconnect(nodes.get(0), splitOp);
            }

            for (LogicalOperator succ : succs) {
                LogicalPlan condPlan = new LogicalPlan();
                LOConst cnst = new LOConst(mPlan, new OperatorKey(scope,
                        idGen.getNextNodeId(scope)), new Boolean(true));
                cnst.setType(DataType.BOOLEAN);
                condPlan.add(cnst);
                LOSplitOutput splitOutput = new LOSplitOutput(mPlan,
                        new OperatorKey(scope, idGen.getNextNodeId(scope)), ++index, condPlan);
                splitOp.addOutput(splitOutput);
                mPlan.add(splitOutput);
                mPlan.insertBetween(splitOp, splitOutput, succ);
                splitOutput.setAlias(splitOp.getAlias());
            }
           
        } catch (Exception e) {
            int errCode = 2047;
            String msg = "Internal error. Unable to introduce split operators.";
View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOSplit

    @Test
    public void testQuerySplitNoSchema() {
        planTester.buildPlan("a = load 'a';");
        LogicalPlan lp = planTester.buildPlan("split a into b if $0 == '3', c if $1 == '3';");
       
        LOSplit split = (LOSplit)lp.getSuccessors(lp.getRoots().get(0)).get(0);
        RequiredFields splitRelevantFields = split.getRelevantInputs(0, 0).get(0);
        assertTrue(splitRelevantFields.needAllFields() == false);
        assertTrue(splitRelevantFields.needNoFields() == true);
        assertTrue(splitRelevantFields.getFields() == null);

        LOSplitOutput splitb = (LOSplitOutput)lp.getSuccessors(split).get(0);
View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOSplit

    @Test
    public void testQuerySplitWithSchema() {
        planTester.buildPlan("a = load 'a' as (url, hitCount);");
        LogicalPlan lp = planTester.buildPlan("split a into b if url == '3', c if hitCount == '3';");
       
        LOSplit split = (LOSplit)lp.getSuccessors(lp.getRoots().get(0)).get(0);
        RequiredFields splitRelevantFields = split.getRelevantInputs(0, 0).get(0);
        assertTrue(splitRelevantFields.needAllFields() == false);
        assertTrue(splitRelevantFields.needNoFields() == true);
        assertTrue(splitRelevantFields.getFields() == null);

        LOSplitOutput splitb = (LOSplitOutput)lp.getSuccessors(split).get(0);
View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOSplit

    @Test
    public void testQuerySplitWithStarSchema() {
        planTester.buildPlan("a = load 'a' as (url, hitCount);");
        LogicalPlan lp = planTester.buildPlan("split a into b if url == '3', c if COUNT(*) == '3';");
       
        LOSplit split = (LOSplit)lp.getSuccessors(lp.getRoots().get(0)).get(0);
        RequiredFields splitRelevantFields = split.getRelevantInputs(0, 0).get(0);
        assertTrue(splitRelevantFields.needAllFields() == false);
        assertTrue(splitRelevantFields.needNoFields() == true);
        assertTrue(splitRelevantFields.getFields() == null);

        LOSplitOutput splitb = (LOSplitOutput)lp.getSuccessors(split).get(0);
View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOSplit

        try {
            // Insert a split and its corresponding SplitOutput nodes into the plan
            // between node 0 and 1 / 2.
            String scope = nodes.get(0).getOperatorKey().scope;
            NodeIdGenerator idGen = NodeIdGenerator.getGenerator();
            LOSplit splitOp = new LOSplit(mPlan, new OperatorKey(scope,
                    idGen.getNextNodeId(scope)), new ArrayList<LogicalOperator>());
            splitOp.setAlias(nodes.get(0).getAlias());
            mPlan.add(splitOp);
           
            // Find all the successors and connect appropriately with split
            // and splitoutput operators.  Keep our own copy
            // of the list, as we're changing the graph by doing these calls
            // and that will change the list of predecessors.
            List<LogicalOperator> succs =
                new ArrayList<LogicalOperator>(mPlan.getSuccessors(nodes.get(0)));
            int index = -1;
            // For two successors of nodes.get(0) here is a pictorial
            // representation of the change required:
            // BEFORE:
            // Succ1  Succ2
            //  \       /
            //  nodes.get(0)
           
            //  SHOULD BECOME:
           
            // AFTER:
            // Succ1          Succ2
            //   |              |
            // SplitOutput SplitOutput
            //      \       /
            //        Split
            //          |
            //        nodes.get(0)
           
            // Here is how this will be accomplished.
            // First (the same) Split Operator will be "inserted between" nodes.get(0)
            // and all its successors. The "insertBetween" API is used which makes sure
            // the ordering of operators in the graph is preserved. So we get the following:
            // Succ1        Succ2
            //    |          |
            //   Split     Split
            //      \      / 
            //      nodes.get(0)
           
            // Then all but the first connection between nodes.get(0) and the Split
            // Operator are removed using "disconnect" - so we get the following:
            // Succ1          Succ2
            //      \       /
            //        Split
            //          |
            //        nodes.get(0)
           
            // Now a new SplitOutputOperator is "inserted between" the Split operator
            // and the successors. So we get:
            // Succ1          Succ2
            //   |              |
            // SplitOutput SplitOutput
            //      \       /
            //        Split
            //          |
            //        nodes.get(0)
           
           
            for (LogicalOperator succ : succs) {
                mPlan.insertBetween(nodes.get(0), splitOp, succ);
            }
           
            for(int i = 1; i < succs.size(); i++) {
                mPlan.disconnect(nodes.get(0), splitOp);
            }

            for (LogicalOperator succ : succs) {
                LogicalPlan condPlan = new LogicalPlan();
                LOConst cnst = new LOConst(mPlan, new OperatorKey(scope,
                        idGen.getNextNodeId(scope)), new Boolean(true));
                cnst.setType(DataType.BOOLEAN);
                condPlan.add(cnst);
                LOSplitOutput splitOutput = new LOSplitOutput(mPlan,
                        new OperatorKey(scope, idGen.getNextNodeId(scope)), ++index, condPlan);
                splitOp.addOutput(splitOutput);
                mPlan.add(splitOutput);
                mPlan.insertBetween(splitOp, splitOutput, succ);
                splitOutput.setAlias(splitOp.getAlias());
                // Patch up the contained plans of succ
                fixUpContainedPlans(nodes.get(0), splitOutput, succ, null);
            }
           
        } catch (Exception e) {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.