Package org.apache.pig.impl.logicalLayer

Examples of org.apache.pig.impl.logicalLayer.LOFilter


        lpTester.typeCheckPlan(lp);
        lpTester.optimizePlan(lp);
        assertEquals("checking partition filter:",            
                    "((mrkt == 'us') and (srcid == 10))",
                    TestLoader.partFilter.toString());
        LOFilter filter = (LOFilter) lp.getLeaves().get(0);
        String actual = PColFilterExtractor.getExpression(
                (ExpressionOperator) filter.getComparisonPlan().
                getLeaves().get(0)).
                toString().toLowerCase();
        assertEquals("checking trimmed filter expression:",
                "((age >= 20) and (f3 == 15))", actual);
    }
View Full Code Here


    //// helper methods ///////
   
    private PColFilterExtractor test(LogicalPlan lp, List<String> partitionCols,
            String expPartFilterString, String expFilterString)
    throws FrontendException {
        LOFilter filter = (LOFilter)lp.getLeaves().get(0);
        PColFilterExtractor pColExtractor = new PColFilterExtractor(
                filter.getComparisonPlan(), partitionCols);
        pColExtractor.visit();
       
        if(expPartFilterString == null) {
            assertEquals("Checking partition column filter:", null,
                    pColExtractor.getPColCondition());
        } else  {
            assertEquals("Checking partition column filter:",
                    expPartFilterString.toLowerCase(),
                    pColExtractor.getPColCondition().toString().toLowerCase());  
        }
       
        if(expFilterString == null) {
            assertTrue("Check that filter can be removed:",
                    pColExtractor.isFilterRemovable());
        } else {
            String actual = PColFilterExtractor.getExpression(
                                (ExpressionOperator) filter.getComparisonPlan().
                                getLeaves().get(0)).
                                toString().toLowerCase();
            assertEquals("checking trimmed filter expression:", expFilterString,
                    actual);
        }
View Full Code Here

        return pColExtractor;
    }
   
    private void negativeTest(LogicalPlan lp, List<String> partitionCols,
            int expectedErrorCode) {
        LOFilter filter = (LOFilter)lp.getLeaves().get(0);
        PColFilterExtractor pColExtractor = new PColFilterExtractor(
                filter.getComparisonPlan(), partitionCols);
        try {
            pColExtractor.visit();
        } catch(Exception e) {
            assertEquals("Checking if exception has right error code",
                    expectedErrorCode, LogUtils.getPigException(e).getErrorCode());
View Full Code Here

    public void testQueryFilterNoSchema() throws FrontendException {
        planTester.buildPlan("a = load 'a';");
        LogicalPlan lp = planTester.buildPlan("b = filter a by $1 == '3';");
       
        LOFilter filter = (LOFilter)lp.getLeaves().get(0);
        List<RequiredFields> filterRelevantFields0 = filter.getRelevantInputs(0, 0);
        assertTrue(filterRelevantFields0.get(0).getFields().size() == 1);
        assertTrue(filterRelevantFields0.get(0).getNeedAllFields()==false);
        assertTrue(filterRelevantFields0.get(0).getNeedAllFields()==false);
        assertTrue(filterRelevantFields0.get(0).getFields().contains(new Pair<Integer, Integer>(0, 0)));
    }
View Full Code Here

    @Test
    public void testQueryFilterWithSchema() throws FrontendException {
        planTester.buildPlan("a = load 'a' as (url,hitCount);");
        LogicalPlan lp = planTester.buildPlan("b = filter a by $1 == '3';");
       
        LOFilter filter = (LOFilter)lp.getLeaves().get(0);
        List<RequiredFields> filterRelevantFields0 = filter.getRelevantInputs(0, 0);
        assertTrue(filterRelevantFields0.get(0).getFields().size() == 1);
        assertTrue(filterRelevantFields0.get(0).getNeedAllFields()==false);
        assertTrue(filterRelevantFields0.get(0).getNeedAllFields()==false);
        assertTrue(filterRelevantFields0.get(0).getFields().contains(new Pair<Integer, Integer>(0, 0)));
    }
View Full Code Here

    @Test
    public void testQueryFilterWithStarNoSchema() throws FrontendException {
        planTester.buildPlan("a = load 'a';");
        LogicalPlan lp = planTester.buildPlan("b = filter a by COUNT(*) == 3;");
       
        LOFilter filter = (LOFilter)lp.getLeaves().get(0);
        RequiredFields filterRelevantFields0 = filter.getRelevantInputs(0, 0).get(0);
        assertTrue(filterRelevantFields0.getNeedAllFields() == false);
        assertTrue(filterRelevantFields0.getNeedNoFields() == false);
        assertTrue(filterRelevantFields0.getFields().size() == 1);
        assertTrue(filterRelevantFields0.getFields().get(0).first == 0);
        assertTrue(filterRelevantFields0.getFields().get(0).second == 0);
       
        RequiredFields filterRelevantFields1 = filter.getRelevantInputs(0, 1).get(0);
        assertTrue(filterRelevantFields1.getNeedAllFields() == false);
        assertTrue(filterRelevantFields1.getNeedNoFields() == false);
        assertTrue(filterRelevantFields1.getFields().size() == 1);
        assertTrue(filterRelevantFields1.getFields().get(0).first == 0);
        assertTrue(filterRelevantFields1.getFields().get(0).second == 1);
View Full Code Here

    @Test
    public void testQueryFilterStarWithSchema() throws FrontendException {
        planTester.buildPlan("a = load 'a' as (url,hitCount);");
        LogicalPlan lp = planTester.buildPlan("b = filter a by COUNT(*) == 3;");
       
        LOFilter filter = (LOFilter)lp.getLeaves().get(0);
        RequiredFields filterRelevantFields0 = filter.getRelevantInputs(0, 0).get(0);
        assertTrue(filterRelevantFields0.getNeedAllFields() == false);
        assertTrue(filterRelevantFields0.getNeedNoFields() == false);
        assertTrue(filterRelevantFields0.getFields().size() == 1);
        assertTrue(filterRelevantFields0.getFields().get(0).first == 0);
        assertTrue(filterRelevantFields0.getFields().get(0).second == 0);
       
        RequiredFields filterRelevantFields1 = filter.getRelevantInputs(0, 1).get(0);
        assertTrue(filterRelevantFields1.getNeedAllFields() == false);
        assertTrue(filterRelevantFields1.getNeedNoFields() == false);
        assertTrue(filterRelevantFields1.getFields().size() == 1);
        assertTrue(filterRelevantFields1.getFields().get(0).first == 0);
        assertTrue(filterRelevantFields1.getFields().get(0).second == 1);
View Full Code Here

    @Test
    public void testErrNull() throws VisitorException, IOException {
        String query = "filter (load 'a') by $0 is null;";
        LogicalPlan plan = buildPlan(query);
        LOFilter filter = (LOFilter)plan.getLeaves().get(0);
        LogicalPlan innerPlan = filter.getComparisonPlan();
        innerPlan.remove(innerPlan.getRoots().get(0));
        try {
            buildPhysicalPlan(plan);
            fail("Expected error.");
        } catch(VisitorException ve) {
View Full Code Here

    }

    @Override
    public boolean check(List<LogicalOperator> nodes) throws OptimizerException {
        try {
            LOFilter filter = (LOFilter) getOperator(nodes);
            List<LogicalOperator> predecessors = (mPlan.getPredecessors(filter) == null ? null
                    : new ArrayList<LogicalOperator>(mPlan
                            .getPredecessors(filter)));

            // if there are no predecessors return false
            if (predecessors == null) {
                return false;
            }

            // if the filter has no predecessors or more than one predecessor
            // return false
            if (predecessors.size() == 0 || predecessors.size() > 1) {
                return false;
            }
               
            LogicalOperator predecessor = predecessors.get(0);

            // if the predecessor is one of LOLoad/LOStore/LOStream/LOLimit
            // return false
            if (predecessor instanceof LOLoad || predecessor instanceof LOStore
                    || predecessor instanceof LOStream
                    || predecessor instanceof LOLimit) {
                return false;
            }
           
            // TODO
            // for now filters cannot be combined
            // remove this check when filters can be combined
            if (predecessor instanceof LOFilter)
                return false;

            // TODO
            // same rule as filters
            if (predecessor instanceof LOSplitOutput) {
                return false;
            }
            if (predecessor instanceof LOSplit) {
                return false;
            }

            UDFFinder udfFinder = new UDFFinder(filter.getComparisonPlan());
            udfFinder.visit();

            // if the filter's inner plan contains any UDF then return false
            if (udfFinder.foundAnyUDF()) {
                return false;
            }

            CastFinder castFinder = new CastFinder(filter.getComparisonPlan());
            castFinder.visit();

            // if the filter's inner plan contains any casts then return false
            if (castFinder.foundAnyCast()) {
                return false;
            }

            List<RequiredFields> filterRequiredFields = filter
                    .getRequiredFields();
            if (filterRequiredFields == null) {
                return false;
            }
            RequiredFields requiredField = filterRequiredFields.get(0);
View Full Code Here

    @Override
    public void transform(List<LogicalOperator> nodes)
            throws OptimizerException {
        try {
            LOFilter filter = (LOFilter) getOperator(nodes);
            LogicalOperator predecessor = mPlan.getPredecessors(filter).get(0);
            if (mSwap) {
                mPlan.swap(predecessor, filter);
            } else if (mPushBefore) {
                if (mPushBeforeInput == -1) {
View Full Code Here

TOP

Related Classes of org.apache.pig.impl.logicalLayer.LOFilter

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.