Examples of PColFilterExtractor

org.apache.pig.impl.logicalLayer.PColFilterExtractor
This Visitor works on the filter condition of a LOFilter which immediately follows a LOLoad that interacts with a metadata system (currently OWL) to read table data. The visitor looks for conditions on partition columns in the filter condition and extracts those conditions out of the filter condition. The condition on partition cols will be used to prune partitions of the table.
org.apache.pig.newplan.PColFilterExtractor
This Visitor works on the filter condition of a LOFilter which immediately follows a LOLoad that interacts with a metadata system (currently OWL) to read table data. The visitor looks for conditions on partition columns in the filter condition and extracts those conditions out of the filter condition. The condition on partition cols will be used to prune partitions of the table.

Examples of org.apache.pig.impl.logicalLayer.PColFilterExtractor

    @Override
    public void transform(List<LogicalOperator> nodes)
            throws OptimizerException {
        try {
            setupColNameMaps();
            PColFilterExtractor pColFilterFinder = new PColFilterExtractor(
                    loFilter.getComparisonPlan(), getMappedKeys(partitionKeys));
            pColFilterFinder.visit();
            Expression partitionFilter = pColFilterFinder.getPColCondition();
            if(partitionFilter != null) {
                // the column names in the filter may be the ones provided by
                // the user in the schema in the load statement - we may need
                // to replace them with partition column names as given by
                // LoadFunc.getSchema()
                updateMappedColNames(partitionFilter);
                loadMetadata.setPartitionFilter(partitionFilter);
                if(pColFilterFinder.isFilterRemovable()) {
                    // remove this filter from the plan                  
                    mPlan.removeAndReconnect(loFilter);
                }
            }
        } catch (Exception e) {

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.PColFilterExtractor

    
    private PColFilterExtractor test(LogicalPlan lp, List<String> partitionCols, 
            String expPartFilterString, String expFilterString) 
    throws FrontendException {
        LOFilter filter = (LOFilter)lp.getLeaves().get(0);
        PColFilterExtractor pColExtractor = new PColFilterExtractor(
                filter.getComparisonPlan(), partitionCols);
        pColExtractor.visit();
        
        if(expPartFilterString == null) {
            assertEquals("Checking partition column filter:", null, 
                    pColExtractor.getPColCondition());
        } else  {
            assertEquals("Checking partition column filter:", 
                    expPartFilterString.toLowerCase(), 
                    pColExtractor.getPColCondition().toString().toLowerCase());   
        }
        
        if(expFilterString == null) {
            assertTrue("Check that filter can be removed:", 
                    pColExtractor.isFilterRemovable());
        } else {
            String actual = PColFilterExtractor.getExpression(
                                (ExpressionOperator) filter.getComparisonPlan().
                                getLeaves().get(0)).
                                toString().toLowerCase();

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.PColFilterExtractor

    }
    
    private void negativeTest(LogicalPlan lp, List<String> partitionCols,
            int expectedErrorCode) {
        LOFilter filter = (LOFilter)lp.getLeaves().get(0);
        PColFilterExtractor pColExtractor = new PColFilterExtractor(
                filter.getComparisonPlan(), partitionCols);
        try {
            pColExtractor.visit();
        } catch(Exception e) {
            assertEquals("Checking if exception has right error code", 
                    expectedErrorCode, LogUtils.getPigException(e).getErrorCode());
            return;
        }

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.PColFilterExtractor

    
    private PColFilterExtractor test(LogicalPlan lp, List<String> partitionCols, 
            String expPartFilterString, String expFilterString) 
    throws FrontendException {
        LOFilter filter = (LOFilter)lp.getLeaves().get(0);
        PColFilterExtractor pColExtractor = new PColFilterExtractor(
                filter.getComparisonPlan(), partitionCols);
        pColExtractor.visit();
        
        if(expPartFilterString == null) {
            assertEquals("Checking partition column filter:", null, 
                    pColExtractor.getPColCondition());
        } else  {
            assertEquals("Checking partition column filter:", 
                    expPartFilterString.toLowerCase(), 
                    pColExtractor.getPColCondition().toString().toLowerCase());   
        }
        
        if(expFilterString == null) {
            assertTrue("Check that filter can be removed:", 
                    pColExtractor.isFilterRemovable());
        } else {
            String actual = PColFilterExtractor.getExpression(
                                (ExpressionOperator) filter.getComparisonPlan().
                                getLeaves().get(0)).
                                toString().toLowerCase();

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.PColFilterExtractor

    }
    
    private void negativeTest(LogicalPlan lp, List<String> partitionCols,
            int expectedErrorCode) {
        LOFilter filter = (LOFilter)lp.getLeaves().get(0);
        PColFilterExtractor pColExtractor = new PColFilterExtractor(
                filter.getComparisonPlan(), partitionCols);
        try {
            pColExtractor.visit();
        } catch(Exception e) {
            assertEquals("Checking if exception has right error code", 
                    expectedErrorCode, LogUtils.getPigException(e).getErrorCode());
            return;
        }

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.PColFilterExtractor

    @Override
    public void transform(List<LogicalOperator> nodes)
            throws OptimizerException {
        try {
            setupColNameMaps();
            PColFilterExtractor pColFilterFinder = new PColFilterExtractor(
                    loFilter.getComparisonPlan(), getMappedKeys(partitionKeys));
            pColFilterFinder.visit();
            Expression partitionFilter = pColFilterFinder.getPColCondition();
            if(partitionFilter != null) {
                // the column names in the filter may be the ones provided by
                // the user in the schema in the load statement - we may need
                // to replace them with partition column names as given by
                // LoadFunc.getSchema()
                updateMappedColNames(partitionFilter);
                loadMetadata.setPartitionFilter(partitionFilter);
                if(pColFilterFinder.isFilterRemovable()) {
                    // remove this filter from the plan                  
                    mPlan.removeAndReconnect(loFilter);
                }
            }
        } catch (Exception e) {

View Full Code Here

Examples of org.apache.pig.newplan.PColFilterExtractor

        @Override
        public void transform(OperatorPlan matched) throws FrontendException {
          subPlan = new OperatorSubPlan( currentPlan );


          setupColNameMaps();
          PColFilterExtractor pColFilterFinder = new PColFilterExtractor(
              loFilter.getFilterPlan(), getMappedKeys( partitionKeys ) );
          pColFilterFinder.visit();
          Expression partitionFilter = pColFilterFinder.getPColCondition();
          if(partitionFilter != null) {
            // the column names in the filter may be the ones provided by
            // the user in the schema in the load statement - we may need
            // to replace them with partition column names as given by
            // LoadFunc.getSchema()
            updateMappedColNames(partitionFilter);
            try {
          loadMetadata.setPartitionFilter(partitionFilter);
        } catch (IOException e) {
          throw new FrontendException( e );
        }
            if(pColFilterFinder.isFilterRemovable()) {  
              currentPlan.removeAndReconnect( loFilter );
            }
          }
        }

View Full Code Here

Examples of org.apache.pig.newplan.PColFilterExtractor

    private PColFilterExtractor test(org.apache.pig.impl.logicalLayer.LogicalPlan lp, List<String> partitionCols, 
            String expPartFilterString, String expFilterString) 
    throws IOException {
      LogicalPlan newLogicalPlan = migratePlan( lp );
        LOFilter filter = (LOFilter)newLogicalPlan.getSinks().get(0);
        PColFilterExtractor pColExtractor = new PColFilterExtractor(
                filter.getFilterPlan(), partitionCols);
        pColExtractor.visit();
        
        if(expPartFilterString == null) {
           Assert.assertEquals("Checking partition column filter:", null, 
                    pColExtractor.getPColCondition());
        } else  {
           Assert.assertEquals("Checking partition column filter:", 
                    expPartFilterString.toLowerCase(), 
                    pColExtractor.getPColCondition().toString().toLowerCase());   
        }
        
        if(expFilterString == null) {
           Assert.assertTrue("Check that filter can be removed:", 
                    pColExtractor.isFilterRemovable());
        } else {
            String actual = PColFilterExtractor.getExpression(
                                (LogicalExpression)filter.getFilterPlan().getSources().get(0)).
                                toString().toLowerCase();
            Assert.assertEquals("checking trimmed filter expression:", expFilterString,

View Full Code Here

Examples of org.apache.pig.newplan.PColFilterExtractor

    
    private void negativeTest(org.apache.pig.impl.logicalLayer.LogicalPlan lp, List<String> partitionCols,
            int expectedErrorCode) throws VisitorException {
      LogicalPlan newLogicalPlan = migratePlan( lp );
        LOFilter filter = (LOFilter)newLogicalPlan.getSinks().get(0);
        PColFilterExtractor pColExtractor = new PColFilterExtractor(
                filter.getFilterPlan(), partitionCols);
        try {
            pColExtractor.visit();
        } catch(Exception e) {
           Assert.assertEquals("Checking if exception has right error code", 
                    expectedErrorCode, LogUtils.getPigException(e).getErrorCode());
            return;
        }

View Full Code Here

Examples of org.apache.pig.newplan.PColFilterExtractor

                "((mrkt == 'us') and (srcid == 10))",
                TestLoader.partFilter.toString());
        Operator op = newLogicalPlan.getSinks().get(0);
        LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
        
        PColFilterExtractor extractor = new PColFilterExtractor(filter.getFilterPlan(), new ArrayList<String>());
        
        String actual = extractor.getExpression(
                (LogicalExpression)filter.getFilterPlan().getSources().get(0)).
                toString().toLowerCase();
        Assert.assertEquals("checking trimmed filter expression:", 
                "((f5 >= 20) and (f3 == 15))", actual);
    }

View Full Code Here

0 1 2

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.