Examples of LOLoad

Examples of org.apache.pig.experimental.logical.relational.LOLoad

    public void testRelationalEqualityOneNullOneNotNullSchema() {
        LogicalPlan lp = new LogicalPlan();
        LogicalSchema aschema1 = new LogicalSchema();
        aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        LOLoad load1 = new LOLoad(new FileSpec("/abc",
            new FuncSpec("foo", new String[] {"x", "y"})), aschema1, lp);
        lp.add(load1);
        
        // Test that one with schema and one without breaks equality
        LOLoad load9 = new LOLoad(new FileSpec("/abc",
            new FuncSpec("foo", new String[] {"x", "z"})), null, lp);
        lp.add(load9);
        
        assertFalse(load1.isEqual(load9));
    }

View Full Code Here

Examples of org.apache.pig.experimental.logical.relational.LOLoad

    public void testJoinDifferentJoinTypes() throws IOException {
       LogicalPlan lp = new LogicalPlan();
       LogicalSchema jaschema1 = new LogicalSchema();
       jaschema1.addField(new LogicalSchema.LogicalFieldSchema(
           "x", null, DataType.INTEGER));
       LOLoad A1 = new LOLoad(new FileSpec("/abc",
           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema1, lp);
       lp.add(A1);
        
        // B = load
        LogicalSchema jbschema1 = new LogicalSchema();
        jbschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LOLoad B1 = new LOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema1, lp);
        lp.add(B1);
        
        // C = join
        LogicalSchema jcschema1 = new LogicalSchema();
        jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LogicalExpressionPlan aprojplan1 = new LogicalExpressionPlan();
        new ProjectExpression(aprojplan1, DataType.INTEGER, 0, 0);
        LogicalExpressionPlan bprojplan1 = new LogicalExpressionPlan();
        new ProjectExpression(bprojplan1, DataType.INTEGER, 1, 0);
        MultiMap<Integer, LogicalExpressionPlan> mm1 = 
            new MultiMap<Integer, LogicalExpressionPlan>();
        mm1.put(0, aprojplan1);
        mm1.put(1, bprojplan1);
        LOJoin C1 = new LOJoin(lp, mm1, JOINTYPE.HASH, new boolean[] {true, true});
        C1.neverUseForRealSetSchema(jcschema1);
        lp.add(C1);
        lp.connect(A1, C1);
        lp.connect(B1, C1);
        
        // A = load
        LogicalSchema jaschema2 = new LogicalSchema();
        jaschema2.addField(new LogicalSchema.LogicalFieldSchema(
           "x", null, DataType.INTEGER));
        LOLoad A2 = new LOLoad(new FileSpec("/abc",
           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema2, lp);
        lp.add(A2);
        
        // B = load
        LogicalSchema jbschema2 = new LogicalSchema();
        jbschema2.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LOLoad B2 = new LOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema2, lp);
        lp.add(B2);
        
        // C = join
        LogicalSchema jcschema2 = new LogicalSchema();

View Full Code Here

Examples of org.apache.pig.experimental.logical.relational.LOLoad

    public void testJoinDifferentInner() throws IOException {
        LogicalPlan lp = new LogicalPlan();
               LogicalSchema jaschema1 = new LogicalSchema();
       jaschema1.addField(new LogicalSchema.LogicalFieldSchema(
           "x", null, DataType.INTEGER));
       LOLoad A1 = new LOLoad(new FileSpec("/abc",
           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema1, lp);
       lp.add(A1);
        
        // B = load
        LogicalSchema jbschema1 = new LogicalSchema();
        jbschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LOLoad B1 = new LOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema1, lp);
        lp.add(B1);
        
        // C = join
        LogicalSchema jcschema1 = new LogicalSchema();
        jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LogicalExpressionPlan aprojplan1 = new LogicalExpressionPlan();
        new ProjectExpression(aprojplan1, DataType.INTEGER, 0, 0);
        LogicalExpressionPlan bprojplan1 = new LogicalExpressionPlan();
        new ProjectExpression(bprojplan1, DataType.INTEGER, 1, 0);
        MultiMap<Integer, LogicalExpressionPlan> mm1 = 
            new MultiMap<Integer, LogicalExpressionPlan>();
        mm1.put(0, aprojplan1);
        mm1.put(1, bprojplan1);
        LOJoin C1 = new LOJoin(lp, mm1, JOINTYPE.HASH, new boolean[] {true, true});
        C1.neverUseForRealSetSchema(jcschema1);
        lp.add(C1);
        lp.connect(A1, C1);
        lp.connect(B1, C1);
        
 
        // Test different inner status
        // A = load
        LogicalSchema jaschema3 = new LogicalSchema();
        jaschema3.addField(new LogicalSchema.LogicalFieldSchema(
           "x", null, DataType.INTEGER));
        LOLoad A3 = new LOLoad(new FileSpec("/abc",
           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema3, lp);
        lp.add(A3);
        
        // B = load
        LogicalSchema jbschema3 = new LogicalSchema();
        jbschema3.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LOLoad B3 = new LOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema3, lp);
        lp.add(B3);
        
        // C = join
        LogicalSchema jcschema3 = new LogicalSchema();

View Full Code Here

Examples of org.apache.pig.experimental.logical.relational.LOLoad

    public void testJoinDifferentNumInputs() throws IOException {
        LogicalPlan lp = new LogicalPlan();
               LogicalSchema jaschema1 = new LogicalSchema();
       jaschema1.addField(new LogicalSchema.LogicalFieldSchema(
           "x", null, DataType.INTEGER));
       LOLoad A1 = new LOLoad(new FileSpec("/abc",
           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema1, lp);
       lp.add(A1);
        
        // B = load
        LogicalSchema jbschema1 = new LogicalSchema();
        jbschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LOLoad B1 = new LOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema1, lp);
        lp.add(B1);
        
        // C = join
        LogicalSchema jcschema1 = new LogicalSchema();
        jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LogicalExpressionPlan aprojplan1 = new LogicalExpressionPlan();
        new ProjectExpression(aprojplan1, DataType.INTEGER, 0, 0);
        LogicalExpressionPlan bprojplan1 = new LogicalExpressionPlan();
        new ProjectExpression(bprojplan1, DataType.INTEGER, 1, 0);
        MultiMap<Integer, LogicalExpressionPlan> mm1 = 
            new MultiMap<Integer, LogicalExpressionPlan>();
        mm1.put(0, aprojplan1);
        mm1.put(1, bprojplan1);
        LOJoin C1 = new LOJoin(lp, mm1, JOINTYPE.HASH, new boolean[] {true, true});
        C1.neverUseForRealSetSchema(jcschema1);
        lp.add(C1);
        lp.connect(A1, C1);
        lp.connect(B1, C1);
 
        // A = load
        LogicalSchema jaschema5 = new LogicalSchema();
        jaschema5.addField(new LogicalSchema.LogicalFieldSchema(
           "x", null, DataType.INTEGER));
        LOLoad A5 = new LOLoad(new FileSpec("/abc",
           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema5, lp);
        lp.add(A5);
        
        // B = load
        LogicalSchema jbschema5 = new LogicalSchema();
        jbschema5.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LOLoad B5 = new LOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema5, lp);
        lp.add(B5);
        
        // Beta = load
        LogicalSchema jbetaschema5 = new LogicalSchema();
        jbetaschema5.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LOLoad Beta5 = new LOLoad(new FileSpec("/ghi",
            new FuncSpec("PigStorage", "\t")), jbetaschema5, lp);
        lp.add(Beta5);
        
        // C = join
        LogicalSchema jcschema5 = new LogicalSchema();

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOLoad

    @Test
    public void testNonDfsLocation() throws Exception {
        LogicalPlanTester lpt = new LogicalPlanTester();
        String nonDfsUrl = "har://hdfs-namenode/user/foo/";
        LogicalPlan lp = lpt.buildPlan("a = load '" + nonDfsUrl + "';");
        LOLoad load = (LOLoad) lp.getRoots().get(0);
        nonDfsUrl = nonDfsUrl.replaceFirst("/$", "");
        Assert.assertEquals(nonDfsUrl, load.getInputFile().getFileName());
    }

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOLoad

                                           fileNameMap);
            Assert.assertTrue(lp.size()>0);
            LogicalOperator op = lp.getRoots().get(0);
            
            Assert.assertTrue(op instanceof LOLoad);
            LOLoad load = (LOLoad)op;
    
            String p = load.getInputFile().getFileName();
            System.err.println("DEBUG: p:" + p + " expected:" + expected +", exectype:" + pc.getExecType());
            if(noConversionExpected) {
                Assert.assertEquals(p, expected);
            } else  {
                if (pc.getExecType() == ExecType.MAPREDUCE) {

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOLoad

        LogicalPlan plan = new LogicalPlan() ;
        FileSpec filespec1 =
            new FileSpec(inputFile, new FuncSpec("org.apache.pig.builtin.PigStorage")) ;
        FileSpec filespec2 =
            new FileSpec(outputFile, new FuncSpec("org.apache.pig.builtin.PigStorage"));
        LOLoad load = new LOLoad(plan, genNewOperatorKeyId(), filespec1, 
                ConfigurationUtil.toConfiguration(dfs.getConfiguration())) ;       
        LOStore store = new LOStore(plan, genNewOperatorKeyId(), filespec2, "new") ;
        
        plan.add(load) ;
        plan.add(store) ;

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOLoad

        LogicalPlan plan = new LogicalPlan() ;
        FileSpec filespec1 =
            new FileSpec(inputFile, new FuncSpec("org.apache.pig.builtin.PigStorage")) ;
        FileSpec filespec2 =
            new FileSpec(outputFile, new FuncSpec("org.apache.pig.builtin.PigStorage"));
        LOLoad load = new LOLoad(plan, genNewOperatorKeyId(), filespec1, 
                ConfigurationUtil.toConfiguration(dfs.getConfiguration())) ;       
        LOStore store = new LOStore(plan, genNewOperatorKeyId(), filespec2, "new") ;
        
        plan.add(load) ;
        plan.add(store) ;

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOLoad

        plan = lpt.buildPlan("store b into 'empty';");  
        LogicalPlan expected = migratePlan(plan);
        
        assertTrue(expected.isEqual(newLogicalPlan));
        
        LOLoad op = (LOLoad)newLogicalPlan.getSources().get(0);
        Map<Integer,Set<String>> annotation = 
                (Map<Integer, Set<String>>) op.getAnnotation(MapKeysPruneHelper.REQUIRED_MAPKEYS);
        assertEquals(annotation.size(), 1);
        Set<String> s = new HashSet<String>();
        s.add("path");
        assertEquals(annotation.get(2), s);
        
        // foreach with join
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' as (id, v1, m:map[]);");
        lpt.buildPlan("b = load 'd.txt' as (id, v1, m:map[]);");
        lpt.buildPlan("c = join a by id, b by id;");
        lpt.buildPlan("d = filter c by a::m#'path' != NULL;");
        lpt.buildPlan("e = foreach d generate a::id, b::id, b::m#'path', a::m;");        
        plan = lpt.buildPlan("store e into 'empty';");  
        newLogicalPlan = migratePlan(plan);
               
        optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' as (id, m:map[]);");
        lpt.buildPlan("b = load 'd.txt' as (id, m:map[]);");
        lpt.buildPlan("c = join a by id, b by id;");
        lpt.buildPlan("d = filter c by a::m#'path' != NULL;");
        lpt.buildPlan("e = foreach d generate a::id, b::id, b::m#'path', a::m;");        
        plan = lpt.buildPlan("store e into 'empty';");  
        expected = migratePlan(plan);
        
        assertTrue(expected.isEqual(newLogicalPlan));
        
        List<Operator> ll = newLogicalPlan.getSources();
        assertEquals(ll.size(), 2);
        LOLoad loada = null;
        LOLoad loadb = null;
        for(Operator opp: ll) {
            if (((LogicalRelationalOperator)opp).getAlias().equals("a")) {
                loada = (LOLoad)opp;
                continue;
            }
            
            if (((LogicalRelationalOperator)opp).getAlias().equals("b")) {
                loadb = (LOLoad)opp;
                continue;
            }
        }
                
        annotation = 
                (Map<Integer, Set<String>>) loada.getAnnotation(MapKeysPruneHelper.REQUIRED_MAPKEYS);
        assertNull(annotation);
        
        annotation = 
            (Map<Integer, Set<String>>) loadb.getAnnotation(MapKeysPruneHelper.REQUIRED_MAPKEYS);
        assertEquals(annotation.size(), 1);
    
        s = new HashSet<String>();
        s.add("path");
        assertEquals(annotation.get(2), s);

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOLoad

    protected OperatorPlan buildPattern() {
        // the pattern that this rule looks for is load
        LogicalPlan plan = new LogicalPlan();
        LogicalRelationalOperator op;
        if (getOperatorClassName().equals(LOLoad.class.getName()))
            op = new LOLoad(null, null, plan, null);
        else // LOStream
            op = new LOStream(plan, null, null, null);
        plan.add(op);
        return plan;
    }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.