Examples of LOStore

Examples of org.apache.pig.impl.logicalLayer.LOStore

150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
        List<LogicalOperator> successors = mPlan.getSuccessors(stream);
        if(successors == null)
            return;
        LogicalOperator successor = successors.get(0);
        if(successor instanceof LOStore) {
            LOStore store = (LOStore)successor;
            // Try and optimize if the store and stream output specs match
            // and input files are to be processed as-is
            StreamingCommand command = stream.getStreamingCommand();
            HandleSpec streamOutputSpec = command.getOutputSpec();
           
            FileSpec storeFileSpec = store.getOutputFile();
           
            // Instantiate both to compare them for equality
            LoadFunc streamLoader =
                (LoadFunc)PigContext.instantiateFuncFromSpec(
                        streamOutputSpec.getSpec());
View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOStore

220
221
222
223
224
225
226
227
228
229
230
231
232
                load.setInputFile(new FileSpec(loadFileSpec.getFileName(), new FuncSpec(BinaryStorage.class.getName())));
                stream.setOptimizedSpec(Handle.INPUT, BinaryStorage.class.getName());
            }
            if(mOptimizeStore) {
                // we have verified in check() that the predecessor was load
                LOStore store = (LOStore)mPlan.getSuccessors(lo).get(0);
                FileSpec storeFileSpec = store.getOutputFile();
                store.setOutputFile(new FileSpec(storeFileSpec.getFileName(), new FuncSpec(BinaryStorage.class.getName())));
                stream.setOptimizedSpec(Handle.OUTPUT, BinaryStorage.class.getName());
            }
        } catch (Exception e) {
            int errCode = 2014;
            String msg = "Unable to optimize load-stream-store optimization";
View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOStore

238
239
240
241
242
243
244
245
246
247
248
        innerPlan.connect(l2, gen);
       
        foreach.setInnerPlan(innerPlan);     
        expected.add(foreach);
       
        LOStore s = new LOStore(expected, new FileSpec("/test/empty", new FuncSpec("org.apache.pig.builtin.PigStorage")));
     
        expected.add(s);
       
        expected.connect(load, foreach);
        expected.connect(foreach, s);
View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOStore

371
372
373
374
375
376
377
378
379
380
381
        innerPlan.connect(l2, gen);
       
        foreach2.setInnerPlan(innerPlan);     
        expected.add(foreach2);
               
        LOStore s = new LOStore(expected, new FileSpec("/test/empty", new FuncSpec("org.apache.pig.builtin.PigStorage")));
     
        expected.add(s);
       
        expected.connect(load, foreach2);
   
View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOStore

117
118
119
120
121
122
123
124
125
126
127
       
       
        f1.setAlias("D");
        plan.add(f1);
       
        LogicalRelationalOperator s1 = new LOStore(plan);
        plan.add(s1);      
       
        // load --|-join - filter - store
        // load --|  
        plan.connect(l1, j1);
View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOStore

60
61
62
63
64
65
66
67
68
            "b = foreach a generate *;" +
            "store b into '111';";
       
        LogicalPlan lp = Util.parseAndPreprocess(query, pc);
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOForEach loForEach = (LOForEach)lp.getPredecessors(loStore).get(0);
        assert(loForEach.getSchema()==null);
    }
View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOStore

73
74
75
76
77
78
79
80
81
82
83
            "B = foreach A generate u.$100;" +
            "store B into '111';";
       
        LogicalPlan lp = Util.parseAndPreprocess(query, pc);
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOForEach loForEach = (LOForEach)lp.getPredecessors(loStore).get(0);
        LogicalSchema schema = loForEach.getSchema();
        Assert.assertTrue(schema.size()==1);
        LogicalFieldSchema bagFieldSchema = schema.getField(0);
        Assert.assertTrue(bagFieldSchema.type==DataType.BAG);
View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOStore

92
93
94
95
96
97
98
99
100
101
102
            "B = foreach A generate u.$100;" +
            "store B into '111';";
       
        LogicalPlan lp = Util.parseAndPreprocess(query, pc);
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOForEach loForEach = (LOForEach)lp.getPredecessors(loStore).get(0);
        LogicalSchema schema = loForEach.getSchema();
        Assert.assertTrue(schema.size()==1);
        Assert.assertTrue(schema.getField(0).type==DataType.BYTEARRAY);
    }
View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOStore

107
108
109
110
111
112
113
114
115
116
117
            "B = foreach A { B1 = filter u by $1==0; generate B1;};" +
            "store B into '111';";
       
        LogicalPlan lp = Util.parseAndPreprocess(query, pc);
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOForEach loForEach = (LOForEach)lp.getPredecessors(loStore).get(0);
        LogicalSchema schema = loForEach.getSchema();
        Assert.assertTrue(schema.size()==1);
        LogicalFieldSchema bagFieldSchema = schema.getField(0);
        Assert.assertTrue(bagFieldSchema.type==DataType.BAG);
View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOStore

125
126
127
128
129
130
131
132
133
134
135
            "B = order A by *;" +
            "store B into '111';";
       
        LogicalPlan lp = Util.parseAndPreprocess(query, pc);
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOSort loSort = (LOSort)lp.getPredecessors(loStore).get(0);
        Operator sortPlanLeaf = loSort.getSortColPlans().get(0).getSources().get(0);
        LogicalFieldSchema sortPlanFS = ((LogicalExpression)sortPlanLeaf).getFieldSchema();
        Assert.assertTrue(sortPlanFS==null);
       
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.