Package org.apache.pig.impl.logicalLayer

Examples of org.apache.pig.impl.logicalLayer.LOStore

1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
                }
                else {
                    // add new store
                    FuncSpec funcSpec = new FuncSpec(InterStorage.class.getName());
                    fileSpec = new FileSpec(FileLocalizer.getTemporaryPath(pigContext).toString(), funcSpec);
                    store = new LOStore(referredPlan, new OperatorKey(scope, NodeIdGenerator.getGenerator().getNextNodeId(scope)),
                            fileSpec, alias);
                    referredPlan.addAsLeaf(store);
                    ((LOStore)store).setTmpStore(true);
                    scalarEntry.getKey().setImplicitReferencedOperator(store);
                }
View Full Code Here
279
280
281
282
283
284
285
286
287
288
289
            new FileSpec(inputFile, new FuncSpec("org.apache.pig.builtin.PigStorage")) ;
        FileSpec filespec2 =
            new FileSpec(outputFile, new FuncSpec("org.apache.pig.builtin.PigStorage"));
        LOLoad load = new LOLoad(plan, genNewOperatorKeyId(), filespec1,
                ConfigurationUtil.toConfiguration(dfs.getConfiguration())) ;      
        LOStore store = new LOStore(plan, genNewOperatorKeyId(), filespec2, "new") ;
       
        plan.add(load) ;
        plan.add(store) ;
       
        plan.connect(load, store) ;    
View Full Code Here
828
829
830
831
832
833
834
835
836
837
838
839
840
        Assert.assertTrue(lp.size()>1);
        LogicalOperator op = lp.getLeaves().get(0);
       
        Assert.assertTrue(op instanceof LOStore);
        LOStore store = (LOStore)op;

        String p = store.getOutputFile().getFileName();
        p = p.replaceAll("hdfs://[0-9a-zA-Z:\\.]*/","/");
       
        if (isTmp) {
            Assert.assertTrue(p.matches("/tmp.*"));
        } else {
View Full Code Here
245
246
247
248
249
250
251
252
253
254
255
            new FileSpec(inputFile, new FuncSpec("org.apache.pig.builtin.PigStorage")) ;
        FileSpec filespec2 =
            new FileSpec(outputFile, new FuncSpec("org.apache.pig.builtin.PigStorage"));
        LOLoad load = new LOLoad(plan, genNewOperatorKeyId(), filespec1,
                ConfigurationUtil.toConfiguration(dfs.getConfiguration())) ;      
        LOStore store = new LOStore(plan, genNewOperatorKeyId(), filespec2, "new") ;
       
        plan.add(load) ;
        plan.add(store) ;
       
        plan.connect(load, store) ;    
View Full Code Here
647
648
649
650
651
652
653
654
655
656
657
658
659
        Assert.assertTrue(lp.size()>1);
        LogicalOperator op = lp.getLeaves().get(0);
       
        Assert.assertTrue(op instanceof LOStore);
        LOStore store = (LOStore)op;

        String p = store.getOutputFile().getFileName();
        p = p.replaceAll("hdfs://[0-9a-zA-Z:\\.]*/","/");
       
        if (isTmp) {
            Assert.assertTrue(p.matches("/tmp.*"));
        } else {
View Full Code Here
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
                }
                else {
                    // add new store
                    FuncSpec funcSpec = new FuncSpec(InterStorage.class.getName());
                    fileSpec = new FileSpec(FileLocalizer.getTemporaryPath(pigContext).toString(), funcSpec);
                    store = new LOStore(referredPlan, new OperatorKey(scope, NodeIdGenerator.getGenerator().getNextNodeId(scope)),
                            fileSpec, alias);
                    referredPlan.addAsLeaf(store);
                    ((LOStore)store).setTmpStore(true);
                    scalarEntry.getKey().setImplicitReferencedOperator(store);
                }
View Full Code Here
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
            pom.setProperty("pig.input.splittable",
                            Boolean.toString(((LOLoad)lo).isSplittable()));
            return pom.getOperatorKey();
        }
        else if (lo instanceof LOStore) {
            LOStore los = (LOStore) lo;
            ((POMapreduce)execEngine.getPhysicalOpTable().get(compiledInputs[0])).outputFileSpec = los.getOutputFileSpec();
           
            ((POMapreduce)execEngine.getPhysicalOpTable().get(compiledInputs[0])).sourceLogicalKey =
                los.getInputs().get(0);
           
            return compiledInputs[0];
        }
        else if (lo instanceof LOUnion) {
            POMapreduce pom = new POMapreduce(lo.getScope(),
View Full Code Here
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
        List<LogicalOperator> successors = mPlan.getSuccessors(stream);
        if(successors == null)
            return;
        LogicalOperator successor = successors.get(0);
        if(successor instanceof LOStore) {
            LOStore store = (LOStore)successor;
            // Try and optimize if the store and stream output specs match
            // and input files are to be processed as-is
            StreamingCommand command = stream.getStreamingCommand();
            HandleSpec streamOutputSpec = command.getOutputSpec();
           
            FileSpec storeFileSpec = store.getOutputFile();
           
            // Instantiate both to compare them for equality
            LoadFunc streamLoader =
                (LoadFunc)PigContext.instantiateFuncFromSpec(
                        streamOutputSpec.getSpec());
View Full Code Here
220
221
222
223
224
225
226
227
228
229
230
231
232
                load.setInputFile(new FileSpec(loadFileSpec.getFileName(), new FuncSpec(BinaryStorage.class.getName())));
                stream.setOptimizedSpec(Handle.INPUT, BinaryStorage.class.getName());
            }
            if(mOptimizeStore) {
                // we have verified in check() that the predecessor was load
                LOStore store = (LOStore)mPlan.getSuccessors(lo).get(0);
                FileSpec storeFileSpec = store.getOutputFile();
                store.setOutputFile(new FileSpec(storeFileSpec.getFileName(), new FuncSpec(BinaryStorage.class.getName())));
                stream.setOptimizedSpec(Handle.OUTPUT, BinaryStorage.class.getName());
            }
        } catch (Exception e) {
            int errCode = 2014;
            String msg = "Unable to optimize load-stream-store optimization";
View Full Code Here
276
277
278
279
280
281
282
283
284
285
286
287
288
        Assert.assertTrue(lp.size()>1);
        LogicalOperator op = lp.getLeaves().get(0);
       
        Assert.assertTrue(op instanceof LOStore);
        LOStore store = (LOStore)op;

        String p = store.getOutputFile().getFileName();
        p = p.replaceAll("hdfs://[0-9a-zA-Z:\\.]*/","/");
       
        if (isTmp) {
            Assert.assertTrue(p.matches("/tmp.*"));
        } else {
View Full Code Here
TOP

Related Classes of org.apache.pig.impl.logicalLayer.LOStore

  • org.apache.pig.backend.hadoop.executionengine.MapreducePlanCompiler
  • org.apache.pig.impl.logicalLayer.optimizer.StreamOptimizer
  • org.apache.pig.impl.logicalLayer.schema.Schema
  • org.apache.pig.impl.plan.ProjectionMap
  • org.apache.pig.impl.plan.RequiredFields
  • org.apache.pig.PigServer
  • org.apache.pig.StoreFunc
  • org.apache.pig.test.TestInputOutputFileValidator
  • org.apache.pig.test.TestStore
  • java.io.IOException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.