Package org.apache.pig.impl.logicalLayer

Examples of org.apache.pig.impl.logicalLayer.LOLoad

263
264
265
266
267
268
269
270
271
    @Test
    public void testNonDfsLocation() throws Exception {
        LogicalPlanTester lpt = new LogicalPlanTester();
        String nonDfsUrl = "har://hdfs-namenode/user/foo/";
        LogicalPlan lp = lpt.buildPlan("a = load '" + nonDfsUrl + "';");
        LOLoad load = (LOLoad) lp.getRoots().get(0);
        nonDfsUrl = nonDfsUrl.replaceFirst("/$", "");
        Assert.assertEquals(nonDfsUrl, load.getInputFile().getFileName());
    }
View Full Code Here
338
339
340
341
342
343
344
345
346
347
348
349
350
                                           fileNameMap);
            Assert.assertTrue(lp.size()>0);
            LogicalOperator op = lp.getRoots().get(0);
           
            Assert.assertTrue(op instanceof LOLoad);
            LOLoad load = (LOLoad)op;
   
            String p = load.getInputFile().getFileName();
            System.err.println("DEBUG: p:" + p + " expected:" + expected +", exectype:" + pc.getExecType());
            if(noConversionExpected) {
                Assert.assertEquals(p, expected);
            } else  {
                if (pc.getExecType() == ExecType.MAPREDUCE) {
View Full Code Here
277
278
279
280
281
282
283
284
285
286
287
        LogicalPlan plan = new LogicalPlan() ;
        FileSpec filespec1 =
            new FileSpec(inputFile, new FuncSpec("org.apache.pig.builtin.PigStorage")) ;
        FileSpec filespec2 =
            new FileSpec(outputFile, new FuncSpec("org.apache.pig.builtin.PigStorage"));
        LOLoad load = new LOLoad(plan, genNewOperatorKeyId(), filespec1,
                ConfigurationUtil.toConfiguration(dfs.getConfiguration())) ;      
        LOStore store = new LOStore(plan, genNewOperatorKeyId(), filespec2, "new") ;
       
        plan.add(load) ;
        plan.add(store) ;
View Full Code Here
243
244
245
246
247
248
249
250
251
252
253
        LogicalPlan plan = new LogicalPlan() ;
        FileSpec filespec1 =
            new FileSpec(inputFile, new FuncSpec("org.apache.pig.builtin.PigStorage")) ;
        FileSpec filespec2 =
            new FileSpec(outputFile, new FuncSpec("org.apache.pig.builtin.PigStorage"));
        LOLoad load = new LOLoad(plan, genNewOperatorKeyId(), filespec1,
                ConfigurationUtil.toConfiguration(dfs.getConfiguration())) ;      
        LOStore store = new LOStore(plan, genNewOperatorKeyId(), filespec2, "new") ;
       
        plan.add(load) ;
        plan.add(store) ;
View Full Code Here
215
216
217
218
219
220
221
222
    @Test
    public void testNonDfsLocation() throws Exception {
        LogicalPlanTester lpt = new LogicalPlanTester();
        String nonDfsUrl = "har://hdfs-namenode/user/foo/";
        LogicalPlan lp = lpt.buildPlan("a = load '" + nonDfsUrl + "';");
        LOLoad load = (LOLoad) lp.getRoots().get(0);
        Assert.assertEquals(nonDfsUrl, load.getInputFile().getFileName());
    }
View Full Code Here
246
247
248
249
250
251
252
253
254
255
256
257
258
                                           fileNameMap);
            Assert.assertTrue(lp.size()>0);
            LogicalOperator op = lp.getRoots().get(0);
           
            Assert.assertTrue(op instanceof LOLoad);
            LOLoad load = (LOLoad)op;
   
            String p = load.getInputFile().getFileName();
            System.err.println("DEBUG: p:" + p + " expected:" + expected +", exectype:" + pc.getExecType());       
            if (pc.getExecType() == ExecType.MAPREDUCE) {
                Assert.assertTrue(p.matches("hdfs://[0-9a-zA-Z:\\.]*.*"));
                Assert.assertEquals(p.replaceAll("hdfs://[0-9a-zA-Z:\\.]*/", "/"),
                        expected);
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
        if((predecessors == null) || (predecessors.size() <= 0)) {
            return;
        }
        LogicalOperator predecessor = predecessors.get(0);
        if(predecessor instanceof LOLoad) {
            LOLoad load = (LOLoad)predecessor;
            if(!load.isSplittable()) {
                // Try and optimize if the load and stream input specs match
                // and input files are to be processed as-is
                StreamingCommand command = stream.getStreamingCommand();
                HandleSpec streamInputSpec = command.getInputSpec();
                FileSpec loadFileSpec = load.getInputFile();
                // Instantiate both LoadFunc objects to compare them for
                // equality
                StoreFunc streamStorer =
                    (StoreFunc)PigContext.instantiateFuncFromSpec(
                            streamInputSpec.getSpec());
View Full Code Here
213
214
215
216
217
218
219
220
221
222
223
224
225
                    lo.getClass().getName());
            }
            LOStream stream = (LOStream)lo;
            if(mOptimizeLoad) {
                // we have verified in check() that the predecessor was load
                LOLoad load = (LOLoad)mPlan.getPredecessors(lo).get(0);
                FileSpec loadFileSpec = load.getInputFile();
                load.setInputFile(new FileSpec(loadFileSpec.getFileName(), new FuncSpec(BinaryStorage.class.getName())));
                stream.setOptimizedSpec(Handle.INPUT, BinaryStorage.class.getName());
            }
            if(mOptimizeStore) {
                // we have verified in check() that the predecessor was load
                LOStore store = (LOStore)mPlan.getSuccessors(lo).get(0);
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
    public static LOLoad genDummyLOLoad(LogicalPlan plan)  {
        String pigStorage = PigStorage.class.getName() ;
        DataStorage dfs = new HDataStorage(new Properties());
        try {
            LOLoad load = new LOLoad(plan,
                                      genNewOperatorKey(),
                                      new FileSpec("pi", new FuncSpec(pigStorage)),
                                      null, dfs, true) ;
            return load ;
        } catch (IOException e) {
View Full Code Here
218
219
220
221
222
223
224
225
226
227
228
229
230
                                       fileNameMap);
        Assert.assertTrue(lp.size()>0);
        LogicalOperator op = lp.getRoots().get(0);
       
        Assert.assertTrue(op instanceof LOLoad);
        LOLoad load = (LOLoad)op;

        String p = load.getInputFile().getFileName();
        p = p.replaceAll("hdfs://[0-9a-zA-Z:\\.]*/","/");

        if (isTmp) {
            Assert.assertTrue(p.matches("/tmp.*"));
        } else {
View Full Code Here
TOP

Related Classes of org.apache.pig.impl.logicalLayer.LOLoad

  • org.apache.hadoop.mapreduce.Job
  • org.apache.pig.impl.logicalLayer.optimizer.StreamOptimizer
  • org.apache.pig.impl.logicalLayer.schema.Schema
  • org.apache.pig.impl.logicalLayer.schema.TupleSchema
  • org.apache.pig.impl.plan.ProjectionMap
  • org.apache.pig.impl.plan.RequiredFields
  • org.apache.pig.LoadMetadata
  • org.apache.pig.LoadPushDown.RequiredField
  • org.apache.pig.LoadPushDown.RequiredFieldResponse
  • org.apache.pig.ResourceSchema
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.