Examples of JobStats

Examples of org.apache.pig.tools.pigstats.JobStats

380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
        System.out.println("============================================");
        JobGraph jp = pigStats.getJobGraph();
        Iterator<JobStats> iter = jp.iterator();
        while (iter.hasNext()) {
            JobStats js = iter.next();
            System.out.println("Map input records : " + js.getMapInputRecords());
            assertEquals(MAX, js.getMapInputRecords());
            System.out.println("Map output records : " + js.getMapOutputRecords());
            assertEquals(MAX, js.getMapOutputRecords());
            System.out.println("Reduce input records : " + js.getReduceInputRecords());
            assertEquals(count, js.getReduceInputRecords());
            System.out.println("Reduce output records : " + js.getReduceOutputRecords());
            assertEquals(count, js.getReduceOutputRecords());
        }
        System.out.println("Hdfs bytes written : " + pigStats.getBytesWritten());
        assertEquals(filesize, pigStats.getBytesWritten());
    }
View Full Code Here

Examples of org.apache.pig.tools.pigstats.JobStats

437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
        System.out.println("============================================");
        System.out.println("Test case MultipleMRJobs");
        System.out.println("============================================");
       
        JobGraph jp = pigStats.getJobGraph();
        JobStats js = (JobStats)jp.getSinks().get(0);
       
        System.out.println("Job id: " + js.getName());
        System.out.println(jp.toString());
       
        System.out.println("Map input records : " + js.getMapInputRecords());
        assertEquals(MAX, js.getMapInputRecords());
        System.out.println("Map output records : " + js.getMapOutputRecords());
        assertEquals(MAX, js.getMapOutputRecords());
        System.out.println("Reduce input records : " + js.getReduceInputRecords());
        assertEquals(count, js.getReduceInputRecords());
        System.out.println("Reduce output records : " + js.getReduceOutputRecords());
        assertEquals(count, js.getReduceOutputRecords());
       
        System.out.println("Hdfs bytes written : " + js.getHdfsBytesWritten());
        assertEquals(filesize, js.getHdfsBytesWritten());

    }
View Full Code Here

Examples of org.apache.pig.tools.pigstats.JobStats

481
482
483
484
485
486
487
488
489
490
491
492
493
       
        cluster.getFileSystem().delete(new Path(file), true);
        cluster.getFileSystem().delete(new Path("/tmp/outout1"), true);
        cluster.getFileSystem().delete(new Path("/tmp/outout2"), true);

        JobStats js = (JobStats)stats.getJobGraph().getSinks().get(0);
       
        Map<String, Long> entry = js.getMultiStoreCounters();
        long counter = 0;
        for (Long val : entry.values()) {
            counter += val;
        }
       
View Full Code Here

Examples of org.apache.pig.tools.pigstats.JobStats

529
530
531
532
533
534
535
536
537
538
539
540
541
              
        cluster.getFileSystem().delete(new Path(file), true);
        cluster.getFileSystem().delete(new Path("/tmp/outout1"), true);
        cluster.getFileSystem().delete(new Path("/tmp/outout2"), true);

        JobStats js = (JobStats)stats.getJobGraph().getSinks().get(0);
       
        Map<String, Long> entry = js.getMultiStoreCounters();
        long counter = 0;
        for (Long val : entry.values()) {
            counter += val;
        }
       
View Full Code Here

Examples of org.apache.pig.tools.pigstats.JobStats

77
78
79
80
81
82
83
84
85
86
87
88
            String[] args = { PIG_FILE };
            PigStats stats = PigRunner.run(args, null);
    
            assertTrue(stats.isSuccessful());
            // the sampler job has zero maps
            JobStats js = (JobStats)stats.getJobGraph().getSources().get(0);
            assertEquals(0, js.getNumberMaps());
           
            FileSystem fs = cluster.getFileSystem();
            FileStatus status = fs.getFileStatus(new Path(OUTPUT_FILE));
            assertTrue(status.isDir());
            assertEquals(0, status.getLen());
View Full Code Here

Examples of org.apache.pig.tools.pigstats.JobStats

107
108
109
110
111
112
113
114
115
116
117
118
            String[] args = { PIG_FILE };
            PigStats stats = PigRunner.run(args, null);
    
            assertTrue(stats.isSuccessful());   
            // the indexer job has zero maps
            JobStats js = (JobStats)stats.getJobGraph().getSources().get(0);
            assertEquals(0, js.getNumberMaps());
           
            FileSystem fs = cluster.getFileSystem();
            FileStatus status = fs.getFileStatus(new Path(OUTPUT_FILE));
            assertTrue(status.isDir());
            assertEquals(0, status.getLen());
View Full Code Here

Examples of org.apache.pig.tools.pigstats.JobStats

138
139
140
141
142
143
144
145
146
147
148
149
            String[] args = { PIG_FILE };
            PigStats stats = PigRunner.run(args, null);
    
            assertTrue(stats.isSuccessful());   
            // the indexer job has zero maps
            JobStats js = (JobStats)stats.getJobGraph().getSources().get(0);
            assertEquals(0, js.getNumberMaps());
           
            FileSystem fs = cluster.getFileSystem();
            FileStatus status = fs.getFileStatus(new Path(OUTPUT_FILE));
            assertTrue(status.isDir());
            assertEquals(0, status.getLen());
View Full Code Here

Examples of org.apache.pig.tools.pigstats.JobStats

116
117
118
119
120
121
122
123
124
125
126
127
128
            }
           
            JobGraph jGraph = PigStats.get().getJobGraph();
            assertEquals(3, jGraph.size());
            // find added map-only concatenate job
            JobStats js = (JobStats)jGraph.getSuccessors(jGraph.getSources().get(0)).get(0);
            assertEquals(1, js.getNumberMaps());  
            assertEquals(0, js.getNumberReduces());
        }
        {
            pigServer.getPigContext().getProperties().setProperty(
                    "pig.noSplitCombination", "true");
           
View Full Code Here

Examples of org.apache.pig.tools.pigstats.JobStats

165
166
167
168
169
170
171
172
173
174
175
176
177
            }
           
            JobGraph jGraph = PigStats.get().getJobGraph();
            assertEquals(3, jGraph.size());
            // find added map-only concatenate job
            JobStats js = (JobStats)jGraph.getSuccessors(jGraph.getSources().get(0)).get(0);
            assertEquals(1, js.getNumberMaps());  
            assertEquals(0, js.getNumberReduces());
            Util.checkLogFileMessage(logFile,
                    new String[] {"number of input files: 0", "failed to get number of input files"},
                    false
            );
        }
View Full Code Here

Examples of org.apache.pig.tools.pigstats.JobStats

306
307
308
309
310
311
312
313
314
315
316
317
318
            }
           
            JobGraph jGraph = PigStats.get().getJobGraph();
            assertEquals(3, jGraph.size());
            // find added map-only concatenate job
            JobStats js = (JobStats)jGraph.getSuccessors(jGraph.getSources().get(0)).get(0);
            assertEquals(1, js.getNumberMaps());  
            assertEquals(0, js.getNumberReduces());  
        }
        {
            pigServer.getPigContext().getProperties().setProperty(
                    "pig.noSplitCombination", "true");
            pigServer.registerQuery("D = join C by $0, B by $0 using 'repl';");
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.