Examples of PigStats

org.apache.pig.tools.pigstats.PigStats
PigStats encapsulates the statistics collected from a running script. It includes status of the execution, the DAG of its MR jobs, as well as information about outputs and inputs of the script.

Examples of org.apache.pig.tools.pigstats.PigStats


        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = group a by $0;");
        pigServer.registerQuery("c = foreach b generate group;");
        PigStats pigStats = pigServer.store("c", "output", "BinStorage").getStatistics();


        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output", pigServer.getPigContext()), ExecType.MAPREDUCE, pigServer.getPigContext().getDfs());
        long filesize = 0;
        while(is.read() != -1) filesize++;
        
        is.close();
        
        Map<String, Map<String, String>> stats = pigStats.getPigStats();
        cluster.getFileSystem().delete(new Path(file), true);
        cluster.getFileSystem().delete(new Path("output"), true);


        System.out.println("============================================");
        System.out.println("Test case MapReduce");
        System.out.println("============================================");
        System.out.println("MRPlan : \n" + pigStats.getMRPlan());
        for(Map.Entry<String, Map<String, String>> entry : stats.entrySet()) {
            System.out.println("============================================");
            System.out.println("Job : " + entry.getKey());
            for(Map.Entry<String, String> e1 : entry.getValue().entrySet()) {
                System.out.println(" - " + e1.getKey() + " : \n" + e1.getValue());
            }
            System.out.println("============================================");
        }


        Map.Entry<String, Map<String, String>> e = stats.entrySet().iterator().next();


        Map<String, String> jobStats = e.getValue();


        System.out.println("Map input records : " + jobStats.get("PIG_STATS_MAP_INPUT_RECORDS"));
        assertEquals(MAX, Integer.parseInt(jobStats.get("PIG_STATS_MAP_INPUT_RECORDS")));
        System.out.println("Map output records : " + jobStats.get("PIG_STATS_MAP_OUTPUT_RECORDS"));
        assertEquals(MAX, Integer.parseInt(jobStats.get("PIG_STATS_MAP_OUTPUT_RECORDS")));
        System.out.println("Reduce input records : " + jobStats.get("PIG_STATS_REDUCE_INPUT_RECORDS"));
        assertEquals(MAX, Integer.parseInt(jobStats.get("PIG_STATS_REDUCE_INPUT_RECORDS")));
        System.out.println("Reduce output records : " + jobStats.get("PIG_STATS_REDUCE_OUTPUT_RECORDS"));
        assertEquals(count, Integer.parseInt(jobStats.get("PIG_STATS_REDUCE_OUTPUT_RECORDS")));


        assertNull(jobStats.get("PIG_STATS_COMBINE_PLAN"));
        assertNotNull(jobStats.get("PIG_STATS_MAP_PLAN"));
        assertNotNull(jobStats.get("PIG_STATS_REDUCE_PLAN"));
        
        assertEquals(count, pigStats.getRecordsWritten());
        assertEquals(filesize, pigStats.getBytesWritten());


    }

View Full Code Here

Examples of org.apache.pig.tools.pigstats.PigStats


        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = group a by $0;");
        pigServer.registerQuery("c = foreach b generate group, SUM(a.$1);");
        PigStats pigStats = pigServer.store("c", "output").getStatistics();


        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output", pigServer.getPigContext()), ExecType.MAPREDUCE, pigServer.getPigContext().getDfs());
        long filesize = 0;
        while(is.read() != -1) filesize++;
        
        is.close();
        Map<String, Map<String, String>> stats = pigStats.getPigStats();
        cluster.getFileSystem().delete(new Path(file), true);
        cluster.getFileSystem().delete(new Path("output"), true);


        System.out.println("============================================");
        System.out.println("Test case MapCombineReduce");
        System.out.println("============================================");
        System.out.println("MRPlan : \n" + pigStats.getMRPlan());
        for(Map.Entry<String, Map<String, String>> entry : stats.entrySet()) {
            System.out.println("============================================");
            System.out.println("Job : " + entry.getKey());
            for(Map.Entry<String, String> e1 : entry.getValue().entrySet()) {
                System.out.println(" - " + e1.getKey() + " : \n" + e1.getValue());
            }
            System.out.println("============================================");
        }


        Map.Entry<String, Map<String, String>> e = stats.entrySet().iterator().next();


        Map<String, String> jobStats = e.getValue();


        System.out.println("Map input records : " + jobStats.get("PIG_STATS_MAP_INPUT_RECORDS"));
        assertEquals(MAX, Integer.parseInt(jobStats.get("PIG_STATS_MAP_INPUT_RECORDS")));
        System.out.println("Map output records : " + jobStats.get("PIG_STATS_MAP_OUTPUT_RECORDS"));
        assertEquals(MAX, Integer.parseInt(jobStats.get("PIG_STATS_MAP_OUTPUT_RECORDS")));
        System.out.println("Reduce input records : " + jobStats.get("PIG_STATS_REDUCE_INPUT_RECORDS"));
        assertEquals(count, Integer.parseInt(jobStats.get("PIG_STATS_REDUCE_INPUT_RECORDS")));
        System.out.println("Reduce output records : " + jobStats.get("PIG_STATS_REDUCE_OUTPUT_RECORDS"));
        assertEquals(count, Integer.parseInt(jobStats.get("PIG_STATS_REDUCE_OUTPUT_RECORDS")));


        assertNotNull(jobStats.get("PIG_STATS_COMBINE_PLAN"));
        assertNotNull(jobStats.get("PIG_STATS_MAP_PLAN"));
        assertNotNull(jobStats.get("PIG_STATS_REDUCE_PLAN"));


        assertEquals(count, pigStats.getRecordsWritten());
        assertEquals(filesize, pigStats.getBytesWritten());
    }

View Full Code Here

Examples of org.apache.pig.tools.pigstats.PigStats


        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = group a by $0;");
        pigServer.registerQuery("c = foreach b generate group, SUM(a.$1);");
        PigStats pigStats = pigServer.store("c", "output", "BinStorage").getStatistics();


        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output", pigServer.getPigContext()), ExecType.MAPREDUCE, pigServer.getPigContext().getDfs());
        long filesize = 0;
        while(is.read() != -1) filesize++;
        
        is.close();
        Map<String, Map<String, String>> stats = pigStats.getPigStats();
        cluster.getFileSystem().delete(new Path(file), true);
        cluster.getFileSystem().delete(new Path("output"), true);


        System.out.println("============================================");
        System.out.println("Test case MapCombineReduce");
        System.out.println("============================================");
        System.out.println("MRPlan : \n" + pigStats.getMRPlan());
        for(Map.Entry<String, Map<String, String>> entry : stats.entrySet()) {
            System.out.println("============================================");
            System.out.println("Job : " + entry.getKey());
            for(Map.Entry<String, String> e1 : entry.getValue().entrySet()) {
                System.out.println(" - " + e1.getKey() + " : \n" + e1.getValue());
            }
            System.out.println("============================================");
        }


        Map.Entry<String, Map<String, String>> e = stats.entrySet().iterator().next();


        Map<String, String> jobStats = e.getValue();


        System.out.println("Map input records : " + jobStats.get("PIG_STATS_MAP_INPUT_RECORDS"));
        assertEquals(MAX, Integer.parseInt(jobStats.get("PIG_STATS_MAP_INPUT_RECORDS")));
        System.out.println("Map output records : " + jobStats.get("PIG_STATS_MAP_OUTPUT_RECORDS"));
        assertEquals(MAX, Integer.parseInt(jobStats.get("PIG_STATS_MAP_OUTPUT_RECORDS")));
        System.out.println("Reduce input records : " + jobStats.get("PIG_STATS_REDUCE_INPUT_RECORDS"));
        assertEquals(count, Integer.parseInt(jobStats.get("PIG_STATS_REDUCE_INPUT_RECORDS")));
        System.out.println("Reduce output records : " + jobStats.get("PIG_STATS_REDUCE_OUTPUT_RECORDS"));
        assertEquals(count, Integer.parseInt(jobStats.get("PIG_STATS_REDUCE_OUTPUT_RECORDS")));


        assertNotNull(jobStats.get("PIG_STATS_COMBINE_PLAN"));
        assertNotNull(jobStats.get("PIG_STATS_MAP_PLAN"));
        assertNotNull(jobStats.get("PIG_STATS_REDUCE_PLAN"));


        assertEquals(count, pigStats.getRecordsWritten());
        assertEquals(filesize, pigStats.getBytesWritten());
    }

View Full Code Here

Examples of org.apache.pig.tools.pigstats.PigStats

        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = order a by $0;");
        pigServer.registerQuery("c = group b by $0;");
        pigServer.registerQuery("d = foreach c generate group, SUM(b.$1);");
        PigStats pigStats = pigServer.store("d", "output").getStatistics();
        
        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output", pigServer.getPigContext()), ExecType.MAPREDUCE, pigServer.getPigContext().getDfs());
        long filesize = 0;
        while(is.read() != -1) filesize++;
        
        is.close();
        Map<String, Map<String, String>> stats = pigStats.getPigStats();
        cluster.getFileSystem().delete(new Path(file), true);
        cluster.getFileSystem().delete(new Path("output"), true);
        
        System.out.println("============================================");
        System.out.println("Test case MultipleMRJobs");
        System.out.println("============================================");
        System.out.println("MRPlan : \n" + pigStats.getMRPlan());
        for(Map.Entry<String, Map<String, String>> entry : stats.entrySet()) {
            System.out.println("============================================");
            System.out.println("Job : " + entry.getKey());
            for(Map.Entry<String, String> e1 : entry.getValue().entrySet()) {
                System.out.println(" - " + e1.getKey() + " : \n" + e1.getValue());
            }
            System.out.println("============================================");
        }


        Map<String, String> jobStats = stats.get(pigStats.getRootJobIDs().get(0));


        System.out.println("Map input records : " + jobStats.get("PIG_STATS_MAP_INPUT_RECORDS"));
        assertEquals(MAX, Integer.parseInt(jobStats.get("PIG_STATS_MAP_INPUT_RECORDS")));
        System.out.println("Map output records : " + jobStats.get("PIG_STATS_MAP_OUTPUT_RECORDS"));
        assertEquals(MAX, Integer.parseInt(jobStats.get("PIG_STATS_MAP_OUTPUT_RECORDS")));
        System.out.println("Reduce input records : " + jobStats.get("PIG_STATS_REDUCE_INPUT_RECORDS"));
        assertEquals(count, Integer.parseInt(jobStats.get("PIG_STATS_REDUCE_INPUT_RECORDS")));
        System.out.println("Reduce output records : " + jobStats.get("PIG_STATS_REDUCE_OUTPUT_RECORDS"));
        assertEquals(count, Integer.parseInt(jobStats.get("PIG_STATS_REDUCE_OUTPUT_RECORDS")));
        
        assertEquals(count, pigStats.getRecordsWritten());
        assertEquals(filesize, pigStats.getBytesWritten());


    }

View Full Code Here

Examples of org.apache.pig.tools.pigstats.PigStats

            String filePath = outputFile.getAbsolutePath();
            outputFile.delete();
            PigServer pig = new PigServer(ExecType.LOCAL);
            pig.registerQuery("A = load 'test/org/apache/pig/test/data/passwd';");
            ExecJob job = pig.store("A", filePath);
            PigStats stats = job.getStatistics();
            File dataFile = new File( outputFile.getAbsoluteFile() + File.separator + "part-00000" );
            assertEquals(dataFile.length(), stats.getBytesWritten());
        } catch (IOException e) {
            e.printStackTrace();
            System.err.println( e.getMessage() );
            fail("IOException happened");
        } finally {

View Full Code Here

Examples of org.apache.pig.tools.pigstats.PigStats

                if (spec != null)
                    leafMap.put(spec.toString(), physOp);
            }
        }
        try {
            PigStats stats = launcher.launchPig(plan, jobName, pigContext);


            for (OutputStats output : stats.getOutputStats()) {
                POStore store = output.getPOStore();               
                String alias = store.getAlias();
                if (output.isSuccessful()) {
                    jobs.add(new HJob(ExecJob.JOB_STATUS.COMPLETED, pigContext, store, alias, stats));
                } else {

View Full Code Here

Examples of org.apache.pig.tools.pigstats.PigStats

        w.println("fs -mv nonexist.file dummy.file");
        w.close();
                                        
        try {
            String[] args = { PIG_FILE };
            PigStats stats = PigRunner.run(args, new TestNotificationListener());
                                                                             
            assertTrue(!stats.isSuccessful());
            assertTrue(stats.getReturnCode() == PigRunner.ReturnCode.IO_EXCEPTION);
        } finally {
            new File(PIG_FILE).delete();
        }
    }

View Full Code Here

Examples of org.apache.pig.tools.pigstats.PigStats

        w.println("store D into '" + OUTPUT_FILE + "';");
        w.close();
        
        try {
            String[] args = { PIG_FILE };
            PigStats stats = PigRunner.run(args, null);
            
            assertTrue(!stats.isSuccessful());
            assertTrue(stats.getReturnCode() != 0);
            
        } finally {
            new File(PIG_FILE).delete();
            Util.deleteFile(cluster, OUTPUT_FILE);
        }

View Full Code Here

Examples of org.apache.pig.tools.pigstats.PigStats

     * @return list of jobs being executed
     * @throws FrontendException
     * @throws ExecException
     */
    public List<ExecJob> executeBatch() throws FrontendException, ExecException {
        PigStats stats = executeBatchEx();
        LinkedList<ExecJob> jobs = new LinkedList<ExecJob>();
        JobGraph jGraph = stats.getJobGraph();
        Iterator<JobStats> iter = jGraph.iterator();
        while (iter.hasNext()) {
            JobStats js = iter.next();
            for (OutputStats output : js.getOutputs()) {
                if (js.isSuccessful()) {

View Full Code Here

Examples of org.apache.pig.tools.pigstats.PigStats

     * @return {@link ExecJob} containing information about this job
     * @throws IOException
     */
    public ExecJob store(String id, String filename, String func) 
            throws IOException {
        PigStats stats = storeEx(id, filename, func);
        if (stats.getOutputStats().size() < 1) {
            throw new IOException("Couldn't retrieve job.");
        }
        OutputStats output = stats.getOutputStats().get(0);


        if(stats.isSuccessful()){
            return  new HJob(JOB_STATUS.COMPLETED, pigContext, output
                    .getPOStore(), output.getAlias(), stats);
        }else{
            HJob job = new HJob(JOB_STATUS.FAILED, pigContext,
                    output.getPOStore(), output.getAlias(), stats);
            
            //check for exception
            Exception ex = null;
            for(JobStats js : stats.getJobGraph()){
                if(js.getException() != null)
                    ex = js.getException();
            }
            job.setException(ex);
            return job;

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.