Examples of org.apache.pig.tools.pigstats.PigStats

org.apache.pig.tools.pigstats.PigStats
PigStats encapsulates the statistics collected from a running script. It includes status of the execution, the DAG of its MR jobs, as well as information about outputs and inputs of the script.

        w.println("C= order B by $0;");
        w.println("store C into '" + OUTPUT_FILE + "';");
        w.close();
        try {
            String[] args = { PIG_FILE };
            PigStats stats = PigRunner.run(args, null);
            Iterator<JobStats> iter = stats.getJobGraph().iterator();
            while (iter.hasNext()) {
                 JobStats js=iter.next();
                 if(js.getState().name().equals("FAILED")) {
                     List<Operator> ops=stats.getJobGraph().getSuccessors(js);
                     for(Operator op : ops ) {
                         assertEquals(((JobStats)op).getState().toString(), "UNKNOWN");
                     }
                 }
            }

View Full Code Here

        w.println("B = group A by a;");
        w.println("C = foreach B generate group, COUNT(A);");
        w.println("store C into '" + OUTPUT_FILE + "';");
        w.close();
        String[] args = { "-c", PIG_FILE };
        PigStats stats = PigRunner.run(args, null);
        assertTrue(stats.getReturnCode() == ReturnCode.PIG_EXCEPTION);
        assertTrue(stats.getErrorCode() == 1000);
        assertEquals("Error during parsing. Invalid alias: a in {a0: int,a1: int,a2: int}", 
                stats.getErrorMessage());
    }

View Full Code Here

    }
    
    @Test
    public void simpleNegativeTest2() throws Exception {
        String[] args = { "-c", "-e", "this is a test" };
        PigStats stats = PigRunner.run(args, new TestNotificationListener());        
        assertTrue(stats.getReturnCode() == ReturnCode.ILLEGAL_ARGS);
    }

View Full Code Here

    }


    @Test
    public void simpleNegativeTest3() throws Exception {
        String[] args = { "-c", "-y" };
        PigStats stats = PigRunner.run(args, new TestNotificationListener());     
        assertTrue(stats.getReturnCode() == ReturnCode.PARSE_EXCEPTION);
        assertEquals("Found unknown option (-y) at position 2", 
                stats.getErrorMessage());
    }

View Full Code Here

        w.println("store E into 'ee';");
        w.close(); 
        
        try {
            String[] args = { PIG_FILE };
            PigStats stats = PigRunner.run(args, null);             
            assertTrue(!stats.isSuccessful());            
            assertTrue(stats.getReturnCode() == ReturnCode.PARTIAL_FAILURE);
            assertTrue(stats.getJobGraph().size() == 2);
            JobStats job = (JobStats)stats.getJobGraph().getSources().get(0);
            assertTrue(job.isSuccessful());
            job = (JobStats)stats.getJobGraph().getSinks().get(0);
            assertTrue(!job.isSuccessful());
            assertTrue(stats.getOutputStats().size() == 3);
            for (OutputStats output : stats.getOutputStats()) {
                if (output.getName().equals("ee")) {
                    assertTrue(!output.isSuccessful());
                } else {
                    assertTrue(output.isSuccessful());
                }

View Full Code Here

        w1.println("store C into '" + OUTPUT_FILE + "';");
        w1.close();
        
        try {
            String[] args = { PIG_FILE };
            PigStats stats = PigRunner.run(args, new TestNotificationListener());
     
            assertTrue(stats.isSuccessful());
            
            assertEquals(1, stats.getNumberJobs());
            List<InputStats> inputs = stats.getInputStats();
            assertEquals(2, inputs.size());
            for (InputStats instats : inputs) {
                assertEquals(5, instats.getNumberRecords());
            }
        } finally {

View Full Code Here

    @Test
    public void testRegisterExternalJar() throws Exception {
        String[] args = { "-Dpig.additional.jars=pig-withouthadoop.jar",
                "-Dmapred.job.queue.name=default",
                "-e", "A = load '" + INPUT_FILE + "';store A into '" + OUTPUT_FILE + "';\n" };
        PigStats stats = PigRunner.run(args, new TestNotificationListener());        


        Util.deleteFile(cluster, OUTPUT_FILE);
        
        java.lang.reflect.Method getPigContext = stats.getClass()
                .getDeclaredMethod("getPigContext");


        getPigContext.setAccessible(true);


        PigContext ctx = (PigContext) getPigContext.invoke(stats);

View Full Code Here

        w.println("store A into '" + OUTPUT_FILE + "';");
        w.close();
        
        try {
            String[] args = { PIG_FILE };
            PigStats stats = PigRunner.run(args, new TestNotificationListener());     
            assertTrue(stats.isSuccessful());          
        } finally {
            new File(PIG_FILE).delete();
            Util.deleteFile(cluster, OUTPUT_FILE);
        }
    }

View Full Code Here

    if (CollectionUtils.isEmpty(execs) || contribution == null) {
      return;
    }


    for (ExecJob execJob : execs) {
      PigStats stats = execJob.getStatistics();


      // embedded pig contains no stats and further more throws Exceptions
      // use CDH3 compatible comparison
      if (stats != null && !stats.getClass().getName().contains("EmbeddedPigStats")) {
        // compute the input stats manually
        List<InputStats> inputStats = stats.getInputStats();


        for (InputStats is : inputStats) {
          for (int i = 0; i < safeLongToInt(is.getNumberRecords()); i++) {
            contribution.incrementReadCount();
          }
        }


        contribution.incrementWriteCount(safeLongToInt(stats.getRecordWritten()));


        // Skip information not available yet
        // workaround: query the internal map/reduce jobs ?
        //contribution.incrementReadSkipCount(safeLongToInt(count.getValue()));
        //

View Full Code Here

     *
     * @return list of jobs being executed
     * @throws IOException 
     */
    public List<ExecJob> executeBatch() throws IOException {
        PigStats stats = null;
        
        if( !isMultiQuery ) {
            // ignore if multiquery is off
            stats = PigStats.get();
        } else {

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.pig.tools.pigstats.PigStats

org.apache.hadoop.fs.Path

org.apache.hadoop.mapred.Counters

org.apache.hadoop.mapred.JobConf

org.apache.hadoop.mapred.JobID

org.apache.hadoop.mapred.RunningJob

org.apache.hive.hcatalog.pig.TestHCatLoader

org.apache.oozie.action.hadoop.PigMain

org.apache.pig.backend.hadoop.executionengine.HExecutionEngine

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.