Examples of org.apache.hadoop.mapreduce.Job.waitForCompletion()

org.apache.hadoop.mapreduce.Job.waitForCompletion()
Submit the job to the cluster and wait for it to finish. @param verbose print the progress to the user @return true if the job succeeded @throws IOException thrown if the communication with the JobTracker is lost

    }, otherArgs);
    if (job == null) {
      System.exit(1);
    }


    boolean waitForCompletion = job.waitForCompletion(true);
    System.exit(waitForCompletion ? 0 : 1);
  }


  public static Job setupJob(Configuration configuration, ControllerPool controllerPool, String... otherArgs)
      throws Exception {

View Full Code Here

    if (otherArgs.length < 2) {
      usage("Wrong number of arguments: " + otherArgs.length);
      System.exit(-1);
    }
    Job job = createSubmittableJob(conf, otherArgs);
    System.exit(job.waitForCompletion(true) ? 0 : 1);
  }
}

View Full Code Here

    TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
      org.apache.hadoop.hbase.util.Bytes.class);


    TableMapReduceUtil.initCredentials(job);


    job.waitForCompletion(true);
  }


  /*
   * Write input file of offsets-per-client for the mapreduce job.
   * @param c Configuration

View Full Code Here

      TableMapReduceUtil.initTableReducerJob(
        Bytes.toString(table.getTableName()),
        IdentityTableReducer.class, job);
      FileOutputFormat.setOutputPath(job, new Path("test"));
      LOG.info("Started " + Bytes.toString(table.getTableName()));
      assertTrue(job.waitForCompletion(true));
      LOG.info("After map/reduce completion");


      // verify map-reduce results
      verify(Bytes.toString(table.getTableName()));
    } finally {

View Full Code Here

      job.setMapperClass(Mapper.class); //identity mapper


      FileOutputFormat.setOutputPath(job, tmpOutput);
      job.setOutputFormatClass(SequenceFileOutputFormat.class);


      boolean success = job.waitForCompletion(true);


      return success ? 0 : 1;
    }


    public int runGenerator(int numMappers, long numNodes, Path tmpOutput,

View Full Code Here


      job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false);
      TableMapReduceUtil.addDependencyJars(job);
      TableMapReduceUtil.initCredentials(job);


      boolean success = job.waitForCompletion(true);


      return success ? 0 : 1;
    }


    public int run(int numMappers, long numNodes, Path tmpOutput,

View Full Code Here

          false, tmpDir);
      job.setReducerClass(TestTableInputFormatScanBase.ScanReducer.class);
      job.setNumReduceTasks(1); // one to get final "first" and "last" key
      FileOutputFormat.setOutputPath(job, new Path(job.getJobName()));
      LOG.info("Started " + job.getJobName());
      assertTrue(job.waitForCompletion(true));
      LOG.info("After map/reduce completion - job " + jobName);
    } finally {
      fs.delete(tmpDir, true);
    }

View Full Code Here

    job.setMapOutputValueClass(ImmutableBytesWritable.class);
    job.setInputFormatClass(TableInputFormat.class);
    job.setNumReduceTasks(1);
    FileOutputFormat.setOutputPath(job, new Path(job.getJobName()));
    TableMapReduceUtil.addDependencyJars(job);
    assertTrue(job.waitForCompletion(true));
  }


  /**
   * Tests a MR scan using specific start and stop rows.
   *

View Full Code Here

      ImmutableBytesWritable.class, ImmutableBytesWritable.class, job);
    job.setReducerClass(ScanReducer.class);
    job.setNumReduceTasks(1); // one to get final "first" and "last" key
    FileOutputFormat.setOutputPath(job, new Path(job.getJobName()));
    LOG.info("Started " + job.getJobName());
    assertTrue(job.waitForCompletion(true));
    LOG.info("After map/reduce completion - job " + jobName);
  }


}

View Full Code Here

    job.setReducerClass(KeyValueSortReducer.class);
    job.setOutputFormatClass(HFileOutputFormat.class);
    job.setNumReduceTasks(4);


    FileOutputFormat.setOutputPath(job, testDir);
    assertTrue(job.waitForCompletion(false));
    FileStatus [] files = fs.listStatus(testDir);
    assertTrue(files.length > 0);
  }


  @Test

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.