Examples of org.apache.hadoop.examples.SleepJob

org.apache.hadoop.examples.SleepJob
Dummy class for testing MR framefork. Sleeps for a defined period of time in mapper and reducer. Generates fake input for map / reduce jobs. Note that generated number of input pairs is in the order of numMappers * mapSleepTime / 100, so the job uses some disk space.

                                    String queueName) 
                                      throws IOException {
    JobConf clientConf = new JobConf();
    clientConf.set("mapred.job.tracker", "localhost:"
        + miniMRCluster.getJobTrackerPort());
    SleepJob job = new SleepJob();
    job.setConf(clientConf);
    clientConf = job.setupJobConf(numMappers, numReducers, 
        mapSleepTime, (int)mapSleepTime/100,
        reduceSleepTime, (int)reduceSleepTime/100);
    if (queueName != null) {
      clientConf.setQueueName(queueName);
    }

View Full Code Here

                                    String queueName) 
                                      throws IOException {
    JobConf clientConf = new JobConf();
    clientConf.set("mapred.job.tracker", "localhost:"
        + miniMRCluster.getJobTrackerPort());
    SleepJob job = new SleepJob();
    job.setConf(clientConf);
    clientConf = job.setupJobConf(numMappers, numReducers, 
        mapSleepTime, (int)mapSleepTime/100,
        reduceSleepTime, (int)reduceSleepTime/100);
    if (queueName != null) {
      clientConf.setQueueName(queueName);
    }

View Full Code Here

  private void launchSleepJob(String mapSleepTime, String reduceSleepTime,
                              String mapTaskCount, String reduceTaskCount,
                              Configuration conf) throws Exception {
    String[] args = { "-m", mapTaskCount, "-r", reduceTaskCount,
                      "-mt", mapSleepTime, "-rt", reduceSleepTime };
    ToolRunner.run(conf, new SleepJob(), args);
  }

View Full Code Here

                         "-m", "1", 
                         "-r", "1", 
                         "-mt", "1000", 
                         "-rt", "1000",
                         "-recordt","100"};
      SleepJob job = new SleepJob();
      JobConf jobConf = new JobConf(conf); 
      int exitStatus = ToolRunner.run(jobConf, job, jobArgs);
      Assert.assertEquals("Exit Code:", 0, exitStatus);
      UtilsForTests.waitFor(100);
      JobClient jobClient = jtClient.getClient();

View Full Code Here

   * @return the job id of the high ram job
   * @throws Exception is thrown when the method fails to run the high ram job
   */
  public JobID runHighRamJob (Configuration conf, JobClient jobClient, 
      JTProtocol remoteJTClient,String assertMessage) throws Exception {
    SleepJob job = new SleepJob();
    String jobArgs []= {"-D","mapred.cluster.max.map.memory.mb=2048", 
                        "-D","mapred.cluster.max.reduce.memory.mb=2048", 
                        "-D","mapred.cluster.map.memory.mb=1024", 
                        "-D","mapreduce.job.complete.cancel.delegation.tokens=false",
                        "-D","mapred.cluster.reduce.memory.mb=1024",

View Full Code Here

     }
  }


  
  public JobID runSleepJob(boolean signalJob) throws Exception{
    SleepJob job = new SleepJob();
    job.setConf(conf);
    conf = job.setupJobConf(5, 1, 100, 5, 100, 5);
    JobConf jconf = new JobConf(conf);
    //Controls the job till all verification is done 
    FinishTaskControlAction.configureControlActionForJob(conf);
    //Submitting the job
    RunningJob rJob = cluster.getJTClient().getClient().submitJob(jconf);

View Full Code Here

    Assert.assertEquals("Job has not been failed...", 
            JobStatus.SUCCEEDED, jStatus.getRunState());
  }


  private void setupJobAndRun() throws IOException { 
    SleepJob job = new SleepJob();
    job.setConf(conf);
    conf = job.setupJobConf(3, 1, 60000, 100, 60000, 100);
    JobConf jobConf = new JobConf(conf);
    cleanup(outputDir, conf);
    jtClient = cluster.getJTClient();
    JobClient client = jtClient.getClient();
    wovenClient = cluster.getJTClient().getProxy();

View Full Code Here


    startCluster(conf);


    JobTracker jobtracker = mr.getJobTrackerRunner().getJobTracker();


    SleepJob job = new SleepJob();
    job.setConf(mr.createJobConf());
    JobConf job1 = job.setupJobConf(1, 0, 1, 1, 1, 1);
    JobClient jc = new JobClient(job1);
    RunningJob rJob1 = jc.submitJob(job1);
    LOG.info("Submitted first job " + rJob1.getID());


    TestJobTrackerInstrumentation.finalizeCall.await();

View Full Code Here

  }


  // starts a sleep job with 1 map task that runs for a long time
  private RunningJob startSleepJobAsUser(String user, final JobConf conf,
      MiniMRCluster cluster) throws Exception {
  final SleepJob sleepJob = new SleepJob();
    sleepJob.setConf(conf);
    UserGroupInformation jobSubmitterUGI = 
        UserGroupInformation.createRemoteUser(user);
    RunningJob job =
        jobSubmitterUGI.doAs(new PrivilegedExceptionAction<RunningJob>() {
        public RunningJob run() throws Exception {
          JobClient jobClient = new JobClient(conf);
          SleepJob sleepJob = new SleepJob();
          sleepJob.setConf(conf);
          JobConf jobConf =
              sleepJob.setupJobConf(1, 0, 900000, 1, 1000, 1000);
          RunningJob runningJob = jobClient.submitJob(jobConf);
          return runningJob;
        }
      });
    JobID jobId = job.getID();

View Full Code Here


    // Let us add group1 and group3 to modify-job-acl. So modifyColleague and
    // viewAndModifyColleague will be able to modify the job
    conf.set(JobContext.JOB_ACL_MODIFY_JOB, " group1,group3");


    final SleepJob sleepJob = new SleepJob();
    final JobConf jobConf = new JobConf(conf);
    sleepJob.setConf(jobConf);
    UserGroupInformation jobSubmitterUGI =
        UserGroupInformation.createRemoteUser(jobSubmitter);
    RunningJob job =
        jobSubmitterUGI.doAs(new PrivilegedExceptionAction<RunningJob>() {
          public RunningJob run() throws Exception {
            JobClient jobClient = new JobClient(jobConf);
            SleepJob sleepJob = new SleepJob();
            sleepJob.setConf(jobConf);
            JobConf jobConf =
                sleepJob.setupJobConf(1, 0, 1000, 1, 1000, 1000);
            RunningJob runningJob = jobClient.runJob(jobConf);
            return runningJob;
          }
        });

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.examples.SleepJob

org.apache.hadoop.conf.Configuration

org.apache.hadoop.corona.TestCompletionEvents

org.apache.hadoop.corona.TestMiniCoronaRunJob

org.apache.hadoop.corona.TestMiniCoronaSpeculativeTask

org.apache.hadoop.corona.TestRJTFailover

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.Path

org.apache.hadoop.io.IntWritable

org.apache.hadoop.mapred.HighRamJobHelper

org.apache.hadoop.mapred.JobConf

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.