Examples of org.apache.hadoop.examples.SleepJob

org.apache.hadoop.examples.SleepJob
Dummy class for testing MR framefork. Sleeps for a defined period of time in mapper and reducer. Generates fake input for map / reduce jobs. Note that generated number of input pairs is in the order of numMappers * mapSleepTime / 100, so the job uses some disk space.


    // Let us add group1 and group3 to modify-job-acl. So modifyColleague and
    // viewAndModifyColleague will be able to modify the job
    conf.set(JobContext.JOB_ACL_MODIFY_JOB, " group1,group3");


    final SleepJob sleepJob = new SleepJob();
    final JobConf jobConf = new JobConf(conf);
    sleepJob.setConf(jobConf);
    UserGroupInformation jobSubmitterUGI =
        UserGroupInformation.createRemoteUser(jobSubmitter);
    RunningJob job =
        jobSubmitterUGI.doAs(new PrivilegedExceptionAction<RunningJob>() {
          public RunningJob run() throws Exception {
            JobClient jobClient = new JobClient(jobConf);
            SleepJob sleepJob = new SleepJob();
            sleepJob.setConf(jobConf);
            JobConf jobConf =
                sleepJob.setupJobConf(1, 0, 1000, 1, 1000, 1000);
            RunningJob runningJob = jobClient.runJob(jobConf);
            return runningJob;
          }
        });

View Full Code Here

                                      throws IOException, InterruptedException {
    JobConf clientConf = new JobConf();
    clientConf.set("mapred.job.tracker", "localhost:"
        + miniMRCluster.getJobTrackerPort());
    UserGroupInformation ugi;
    SleepJob job = new SleepJob();
    job.setConf(clientConf);
    clientConf = job.setupJobConf(numMappers, numReducers, 
        mapSleepTime, (int)mapSleepTime/100,
        reduceSleepTime, (int)reduceSleepTime/100);
    if (queueName != null) {
      clientConf.setQueueName(queueName);
    }

View Full Code Here


        // Run a job with jvm reuse
        JobConf myConf = getClusterConf();
        myConf.set("mapred.job.reuse.jvm.num.tasks", "-1");
        String[] args = { "-m", "6", "-r", "3", "-mt", "1000", "-rt", "1000" };
        assertEquals(0, ToolRunner.run(myConf, new SleepJob(), args));
        return null;
      }
    });
  }

View Full Code Here

      MyTaskTrackerMetricsInst instr = (MyTaskTrackerMetricsInst)
        mr.getTaskTrackerRunner(0).getTaskTracker()
        .getTaskTrackerInstrumentation();


      JobConf conf = mr.createJobConf();
      SleepJob job = new SleepJob();
      job.setConf(conf);
      int numMapTasks = 3;
      int numReduceTasks = 2;
      job.run(numMapTasks, numReduceTasks, 1, 1, 1, 1);


      synchronized (instr) {
        // 5 regular tasks + 2 setup/cleanup tasks.
        assertEquals(7, instr.complete);
        assertEquals(7, instr.end);

View Full Code Here

  //got a KillJobAction).
  private static final Log LOG =
        LogFactory.getLog(TestEmptyJob.class.getName());
  private void runSleepJob(JobConf conf) throws Exception {
    String[] args = { "-m", "1", "-r", "10", "-mt", "1000", "-rt", "10000" };
    ToolRunner.run(conf, new SleepJob(), args);
  }

View Full Code Here

      mr = new MiniMRCluster(2, "file:///", 3, null, null, jtConf);
      MyJobTrackerMetricsInst instr = (MyJobTrackerMetricsInst) 
        mr.getJobTrackerRunner().getJobTracker().getInstrumentation();


      JobConf conf = mr.createJobConf();
      SleepJob job = new SleepJob();
      job.setConf(conf);
      int numMapTasks = 3;
      int numReduceTasks = 2;
      job.run(numMapTasks, numReduceTasks, 10000, 1, 10000, 1);
      
      synchronized (instr) {
        //after the job completes, incr and decr should be equal
        assertEquals(instr.incrOccupiedMapSlots, 
            instr.decrOccupiedMapSlots);

View Full Code Here

    private String launchSleepJob(int sleep) throws Exception {
        JobConf jobConf = Services.get().get(HadoopAccessorService.class)
                .createJobConf(new URI(getNameNodeUri()).getAuthority());
        JobClient jobClient = createJobClient();


        SleepJob sleepjob = new SleepJob();
        sleepjob.setConf(jobConf);
        jobConf = sleepjob.setupJobConf(1, 1, sleep, 1, sleep, 1);


        final RunningJob runningJob = jobClient.submitJob(jobConf);
        return runningJob.getID().toString();
    }

View Full Code Here

    startCluster(1, clusterProps, schedulerProps);
    JobConf conf = getJobConf();
    conf.setSpeculativeExecution(false);
    conf.set("mapred.committer.job.setup.cleanup.needed", "false");
    conf.setNumTasksToExecutePerJvm(-1);
    SleepJob sleepJob = new SleepJob();
    sleepJob.setConf(conf);
    JobConf job = sleepJob.setupJobConf(3, 3, 1, 1, 1, 1);
    RunningJob rjob;
    try {
      rjob = runJob(job, false);
      fail("The job should have thrown Exception");
    } catch (Exception e) {

View Full Code Here

    JobConf conf = getJobConf();
    conf.setSpeculativeExecution(false);
    conf.set("mapred.committer.job.setup.cleanup.needed", "false");
    conf.setNumTasksToExecutePerJvm(-1);
    conf.setQueueName(queues[0]);
    SleepJob sleepJob1 = new SleepJob();
    sleepJob1.setConf(conf);
    JobConf sleepJobConf = sleepJob1.setupJobConf(1, 1, 1, 1, 1, 1);
    jobs[0] = runJob(sleepJobConf, true);


    JobConf conf2 = getJobConf();
    conf2.setSpeculativeExecution(false);
    conf2.set("mapred.committer.job.setup.cleanup.needed", "false");
    conf2.setNumTasksToExecutePerJvm(-1);
    conf2.setQueueName(queues[1]);
    SleepJob sleepJob2 = new SleepJob();
    sleepJob2.setConf(conf2);
    JobConf sleep2 = sleepJob2.setupJobConf(3, 3, 5, 3, 5, 3);
    jobs[1] = runJob(sleep2, false);
    assertTrue("Sleep job submitted to queue 1 is not successful", jobs[0]
        .isSuccessful());
    assertTrue("Sleep job submitted to queue 2 is not successful", jobs[1]
        .isSuccessful());

View Full Code Here

                                      throws IOException, InterruptedException {
    JobConf clientConf = new JobConf();
    clientConf.set("mapred.job.tracker", "localhost:"
        + miniMRCluster.getJobTrackerPort());
    UserGroupInformation ugi;
    SleepJob job = new SleepJob();
    job.setConf(clientConf);
    clientConf = job.setupJobConf(numMappers, numReducers, 
        mapSleepTime, (int)mapSleepTime/100,
        reduceSleepTime, (int)reduceSleepTime/100);
    if (queueName != null) {
      clientConf.setQueueName(queueName);
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.examples.SleepJob

org.apache.hadoop.conf.Configuration

org.apache.hadoop.corona.TestCompletionEvents

org.apache.hadoop.corona.TestMiniCoronaRunJob

org.apache.hadoop.corona.TestMiniCoronaSpeculativeTask

org.apache.hadoop.corona.TestRJTFailover

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.Path

org.apache.hadoop.io.IntWritable

org.apache.hadoop.mapred.HighRamJobHelper

org.apache.hadoop.mapred.JobConf

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.