Examples of org.apache.hadoop.mapreduce.SleepJob$SleepMapper

org.apache.hadoop.mapreduce.SleepJob
Dummy class for testing MR framefork. Sleeps for a defined period of time in mapper and reducer. Generates fake input for map / reduce jobs. Note that generated number of input pairs is in the order of numMappers * mapSleepTime / 100, so the job uses some disk space.

     root.getChildren().get(0).getQueueSchedulingContext().setCapacityPercent(100);


    JobConf conf = getJobConf();
    conf.setSpeculativeExecution(false);
    conf.setNumTasksToExecutePerJvm(-1);
    SleepJob sleepJob = new SleepJob();
    sleepJob.setConf(conf);
    Job job = sleepJob.createJob(3, 3, 1, 1, 1, 1);
    job.waitForCompletion(false);
    assertFalse(
      "The submitted job successfully completed",
      job.isSuccessful());

View Full Code Here

    JobConf conf = getJobConf();
    conf.setSpeculativeExecution(false);
    conf.set(MRJobConfig.SETUP_CLEANUP_NEEDED, "false");
    conf.setNumTasksToExecutePerJvm(-1);
    conf.setQueueName(queues[0]);
    SleepJob sleepJob1 = new SleepJob();
    sleepJob1.setConf(conf);
    jobs[0] = sleepJob1.createJob(1, 1, 1, 1, 1, 1);
    jobs[0].submit();


    JobConf conf2 = getJobConf();
    conf2.setSpeculativeExecution(false);
    conf2.setNumTasksToExecutePerJvm(-1);
    conf2.setQueueName(queues[1]);
    SleepJob sleepJob2 = new SleepJob();
    sleepJob2.setConf(conf2);
    jobs[1] = sleepJob2.createJob(3, 3, 5, 3, 5, 3);
    jobs[1].waitForCompletion(false);
    assertTrue(
      "Sleep job submitted to queue 1 is not successful", jobs[0]
        .isSuccessful());
    assertTrue(

View Full Code Here


        // Run a job with jvm reuse
        JobConf myConf = getClusterConf();
        myConf.set(JobContext.JVM_NUMTASKS_TORUN, "-1");
        String[] args = { "-m", "6", "-r", "3", "-mt", "1000", "-rt", "1000" };
        assertEquals(0, ToolRunner.run(myConf, new SleepJob(), args)); 
        return null;
      }
    });


  }

View Full Code Here

    jobOwner.doAs(new PrivilegedExceptionAction<Object>() {
      public Object run() throws Exception {
        JobConf conf = getClusterConf();
        conf.setInt(JobContext.TASK_TIMEOUT, 10000);
        conf.setInt(Job.COMPLETION_POLL_INTERVAL_KEY, 50);
        SleepJob sleepJob = new SleepJob();
        sleepJob.setConf(conf);
        Job job = sleepJob.createJob(1, 0, 30000, 1, 0, 0);
        job.setMaxMapAttempts(1);
        int prevNumSigQuits = MyLinuxTaskController.attemptedSigQuits;
        job.waitForCompletion(true);
        assertTrue("Did not detect a new SIGQUIT!",
            prevNumSigQuits < MyLinuxTaskController.attemptedSigQuits);

View Full Code Here

    }
  }


  private int runSleepJob(JobConf conf) throws Exception {
    String[] args = { "-m", "3", "-r", "1", "-mt", "3000", "-rt", "1000" };
    return ToolRunner.run(conf, new SleepJob(), args);
  }

View Full Code Here

    JobConf conf = new JobConf(miniMRCluster.createJobConf());
    conf.setMemoryForMapTask(PER_TASK_LIMIT);
    conf.setMemoryForReduceTask(PER_TASK_LIMIT);


    JobClient jClient = new JobClient(conf);
    SleepJob sleepJob = new SleepJob();
    sleepJob.setConf(conf);
    // Start the job
    Job job = sleepJob.createJob(1, 1, 5000, 1, 1000, 1);
    job.submit();
    boolean TTOverFlowMsgPresent = false;
    while (true) {
      List<TaskReport> allTaskReports = new ArrayList<TaskReport>();
      allTaskReports.addAll(Arrays.asList(jClient

View Full Code Here

    JobConf conf = new JobConf(miniMRCluster.createJobConf());
    // Set per task physical memory limits to be a higher value
    conf.setLong(MRJobConfig.MAP_MEMORY_PHYSICAL_MB, 2 * 1024L);
    conf.setLong(MRJobConfig.REDUCE_MEMORY_PHYSICAL_MB, 2 * 1024L);
    JobClient jClient = new JobClient(conf);
    SleepJob sleepJob = new SleepJob();
    sleepJob.setConf(conf);
    // Start the job
    Job job = sleepJob.createJob(1, 1, 100000, 1, 100000, 1);
    job.submit();
    boolean TTOverFlowMsgPresent = false;
    while (true) {
      List<TaskReport> allTaskReports = new ArrayList<TaskReport>();
      allTaskReports.addAll(Arrays.asList(jClient

View Full Code Here

      IOException {
    String[] args = { "-m", "0", "-r", "0", "-mt", "0", "-rt", "0" };
    boolean throwsException = false;
    String msg = null;
    try {
      ToolRunner.run(jobConf, new SleepJob(), args);
    } catch (RemoteException re) {
      throwsException = true;
      msg = re.unwrapRemoteException().getMessage();
    }
    assertTrue(throwsException);

View Full Code Here

      Path inDir = new Path(testDir, "in-1");
      Path outDir = new Path(testDir, "out-1");
      JobConf jConf = mr.createJobConf();
      FileInputFormat.setInputPaths(jConf, new Path[] {inDir});
      FileOutputFormat.setOutputPath(jConf, outDir);
      SleepJob sleepJob = new SleepJob();
      sleepJob.setConf(jConf);
      Job job = sleepJob.createJob(1, 1, 0, 1, 0, 1);


      job.submit();
      JobID id = JobID.downgrade(job.getStatus().getJobID());
      JobInProgress jip = jobtracker.getJob(id);

View Full Code Here

      IOException {
    String[] args = { "-m", "0", "-r", "0", "-mt", "0", "-rt", "0" };
    boolean throwsException = false;
    String msg = null;
    try {
      ToolRunner.run(jobConf, new SleepJob(), args);
    } catch (RemoteException re) {
      throwsException = true;
      msg = re.unwrapRemoteException().getMessage();
    }
    assertTrue(throwsException);

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.apache.hadoop.mapreduce.SleepJob$SleepMapper

org.apache.hadoop.conf.Configuration

org.apache.hadoop.fs.Path

org.apache.hadoop.io.IntWritable

org.apache.hadoop.mapred.QueueManagerTestUtils

org.apache.hadoop.mapred.TestCapacitySchedulerWithJobTracker

org.apache.hadoop.mapred.TestCluster

org.apache.hadoop.mapred.TestControlledJob

org.apache.hadoop.mapred.TestDistributedCacheModifiedFile

org.apache.hadoop.mapred.TestDistributedCachePrivateFile

org.apache.hadoop.mapred.TestDistributedCacheUnModifiedFile

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.