Examples of org.apache.hadoop.examples.SleepJob

org.apache.hadoop.examples.SleepJob
Dummy class for testing MR framefork. Sleeps for a defined period of time in mapper and reducer. Generates fake input for map / reduce jobs. Note that generated number of input pairs is in the order of numMappers * mapSleepTime / 100, so the job uses some disk space.

    }
  }


  private void runSleepJob(JobConf conf) throws Exception {
    String[] args = { "-m", "3", "-r", "1", "-mt", "3000", "-rt", "1000" };
    ToolRunner.run(conf, new SleepJob(), args);
  }

View Full Code Here

    JobConf conf = new JobConf(miniMRCluster.createJobConf());
    conf.setMemoryForMapTask(PER_TASK_LIMIT);
    conf.setMemoryForReduceTask(PER_TASK_LIMIT);


    JobClient jClient = new JobClient(conf);
    SleepJob sleepJob = new SleepJob();
    sleepJob.setConf(conf);
    // Start the job
    RunningJob job =
        jClient.submitJob(sleepJob.setupJobConf(1, 1, 5000, 1, 1000, 1));
    boolean TTOverFlowMsgPresent = false;
    while (true) {
      List<TaskReport> allTaskReports = new ArrayList<TaskReport>();
      allTaskReports.addAll(Arrays.asList(jClient
          .getSetupTaskReports((org.apache.hadoop.mapred.JobID) job.getID())));

View Full Code Here

  @Test
  public void testFailedTaskJobStatus() throws IOException, 
          InterruptedException {
    conf = remoteJTClient.getDaemonConf();
    TaskInfo taskInfo = null;
    SleepJob job = new SleepJob();
    job.setConf(conf);
    JobConf jobConf = job.setupJobConf(1, 1, 10000, 4000, 100, 100);
    RunningJob runJob = jobClient.submitJob(jobConf);
    JobID jobId = runJob.getID();
    JobInfo jInfo = remoteJTClient.getJobInfo(jobId);
    Assert.assertTrue("Job has not been started for 1 min.", 
        jtClient.isJobStarted(jobId));

View Full Code Here

    String localTaskDir = null;
    TaskInfo taskInfo = null;
    TaskID tID = null;
    boolean isTempFolderExists = false;
    conf = remoteJTClient.getDaemonConf();
    SleepJob job = new SleepJob();
    job.setConf(conf);
    JobConf jobConf = job.setupJobConf(1, 0, 10000,100, 10, 10);
    RunningJob runJob = jobClient.submitJob(jobConf);
    JobID id = runJob.getID();
    JobInfo jInfo = remoteJTClient.getJobInfo(id);
    Assert.assertTrue("Job has not been started for 1 min.", 
       jtClient.isJobStarted(id));

View Full Code Here

  public void testAllTaskAttemptKill() throws Exception {
    Configuration conf = new Configuration(cluster.getConf());


    JobStatus[] jobStatus = null;


    SleepJob job = new SleepJob();
    job.setConf(conf);
    conf = job.setupJobConf(2, 1, 40000, 1000, 100, 100);
    JobConf jconf = new JobConf(conf);


    //Submitting the job
    RunningJob rJob = cluster.getJTClient().getClient().submitJob(jconf);

View Full Code Here


    //This counter will check for count of a loop,
    //which might become infinite.
    int count = 0;


    SleepJob job = new SleepJob();
    job.setConf(conf);
    int totalMapTasks = 5;
    int totalReduceTasks = 1;
    conf = job.setupJobConf(totalMapTasks, totalReduceTasks, 
      100, 100, 100, 100);
    JobConf jconf = new JobConf(conf);


    count = 0;
    //The last hour and last day are given 60 seconds and 120 seconds

View Full Code Here


    //This counter will check for count of a loop,
    //which might become infinite.
    int count = 0;


    SleepJob job = new SleepJob();
    job.setConf(conf);
    int totalMapTasks = 5;
    int totalReduceTasks = 1;
    conf = job.setupJobConf(totalMapTasks, totalReduceTasks, 
      100, 100, 100, 100);
    JobConf jconf = new JobConf(conf);


    count = 0;
    //The last hour and last day are given 60 seconds and 120 seconds

View Full Code Here

    //object is removed by checking the getJobInfo. In the
    //second iteration, verify if a killed job JobInProgress
    //is removed.
    do {
      testLoopCount++;
      SleepJob job = new SleepJob();
      job.setConf(conf);
      conf = job.setupJobConf(5, 1, 100, 100, 100, 100);
      //Get the value of mapred.jobtracker.retirejob.check. If not
      //found then use 60000 milliseconds, which is the application default.
      retiredJobInterval =
        conf.getInt("mapred.jobtracker.retirejob.check", 60000);
      //Assert if retiredJobInterval is 0

View Full Code Here

  }
  
  @Test
  public void testJobSubmission() throws Exception {
    Configuration conf = new Configuration(cluster.getConf());
    SleepJob job = new SleepJob();
    job.setConf(conf);
    conf = job.setupJobConf(1, 1, 100, 100, 100, 100);
    RunningJob rJob = cluster.getJTClient().submitAndVerifyJob(conf);
    cluster.getJTClient().verifyJobHistory(rJob.getID());
  }

View Full Code Here

  @Test
  public void testTaskDetails() throws Exception {
    Configuration conf = new Configuration(cluster.getConf());
    JTProtocol wovenClient = cluster.getJTClient().getProxy();
    FinishTaskControlAction.configureControlActionForJob(conf);
    SleepJob job = new SleepJob();
    job.setConf(conf);


    conf = job.setupJobConf(1, 1, 100, 100, 100, 100);
    JobClient client = cluster.getJTClient().getClient();


    RunningJob rJob = client.submitJob(new JobConf(conf));
    JobID id = rJob.getID();

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.examples.SleepJob

org.apache.hadoop.conf.Configuration

org.apache.hadoop.corona.TestCompletionEvents

org.apache.hadoop.corona.TestMiniCoronaRunJob

org.apache.hadoop.corona.TestMiniCoronaSpeculativeTask

org.apache.hadoop.corona.TestRJTFailover

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.Path

org.apache.hadoop.io.IntWritable

org.apache.hadoop.mapred.HighRamJobHelper

org.apache.hadoop.mapred.JobConf

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.