Examples of org.apache.hadoop.mapred.JobClient

org.apache.hadoop.mapred.JobClient
te a new JobConf JobConf job = new JobConf(new Configuration(), MyJob.class); // Specify various job-specific parameters job.setJobName("myjob"); job.setInputPath(new Path("in")); job.setOutputPath(new Path("out")); job.setMapperClass(MyJob.MyMapper.class); job.setReducerClass(MyJob.MyReducer.class); // Submit the job, then poll for progress until the job is complete JobClient.runJob(job);

Job Control

At times clients would chain map-reduce jobs to accomplish complex tasks which cannot be done via a single map-reduce job. This is fairly easy since the output of the job, typically, goes to distributed file-system and that can be used as the input for the next job.

However, this also means that the onus on ensuring jobs are complete (success/failure) lies squarely on the clients. In such situations the various job-control options are:
1. {@link #runJob(JobConf)} : submits the job and returns only after the job has completed.
2. {@link #submitJob(JobConf)} : only submits the job, then poll the returned handle to the {@link RunningJob} to query status and make scheduling decisions.
3. {@link JobConf#setJobEndNotificationURI(String)} : setup a notificationon job-completion, thus avoiding polling.
@see JobConf @see ClusterStatus @see Tool @see DistributedCache

        ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(job, action, false, false);
        MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor();
        JobConf conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action.getConf()));
        String user = conf.get("user.name");
        String group = conf.get("group.name");
        JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);


        String launcherId = action.getExternalId();


        final RunningJob launcherJob = jobClient.getJob(JobID.forName(launcherId));


        waitFor(240 * 1000, new Predicate() {
            public boolean evaluate() throws Exception {
                return launcherJob.isComplete();
            }

View Full Code Here

        }
        String jobTracker = conf.get("mapred.job.tracker");
        validateJobTracker(jobTracker);
        try {
            UserGroupInformation ugi = getUGI(user);
            JobClient jobClient = ugi.doAs(new PrivilegedExceptionAction<JobClient>() {
                public JobClient run() throws Exception {
                    return new JobClient(conf);
                }
            });
            Token<DelegationTokenIdentifier> mrdt = jobClient.getDelegationToken(new Text("mr token"));
            conf.getCredentials().addToken(new Text("mr token"), mrdt);
            return jobClient;
        }
        catch (InterruptedException ex) {
            throw new HadoopAccessorException(ErrorCode.E0902, ex);

View Full Code Here


        JobConf jobConf = Services.get().get(HadoopAccessorService.class).createJobConf(jobTracker);
        XConfiguration.copy(conf, jobConf);
        String user = jobConf.get("user.name");
        String group = jobConf.get("group.name");
        JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, jobConf);
        final RunningJob runningJob = jobClient.getJob(JobID.forName(jobId));
        assertNotNull(runningJob);
        return runningJob;
    }

View Full Code Here

   * @param job
   * @return true for success
   * @throws IOException
   */
  public static boolean runJob(JobConf job) throws IOException {
    JobClient jc = new JobClient(job);
    boolean sucess = true;
    RunningJob running = null;
    try {
      running = jc.submitJob(job);
      String jobId = running.getJobID();
      System.out.println("Job " + jobId + " is submitted");
      while (!running.isComplete()) {
        System.out.println("Job " + jobId + " is still running.");
        try {
          Thread.sleep(60000);
        } catch (InterruptedException e) {
        }
        running = jc.getJob(jobId);
      }
      sucess = running.isSuccessful();
    } finally {
      if (!sucess && (running != null)) {
        running.killJob();
      }
      jc.close();
    }
    return sucess;
  }

View Full Code Here

      throws IOException {
    // from RandomWriter
    job.setInputFormat(RandomInputFormat.class);
    job.setMapperClass(RandomMapOutput.class);


    final ClusterStatus cluster = new JobClient(job).getClusterStatus();
    int numMapsPerHost = job.getInt("test.randomtextwrite.maps_per_host", 10);
    long numBytesToWritePerMap =
      job.getLong("test.randomtextwrite.bytes_per_map", 1*1024*1024*1024);
    if (numBytesToWritePerMap == 0) {
      throw new IOException(

View Full Code Here

  }


  public static Token<org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier> getJobTrackerDelegationToken(
    Configuration conf, String userName) throws Exception {
    // LOG.info("getJobTrackerDelegationToken("+conf+","+userName+")");
    JobClient jcl = new JobClient(new JobConf(conf, HCatOutputFormat.class));
    Token<org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier> t = jcl
      .getDelegationToken(new Text(userName));
    // LOG.info("got "+t);
    return t;


    // return null;

View Full Code Here

  }


  public static Token<org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier> getJobTrackerDelegationToken(
    Configuration conf, String userName) throws Exception {
    // LOG.info("getJobTrackerDelegationToken("+conf+","+userName+")");
    JobClient jcl = new JobClient(new JobConf(conf, HCatOutputFormat.class));
    Token<org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier> t = jcl
      .getDelegationToken(new Text(userName));
    // LOG.info("got "+t);
    return t;


    // return null;

View Full Code Here

      // logs
      String pwd = HiveConf.getVar(job, HiveConf.ConfVars.METASTOREPWD);
      if (pwd != null) {
        HiveConf.setVar(job, HiveConf.ConfVars.METASTOREPWD, "HIVE");
      }
      JobClient jc = new JobClient(job);


      String addedJars = Utilities.getResourceFiles(job, SessionState.ResourceType.JAR);
      if (!addedJars.isEmpty()) {
        job.set("tmpjars", addedJars);
      }


      // make this client wait if job trcker is not behaving well.
      Throttle.checkJobTracker(job, LOG);


      // Finally SUBMIT the JOB!
      rj = jc.submitJob(job);


      returnVal = jobExecHelper.progress(rj, jc);
      success = (returnVal == 0);


    } catch (Exception e) {

View Full Code Here

   */
  public ClusterStatus getClusterStatus() throws Exception {
    ClusterStatus cs;
    try {
      JobConf job = new JobConf(conf);
      JobClient jc = new JobClient(job);
      cs = jc.getClusterStatus();
    } catch (Exception e) {
      e.printStackTrace();
      throw e;
    }
    LOG.info("Returning cluster status: " + cs.toString());

View Full Code Here


  private void addJTDelegationToken(JobConf job) throws IOException {
    // Get jobTracker delegation token if security is enabled
    // we need to launch the ImportSequenceFile job
    if (User.isSecurityEnabled()) {
      JobClient jobClient = new JobClient(new JobConf(job));
      try {
        job.getCredentials().addToken(new Text("my mr token"),
          jobClient.getDelegationToken(null));
      } catch (InterruptedException e) {
        throw new IOException("Error while getting JT delegation token", e);
      }
    }
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapred.JobClient

com.atlantbh.jmeter.plugins.hadooputilities.jobstatistics.JobLayer

com.atlantbh.jmeter.plugins.hadooputilities.jobstatistics.TaskLayer

com.cloudera.circus.test.TestXTest

com.cloudera.lib.service.hadoop.HadoopService

com.cloudera.lib.service.hadoop.TestHadoopService

com.fb2pdf.hadoop.cluster.MeanShiftCanopyDriver

com.ikanow.infinit.e.processing.custom.CustomProcessingController

com.linkedin.camus.etl.kafka.CamusJob

com.netflix.lipstick.pigtolipstick.BasicP2LClient

com.netflix.lipstick.warnings.JobWarnings

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.

Examples of org.apache.hadoop.mapred.JobClient

Job Control

Related Classes of org.apache.hadoop.mapred.JobClient