Examples of org.apache.hadoop.mapred.JobClient

org.apache.hadoop.mapred.JobClient
te a new JobConf JobConf job = new JobConf(new Configuration(), MyJob.class); // Specify various job-specific parameters job.setJobName("myjob"); job.setInputPath(new Path("in")); job.setOutputPath(new Path("out")); job.setMapperClass(MyJob.MyMapper.class); job.setReducerClass(MyJob.MyReducer.class); // Submit the job, then poll for progress until the job is complete JobClient.runJob(job);

Job Control

At times clients would chain map-reduce jobs to accomplish complex tasks which cannot be done via a single map-reduce job. This is fairly easy since the output of the job, typically, goes to distributed file-system and that can be used as the input for the next job.

However, this also means that the onus on ensuring jobs are complete (success/failure) lies squarely on the clients. In such situations the various job-control options are:
1. {@link #runJob(JobConf)} : submits the job and returns only after the job has completed.
2. {@link #submitJob(JobConf)} : only submits the job, then poll the returned handle to the {@link RunningJob} to query status and make scheduling decisions.
3. {@link JobConf#setJobEndNotificationURI(String)} : setup a notificationon job-completion, thus avoiding polling.
@see JobConf @see ClusterStatus @see Tool @see DistributedCache

                                         boolean sequentialAccess) throws IOException {
    if (normPower != NO_NORMALIZING && normPower < 0) {
      throw new IllegalArgumentException("normPower must either be -1 or >= 0");
    }
    
    Configurable client = new JobClient();
    JobConf conf = new JobConf(PartialVectorMerger.class);
    conf.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
                                  + "org.apache.hadoop.io.serializer.WritableSerialization");
    // this conf parameter needs to be set enable serialisation of conf values
    conf.setJobName("PartialVectorMerger::MergePartialVectors");
    conf.setBoolean(SEQUENTIAL_ACCESS, sequentialAccess);
    conf.setInt(DIMENSION, dimension);
    conf.setFloat(NORMALIZATION_POWER, normPower);
    
    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(VectorWritable.class);
    
    FileInputFormat.setInputPaths(conf, getCommaSeparatedPaths(partialVectorPaths));
    
    Path outputPath = new Path(output);
    FileOutputFormat.setOutputPath(conf, outputPath);
    
    conf.setMapperClass(IdentityMapper.class);
    conf.setInputFormat(SequenceFileInputFormat.class);
    conf.setReducerClass(PartialVectorMergeReducer.class);
    conf.setOutputFormat(SequenceFileOutputFormat.class);
    
    FileSystem dfs = FileSystem.get(outputPath.toUri(), conf);
    if (dfs.exists(outputPath)) {
      dfs.delete(outputPath, true);
    }
    
    client.setConf(conf);
    JobClient.runJob(conf);
  }

View Full Code Here

                log.info("Connecting to map-reduce job tracker at: " + properties.get(JOB_TRACKER_LOCATION));
        }


        try {
            // Set job-specific configuration knobs
            jobClient = new JobClient(new JobConf(configuration));
        }
        catch (IOException e) {
            int errCode = 6009;
            String msg = "Failed to create job client:" + e.getMessage();
            throw new ExecException(msg, errCode, PigException.BUG, e);

View Full Code Here

        } else {                
            js.setSuccessful(true);
                           
            js.addMapReduceStatistics(ps.getJobClient());
            
            JobClient client = ps.getJobClient();
            RunningJob rjob = null;
            try {
                rjob = client.getJob(job.getAssignedJobID());
            } catch (IOException e) {
                LOG.warn("Failed to get running job", e);
            }
            if (rjob == null) {
                LOG.warn("Failed to get RunningJob for job "

View Full Code Here


    @Override
    protected void processKill(String jobid) throws IOException
    {
        if (mJobConf != null) {
            JobClient jc = new JobClient(mJobConf);
            JobID id = JobID.forName(jobid);
            RunningJob job = jc.getJob(id);
            if (job == null)
                System.out.println("Job with id " + jobid + " is not active");
            else
            {    
                job.killJob();

View Full Code Here

   * @param job
   * @return true for success
   * @throws IOException
   */
  public static boolean runJob(JobConf job) throws IOException {
    JobClient jc = new JobClient(job);
    boolean sucess = true;
    RunningJob running = null;
    try {
      running = jc.submitJob(job);
      String jobId = running.getJobID();
      System.out.println("Job " + jobId + " is submitted");
      while (!running.isComplete()) {
        System.out.println("Job " + jobId + " is still running.");
        try {
          Thread.sleep(60000);
        } catch (InterruptedException e) {
        }
        running = jc.getJob(jobId);
      }
      sucess = running.isSuccessful();
    } finally {
      if (!sucess && (running != null)) {
        running.killJob();
      }
      jc.close();
    }
    return sucess;
  }

View Full Code Here

   * @param job
   * @return true for success
   * @throws IOException
   */
  public static boolean runJob(JobConf job) throws IOException {
    JobClient jc = new JobClient(job);
    boolean sucess = true;
    RunningJob running = null;
    try {
      running = jc.submitJob(job);
      String jobId = running.getJobID();
      System.out.println("Job " + jobId + " is submitted");
      while (!running.isComplete()) {
        System.out.println("Job " + jobId + " is still running.");
        try {
          Thread.sleep(60000);
        } catch (InterruptedException e) {
        }
        running = jc.getJob(jobId);
      }
      sucess = running.isSuccessful();
    } finally {
      if (!sucess && (running != null)) {
        running.killJob();
      }
      jc.close();
    }
    return sucess;
  }

View Full Code Here

    this.state = Job.WAITING;
    this.jobID = "unassigned";
    this.mapredJobID = "unassigned";
    this.jobName = "unassigned";
    this.message = "just initialized";
    this.jc = new JobClient(jobConf);
  }

View Full Code Here

   * @param job
   * @return true for success
   * @throws IOException
   */
  public static boolean runJob(JobConf job) throws IOException {
    JobClient jc = new JobClient(job);
    boolean sucess = true;
    RunningJob running = null;
    try {
      running = jc.submitJob(job);
      String jobId = running.getJobID();
      System.out.println("Job " + jobId + " is submitted");
      while (!running.isComplete()) {
        System.out.println("Job " + jobId + " is still running.");
        try {
          Thread.sleep(60000);
        } catch (InterruptedException e) {
        }
        running = jc.getJob(jobId);
      }
      sucess = running.isSuccessful();
    } finally {
      if (!sucess && (running != null)) {
        running.killJob();
      }
      jc.close();
    }
    return sucess;
  }

View Full Code Here

        MROperPlan mrp = compile(php, pc);
                
        HExecutionEngine exe = pc.getExecutionEngine();
        ConfigurationValidator.validatePigProperties(exe.getConfiguration());
        Configuration conf = ConfigurationUtil.toConfiguration(exe.getConfiguration());
        JobClient jobClient = new JobClient(exe.getJobConf());
        
        JobControlCompiler jcc = new JobControlCompiler(pc, conf);
        
        // start collecting statistics
        PigStatsUtil.startCollection(pc, jobClient, jcc, mrp);

View Full Code Here

      File wd = new File(".").getAbsoluteFile();
      StreamUtil.unJar(new File(jar_), wd);
    }


    // if jobConf_ changes must recreate a JobClient
    jc_ = new JobClient(jobConf_);
    boolean error = true;
    running_ = null;
    String lastReport = null;
    try {
      running_ = jc_.submitJob(jobConf_);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapred.JobClient

com.atlantbh.jmeter.plugins.hadooputilities.jobstatistics.JobLayer

com.atlantbh.jmeter.plugins.hadooputilities.jobstatistics.TaskLayer

com.cloudera.circus.test.TestXTest

com.cloudera.lib.service.hadoop.HadoopService

com.cloudera.lib.service.hadoop.TestHadoopService

com.fb2pdf.hadoop.cluster.MeanShiftCanopyDriver

com.ikanow.infinit.e.processing.custom.CustomProcessingController

com.linkedin.camus.etl.kafka.CamusJob

com.netflix.lipstick.pigtolipstick.BasicP2LClient

com.netflix.lipstick.warnings.JobWarnings

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.

Examples of org.apache.hadoop.mapred.JobClient

Job Control

Related Classes of org.apache.hadoop.mapred.JobClient