Examples of org.apache.hadoop.mapreduce.OutputCommitter.commitJob()

Class org.apache.hadoop.mapreduce.OutputCommitter

Examples of org.apache.hadoop.mapreduce.OutputCommitter.commitJob()

org.apache.hadoop.mapreduce.OutputCommitter.commitJob()
For committing job's output after successful job completion. Note that this is invoked for jobs with final runstate as SUCCESSFUL. This is called from the application master process for the entire job. This is guaranteed to only be called once. If it throws an exception the entire job will fail. @param jobContext Context of the job whose output is being written. @throws IOException

        LOG.debug("Committing output results: {}", format.getClass().getName());
        try {
            writer.close(context);
            OutputCommitter comitter = format.getOutputCommitter(context);
            comitter.commitTask(context);
            comitter.commitJob(context);
        } catch (InterruptedException e) {
            throw (InterruptedIOException) new InterruptedIOException().initCause(e);
        }
    }
}

View Full Code Here

        }


        @Override
        public void commitJob(JobContext jobContext) throws IOException {
          try {
            baseOutputCommitter.commitJob(jobContext);
            Configuration conf = jobContext.getConfiguration();
            try {
              //import hfiles
              new LoadIncrementalHFiles(conf)
                .doBulkLoad(HFileOutputFormat.getOutputPath(jobContext),

View Full Code Here

    publishTest(job);
  }


  public void publishTest(Job job) throws Exception {
    OutputCommitter committer = new FileOutputCommitterContainer(job, null);
    committer.commitJob(job);


    Partition part = client.getPartition(dbName, tblName, Arrays.asList("p1"));
    assertNotNull(part);


    StorerInfo storer = InternalUtil.extractStorerInfo(part.getSd(), part.getParameters());

View Full Code Here

                }


                @Override
                public void commitJob(JobContext jobContext) throws IOException {
                    try {
                        baseOutputCommitter.commitJob(jobContext);
                        Configuration conf = jobContext.getConfiguration();
                        try {
                            //import hfiles
                            new LoadIncrementalHFiles(conf)
                                .doBulkLoad(HFileOutputFormat.getOutputPath(jobContext),

View Full Code Here

        publishTest(job);
    }


    public void publishTest(Job job) throws Exception {
        OutputCommitter committer = new FileOutputCommitterContainer(job, null);
        committer.commitJob(job);


        Partition part = client.getPartition(dbName, tblName, Arrays.asList("p1"));
        assertNotNull(part);


        StorerInfo storer = InternalUtil.extractStorerInfo(part.getSd(), part.getParameters());

View Full Code Here

        VertexOutputFormat vertexOutputFormat =
          conf.createVertexOutputFormat();
        OutputCommitter outputCommitter =
          vertexOutputFormat.getOutputCommitter(proxy);
        // now we will have our output in OUTDIR if all went well...
        outputCommitter.commitJob(proxy);
        LOG.info("Master has committed the final job output data.");
      } catch (InterruptedException ie) {
        LOG.error("Interrupted while attempting to obtain " +
          "OutputCommitter.", ie);
      } catch (IOException ioe) {

View Full Code Here

    TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config);
    JobContext jobContext = new JobContext(taskAttemptContext.getConfiguration(),
        taskAttemptContext.getTaskAttemptID().getJobID());
    try {
      OutputCommitter committer = new CopyCommitter(null, taskAttemptContext);
      committer.commitJob(jobContext);
      Assert.assertEquals(taskAttemptContext.getStatus(), "Commit Successful");


      //Test for idempotent commit
      committer.commitJob(jobContext);
      Assert.assertEquals(taskAttemptContext.getStatus(), "Commit Successful");

View Full Code Here

      OutputCommitter committer = new CopyCommitter(null, taskAttemptContext);
      committer.commitJob(jobContext);
      Assert.assertEquals(taskAttemptContext.getStatus(), "Commit Successful");


      //Test for idempotent commit
      committer.commitJob(jobContext);
      Assert.assertEquals(taskAttemptContext.getStatus(), "Commit Successful");
    } catch (IOException e) {
      LOG.error("Exception encountered ", e);
      Assert.fail("Commit failed");
    }

View Full Code Here

      Path listingFile = new Path("/tmp1/" + String.valueOf(rand.nextLong()));
      listing.buildListing(listingFile, options);


      conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, targetBase);


      committer.commitJob(jobContext);
      if (!checkDirectoryPermissions(fs, targetBase, sourcePerm)) {
        Assert.fail("Permission don't match");
      }


      //Test for idempotent commit

View Full Code Here

      if (!checkDirectoryPermissions(fs, targetBase, sourcePerm)) {
        Assert.fail("Permission don't match");
      }


      //Test for idempotent commit
      committer.commitJob(jobContext);
      if (!checkDirectoryPermissions(fs, targetBase, sourcePerm)) {
        Assert.fail("Permission don't match");
      }


    } catch (IOException e) {

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.