Examples of org.apache.hadoop.mapreduce.OutputCommitter.commitJob()

Class org.apache.hadoop.mapreduce.OutputCommitter

Examples of org.apache.hadoop.mapreduce.OutputCommitter.commitJob()

org.apache.hadoop.mapreduce.OutputCommitter.commitJob()
For committing job's output after successful job completion. Note that this is invoked for jobs with final runstate as SUCCESSFUL. This is called from the application master process for the entire job. This is guaranteed to only be called once. If it throws an exception the entire job will fail. @param jobContext Context of the job whose output is being written. @throws IOException

      listing.buildListing(listingFile, options);


      conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, targetBase);
      conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, targetBase);


      committer.commitJob(jobContext);
      if (!TestDistCpUtils.checkIfFoldersAreInSync(fs, targetBase, sourceBase)) {
        Assert.fail("Source and target folders are not in sync");
      }
      if (!TestDistCpUtils.checkIfFoldersAreInSync(fs, sourceBase, targetBase)) {
        Assert.fail("Source and target folders are not in sync");

View Full Code Here

      if (!TestDistCpUtils.checkIfFoldersAreInSync(fs, sourceBase, targetBase)) {
        Assert.fail("Source and target folders are not in sync");
      }


      //Test for idempotent commit
      committer.commitJob(jobContext);
      if (!TestDistCpUtils.checkIfFoldersAreInSync(fs, targetBase, sourceBase)) {
        Assert.fail("Source and target folders are not in sync");
      }
      if (!TestDistCpUtils.checkIfFoldersAreInSync(fs, sourceBase, targetBase)) {
        Assert.fail("Source and target folders are not in sync");

View Full Code Here

      listing.buildListing(listingFile, options);


      conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, targetBase);
      conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, targetBase);


      committer.commitJob(jobContext);
      if (!TestDistCpUtils.checkIfFoldersAreInSync(fs, targetBase, sourceBase)) {
        Assert.fail("Source and target folders are not in sync");
      }
      Assert.assertEquals(fs.listStatus(new Path(targetBase)).length, 4);

View Full Code Here

        Assert.fail("Source and target folders are not in sync");
      }
      Assert.assertEquals(fs.listStatus(new Path(targetBase)).length, 4);


      //Test for idempotent commit
      committer.commitJob(jobContext);
      if (!TestDistCpUtils.checkIfFoldersAreInSync(fs, targetBase, sourceBase)) {
        Assert.fail("Source and target folders are not in sync");
      }
      Assert.assertEquals(fs.listStatus(new Path(targetBase)).length, 4);
    } catch (IOException e) {

View Full Code Here

      conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, finalPath);
      conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, true);


      Assert.assertTrue(fs.exists(new Path(workPath)));
      Assert.assertFalse(fs.exists(new Path(finalPath)));
      committer.commitJob(jobContext);
      Assert.assertFalse(fs.exists(new Path(workPath)));
      Assert.assertTrue(fs.exists(new Path(finalPath)));


      //Test for idempotent commit
      committer.commitJob(jobContext);

View Full Code Here

      committer.commitJob(jobContext);
      Assert.assertFalse(fs.exists(new Path(workPath)));
      Assert.assertTrue(fs.exists(new Path(finalPath)));


      //Test for idempotent commit
      committer.commitJob(jobContext);
      Assert.assertFalse(fs.exists(new Path(workPath)));
      Assert.assertTrue(fs.exists(new Path(finalPath)));


    } catch (IOException e) {
      LOG.error("Exception encountered while testing for preserve status", e);

View Full Code Here

      conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, true);


      Assert.assertTrue(fs.exists(new Path(workPath)));
      Assert.assertTrue(fs.exists(new Path(finalPath)));
      try {
        committer.commitJob(jobContext);
        Assert.fail("Should not be able to atomic-commit to pre-existing path.");
      } catch(Exception exception) {
        Assert.assertTrue(fs.exists(new Path(workPath)));
        Assert.assertTrue(fs.exists(new Path(finalPath)));
        LOG.info("Atomic-commit Test pass.");

View Full Code Here

      }
    } finally {
      writer.close(context);
    }
    committer.commitTask(context);
    committer.commitJob(job);


    InputFormat<IntWritable, DoubleWritable> iformat =
      new SequenceFileInputFormat<IntWritable, DoubleWritable>();
    int count = 0;
    r.setSeed(seed);

View Full Code Here

      }
    } finally {
      writer.close(context);
    }
    committer.commitTask(context);
    committer.commitJob(job);


    InputFormat<IntWritable, DoubleWritable> iformat =
      new SequenceFileInputFormat<IntWritable, DoubleWritable>();
    int count = 0;
    r.setSeed(seed);

View Full Code Here

                }


                @Override
                public void commitJob(JobContext jobContext) throws IOException {
                    try {
                        baseOutputCommitter.commitJob(jobContext);
                        Configuration conf = jobContext.getConfiguration();
                        try {
                        //import hfiles
                        new LoadIncrementalHFiles(conf)
                                .doBulkLoad(HFileOutputFormat.getOutputPath(jobContext),

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.