Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.JobContextImpl

109
110
111
112
113
114
115
116
117
118
119
  }

  public void testAbort() throws IOException {
    JobConf job = new JobConf();
    setConfForFileOutputCommitter(job);
    JobContext jContext = new JobContextImpl(job, taskID.getJobID());
    TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID);
    FileOutputCommitter committer = new FileOutputCommitter();
    FileOutputFormat.setWorkOutputPath(job, committer
        .getTempTaskOutputPath(tContext));
View Full Code Here
164
165
166
167
168
169
170
171
172
173
174
  public void testFailAbort() throws IOException {
    JobConf job = new JobConf();
    job.set(FileSystem.FS_DEFAULT_NAME_KEY, "faildel:///");
    job.setClass("fs.faildel.impl", FakeFileSystem.class, FileSystem.class);
    setConfForFileOutputCommitter(job);
    JobContext jContext = new JobContextImpl(job, taskID.getJobID());
    TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID);
    FileOutputCommitter committer = new FileOutputCommitter();
    FileOutputFormat.setWorkOutputPath(job, committer
        .getTempTaskOutputPath(tContext));
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
        assert nativeSplit != null;

        OutputFormat outputFormat = null;
        Exception err = null;

        JobContextImpl jobCtx = taskCtx.jobContext();

        try {
            InputFormat inFormat = ReflectionUtils.newInstance(jobCtx.getInputFormatClass(),
                hadoopContext().getConfiguration());

            RecordReader reader = inFormat.createRecordReader(nativeSplit, hadoopContext());

            reader.initialize(nativeSplit, hadoopContext());

            hadoopContext().reader(reader);

            GridHadoopJobInfo jobInfo = taskCtx.job().info();

            outputFormat = jobInfo.hasCombiner() || jobInfo.hasReducer() ? null : prepareWriter(jobCtx);

            Mapper mapper = ReflectionUtils.newInstance(jobCtx.getMapperClass(), hadoopContext().getConfiguration());

            try {
                mapper.run(new WrappedMapper().getMapContext(hadoopContext()));
            }
            finally {
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
    @SuppressWarnings({"ConstantConditions", "unchecked"})
    @Override public void run0(GridHadoopV2TaskContext taskCtx) throws GridException {
        OutputFormat outputFormat = null;
        Exception err = null;

        JobContextImpl jobCtx = taskCtx.jobContext();

        try {
            outputFormat = reduce || !taskCtx.job().info().hasReducer() ? prepareWriter(jobCtx) : null;

            Reducer reducer = ReflectionUtils.newInstance(reduce ? jobCtx.getReducerClass() : jobCtx.getCombinerClass(),
                jobCtx.getConfiguration());

            try {
                reducer.run(new WrappedReducer().getReducerContext(hadoopContext()));
            }
            finally {
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
    }

    /** {@inheritDoc} */
    @SuppressWarnings("ConstantConditions")
    @Override public void run0(GridHadoopV2TaskContext taskCtx) throws GridException {
        JobContextImpl jobCtx = taskCtx.jobContext();

        try {
            OutputFormat outputFormat = getOutputFormat(jobCtx);

            OutputCommitter committer = outputFormat.getOutputCommitter(hadoopContext());
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
    /** {@inheritDoc} */
    @SuppressWarnings("ConstantConditions")
    @Override protected void run0(GridHadoopV2TaskContext taskCtx) throws GridException {
        try {
            JobContextImpl jobCtx = taskCtx.jobContext();

            OutputFormat outputFormat = getOutputFormat(jobCtx);

            outputFormat.checkOutputSpecs(jobCtx);

View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.JobContextImpl

  • org.apache.hadoop.mapred.TestFileOutputCommitter
  • org.apache.hadoop.mapred.TestMRCJCFileOutputCommitter
  • org.gridgain.grid.kernal.processors.hadoop.v2.GridHadoopV2CleanupTask
  • org.gridgain.grid.kernal.processors.hadoop.v2.GridHadoopV2MapTask
  • org.gridgain.grid.kernal.processors.hadoop.v2.GridHadoopV2ReduceTask
  • org.gridgain.grid.kernal.processors.hadoop.v2.GridHadoopV2SetupTask
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.