Examples of JobConf

cgl.imr.base.impl.JobConf
com.dp.nebula.wormhole.common.config.JobConf
org.apache.hadoop.mapred.JobConf
te a new JobConf JobConf job = new JobConf(new Configuration(), MyJob.class); // Specify various job-specific parameters job.setJobName("myjob"); job.setInputPath(new Path("in")); job.setOutputPath(new Path("out")); job.setMapperClass(MyJob.MyMapper.class); job.setCombinerClass(MyJob.MyReducer.class); job.setReducerClass(MyJob.MyReducer.class); job.setInputFormat(SequenceFileInputFormat.class); job.setOutputFormat(SequenceFileOutputFormat.class);
@see JobClient @see ClusterStatus @see Tool @see DistributedCache
org.apache.nutch.mapReduce.JobConf
A map/reduce job configuration. This names the {@link Mapper}, combiner (if any), {@link Partitioner}, {@link Reducer}, {@link InputFormat}, and {@link OutputFormat} implementations to be used. It also indicates the setof input files, and where the output files should be written.

Examples of org.apache.hadoop.mapred.JobConf

    }
  }


  //Job configuration
  private static JobConf createJobConf(Configuration conf) {
    JobConf jobconf = new JobConf(conf, CopyFiles.class);
    jobconf.setJobName(NAME);


    // turn off speculative execution, because DFS doesn't handle
    // multiple writers to the same file.
    jobconf.setMapSpeculativeExecution(false);


    jobconf.setInputFormat(CopyInputFormat.class);
    jobconf.setOutputKeyClass(Text.class);
    jobconf.setOutputValueClass(Text.class);


    jobconf.setMapperClass(FSCopyFilesMapper.class);
    jobconf.setNumReduceTasks(0);
    return jobconf;
  }

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf

    // new API - just delegates to the Old API
    //
    @Override
    public List<InputSplit> getSplits(JobContext context) throws IOException {
        org.elasticsearch.hadoop.mr.compat.JobContext compatJobContext = CompatHandler.jobContext(context);
        JobConf conf = HadoopCfgUtils.asJobConf(compatJobContext.getConfiguration());
        // NOTE: this method expects a ShardInputSplit to be returned (which implements both the old and the new API).
        return Arrays.asList((InputSplit[]) getSplits(conf, conf.getNumMapTasks()));
    }

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf

        }
    }


    @Parameters
    public static Collection<Object[]> configs() {
        JobConf conf = HdpBootstrap.hadoopConfig();


        conf.setInputFormat(SplittableTextInputFormat.class);
        conf.setOutputFormat(EsOutputFormat.class);
        conf.setReducerClass(IdentityReducer.class);
        HadoopCfgUtils.setGenericOptions(conf);
        conf.setNumMapTasks(2);
        conf.setInt("actual.splits", 2);
        conf.setNumReduceTasks(0);




        JobConf standard = new JobConf(conf);
        standard.setMapperClass(TabMapper.class);
        standard.setMapOutputValueClass(LinkedMapWritable.class);
        standard.set(ConfigurationOptions.ES_INPUT_JSON, "false");
        FileInputFormat.setInputPaths(standard, new Path(TestUtils.sampleArtistsDat(conf)));


        JobConf json = new JobConf(conf);
        json.setMapperClass(IdentityMapper.class);
        json.setMapOutputValueClass(Text.class);
        json.set(ConfigurationOptions.ES_INPUT_JSON, "true");
        FileInputFormat.setInputPaths(json, new Path(TestUtils.sampleArtistsJson(conf)));


        return Arrays.asList(new Object[][] {
                { standard, "" },
                { json, "json-" }

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf

        copyFromLocal(localPath, localPath);
    }


    public static void copyFromLocal(String localPath, String destination) {
        try {
            JobConf hadoopConfig = HdpBootstrap.hadoopConfig();
            FileSystem fs = FileSystem.get(hadoopConfig);
            if (!(fs instanceof LocalFileSystem)) {
                Path src = new Path(localPath);
                Path dst = new Path(destination);
                fs.copyFromLocalFile(false, true, src, dst);

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf

        this.config = config;
    }


  @Test
  public void testBasicMultiSave() throws Exception {
    JobConf conf = createJobConf();
    conf.set(ConfigurationOptions.ES_RESOURCE, "oldapi/multi-save");


    MultiOutputFormat.addOutputFormat(conf, EsOutputFormat.class);
    MultiOutputFormat.addOutputFormat(conf, PrintStreamOutputFormat.class);
    //MultiOutputFormat.addOutputFormat(conf, TextOutputFormat.class);


    PrintStreamOutputFormat.stream(conf, Stream.OUT);
    //conf.set("mapred.output.dir", "foo/bar");
    //FileOutputFormat.setOutputPath(conf, new Path("foo/bar"));


    conf.setClass("mapred.output.format.class", MultiOutputFormat.class, OutputFormat.class);
    runJob(conf);
  }

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf

  }




    @Test
    public void testNoInput() throws Exception {
        JobConf conf = createJobConf();


        // use only when dealing with constant input
        assumeFalse(conf.get(ConfigurationOptions.ES_INPUT_JSON).equals("true"));
        conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/constant");
        conf.setMapperClass(ConstantMapper.class);


        runJob(conf);
    }

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf

        runJob(conf);
    }


    @Test
    public void testBasicIndex() throws Exception {
        JobConf conf = createJobConf();
        conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/save");


        runJob(conf);
    }

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf

        runJob(conf);
    }


    @Test
    public void testBasicIndexWithId() throws Exception {
        JobConf conf = createJobConf();
        conf.set(ConfigurationOptions.ES_MAPPING_ID, "number");
        conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/savewithid");


        runJob(conf);
    }

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf

        runJob(conf);
    }


    @Test
    public void testCreateWithId() throws Exception {
        JobConf conf = createJobConf();
        conf.set(ConfigurationOptions.ES_WRITE_OPERATION, "create");
        conf.set(ConfigurationOptions.ES_MAPPING_ID, "number");
        conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/createwithid");


        runJob(conf);
    }

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf

        testCreateWithId();
    }


    @Test(expected = IOException.class)
    public void testUpdateWithoutId() throws Exception {
        JobConf conf = createJobConf();
        conf.set(ConfigurationOptions.ES_WRITE_OPERATION, "upsert");
        conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/update");


        runJob(conf);
    }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.