Examples of JobConf

cgl.imr.base.impl.JobConf
com.dp.nebula.wormhole.common.config.JobConf
org.apache.hadoop.mapred.JobConf
te a new JobConf JobConf job = new JobConf(new Configuration(), MyJob.class); // Specify various job-specific parameters job.setJobName("myjob"); job.setInputPath(new Path("in")); job.setOutputPath(new Path("out")); job.setMapperClass(MyJob.MyMapper.class); job.setCombinerClass(MyJob.MyReducer.class); job.setReducerClass(MyJob.MyReducer.class); job.setInputFormat(SequenceFileInputFormat.class); job.setOutputFormat(SequenceFileOutputFormat.class);
@see JobClient @see ClusterStatus @see Tool @see DistributedCache
org.apache.nutch.mapReduce.JobConf
A map/reduce job configuration. This names the {@link Mapper}, combiner (if any), {@link Partitioner}, {@link Reducer}, {@link InputFormat}, and {@link OutputFormat} implementations to be used. It also indicates the setof input files, and where the output files should be written.

Examples of org.apache.hadoop.mapred.JobConf

        
        System.exit(0);
    }


    public static JobConf config() {
        JobConf conf = new JobConf(Recommend.class);
        conf.setJobName("Recommand");
        conf.addResource("classpath:/hadoop/core-site.xml");
        conf.addResource("classpath:/hadoop/hdfs-site.xml");
        conf.addResource("classpath:/hadoop/mapred-site.xml");
        conf.set("io.sort.mb", "1024");
        return conf;
    }

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf

        }


    }


    public static void run(Map<String, String> path) throws IOException, InterruptedException, ClassNotFoundException {
        JobConf conf = config();
        String local_data = path.get("purchase");
        String input = path.get("input");
        String output = path.get("output");


        // 初始化purchase

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf


        job.waitForCompletion(true);
    }


    public static JobConf config() {// Hadoop集群的远程配置信息
        JobConf conf = new JobConf(Purchase.class);
        conf.setJobName("purchase");
        conf.addResource("classpath:/hadoop/core-site.xml");
        conf.addResource("classpath:/hadoop/hdfs-site.xml");
        conf.addResource("classpath:/hadoop/mapred-site.xml");
        return conf;
    }

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf


    private String hdfsPath;
    private Configuration conf;


    public static void main(String[] args) throws IOException {
        JobConf conf = config();
        HdfsDAO hdfs = new HdfsDAO(conf);
//        hdfs.copyFile("datafile/item.csv", "/tmp/new");
//        hdfs.ls("/tmp/new");
        hdfs.rename("/user/hdfs/pagerank/tmp3", "/user/hdfs/pagerank/tmp4");
    }

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf

//        hdfs.ls("/tmp/new");
        hdfs.rename("/user/hdfs/pagerank/tmp3", "/user/hdfs/pagerank/tmp4");
    }


    public static JobConf config() {
        JobConf conf = new JobConf(HdfsDAO.class);
        conf.setJobName("HdfsDAO");
        conf.addResource("classpath:/hadoop/core-site.xml");
        conf.addResource("classpath:/hadoop/hdfs-site.xml");
        conf.addResource("classpath:/hadoop/mapred-site.xml");
        return conf;
    }

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf

            }
        }
    }


    public static void run(Map<String, String> path) throws IOException {
        JobConf conf = Recommend.config();


        String input1 = path.get("Step4Input1");
        String input2 = path.get("Step4Input2");
        String output = path.get("Step4Output");


        HdfsDAO hdfs = new HdfsDAO(Recommend.HDFS, conf);
        hdfs.rmr(output);


        conf.setOutputKeyClass(IntWritable.class);
        conf.setOutputValueClass(Text.class);


        conf.setMapperClass(Step4_PartialMultiplyMapper.class);
        conf.setCombinerClass(Step4_AggregateAndRecommendReducer.class);
        conf.setReducerClass(Step4_AggregateAndRecommendReducer.class);


        conf.setInputFormat(TextInputFormat.class);
        conf.setOutputFormat(TextOutputFormat.class);


        FileInputFormat.setInputPaths(conf, new Path(input1), new Path(input2));
        FileOutputFormat.setOutputPath(conf, new Path(output));
        
        RunningJob job = JobClient.runJob(conf);

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf

            }
        }
    }


    public static void run(Map<String, String> path) throws IOException, InterruptedException, ClassNotFoundException {
        JobConf conf = PageRankJob.config();


        String input = path.get("input_pr");
        String output = path.get("result");


        HdfsDAO hdfs = new HdfsDAO(PageRankJob.HDFS, conf);

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf

        }


    }


    public static void run(Map<String, String> path) throws IOException, InterruptedException, ClassNotFoundException {
        JobConf conf = PageRankJob.config();


        String input = path.get("tmp1");
        String output = path.get("tmp2");
        String pr = path.get("input_pr");
        nums = Integer.parseInt(path.get("nums"));//页面数

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf


    public static void main(String[] args) throws Exception {
        String input = "hdfs://192.168.1.210:9000/user/hdfs/log_kpi/";
        String output = "hdfs://192.168.1.210:9000/user/hdfs/log_kpi/pv";


        JobConf conf = new JobConf(KPIPV.class);
        conf.setJobName("KPIPV");
        conf.addResource("classpath:/hadoop/core-site.xml");
        conf.addResource("classpath:/hadoop/hdfs-site.xml");
        conf.addResource("classpath:/hadoop/mapred-site.xml");


        conf.setMapOutputKeyClass(Text.class);
        conf.setMapOutputValueClass(IntWritable.class);


        conf.setOutputKeyClass(Text.class);
        conf.setOutputValueClass(IntWritable.class);


        conf.setMapperClass(KPIPVMapper.class);
        conf.setCombinerClass(KPIPVReducer.class);
        conf.setReducerClass(KPIPVReducer.class);


        conf.setInputFormat(TextInputFormat.class);
        conf.setOutputFormat(TextOutputFormat.class);


        FileInputFormat.setInputPaths(conf, new Path(input));
        FileOutputFormat.setOutputPath(conf, new Path(output));


        JobClient.runJob(conf);

View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf


    public static void main(String[] args) throws Exception {
        String input = "hdfs://192.168.1.210:9000/user/hdfs/log_kpi";
        String output = "hdfs://192.168.1.210:9000/user/hdfs/log_kpi/time";


        JobConf conf = new JobConf(KPITime.class);
        conf.setJobName("KPITime");
        conf.addResource("classpath:/hadoop/core-site.xml");
        conf.addResource("classpath:/hadoop/hdfs-site.xml");
        conf.addResource("classpath:/hadoop/mapred-site.xml");


        conf.setOutputKeyClass(Text.class);
        conf.setOutputValueClass(IntWritable.class);


        conf.setMapperClass(KPITimeMapper.class);
        conf.setCombinerClass(KPITimeReducer.class);
        conf.setReducerClass(KPITimeReducer.class);


        conf.setInputFormat(TextInputFormat.class);
        conf.setOutputFormat(TextOutputFormat.class);


        FileInputFormat.setInputPaths(conf, new Path(input));
        FileOutputFormat.setOutputPath(conf, new Path(output));


        JobClient.runJob(conf);

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.