Examples of JobConf

Examples of org.apache.hadoop.mapred.JobConf

62
63
64
65
66
67
68
69
70
71
72
73
        path.put("result", HDFS + "/user/hdfs/pagerank/result");// 计算结果的PR
        return path;
    }

    public static JobConf config() {// Hadoop集群的远程配置信息
        JobConf conf = new JobConf(PageRankJob.class);
        conf.setJobName("PageRank");
        conf.addResource("classpath:/hadoop/core-site.xml");
        conf.addResource("classpath:/hadoop/hdfs-site.xml");
        conf.addResource("classpath:/hadoop/mapred-site.xml");
        return conf;
    }
View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf

63
64
65
66
67
68
69
70
71
72
73
            context.write(key, v);
        }
    }

    public static void run(Map<String, String> path) throws IOException, InterruptedException, ClassNotFoundException {
        JobConf conf = PageRankJob.config();

        String input = path.get("input");
        String input_pr = path.get("input_pr");
        String output = path.get("tmp1");
        String page = path.get("page");
View Full Code Here

Examples of org.apache.hadoop.mapred.JobConf

111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
      // need to get unique dir per cluster
      System.setProperty("test.build.data", "build/test/data/" + clusterName);

      log.info("Starting cluster=" + clusterName);

      Configuration config = new JobConf();

      // umask trick
      String umask = getCurrentUmask(tmpDir, config);
      if (umask != null) {
        log.info("Setting expected umask to " + umask);
        config.set("dfs.datanode.data.dir.perm", umask);
      }

      // dfs cluster is updating config
      // newer dfs cluster are using builder pattern
      // but we need to support older versions in
View Full Code Here

Examples of org.apache.nutch.mapReduce.JobConf

45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
    File tempDir =
      new File("grep-temp-"+
               Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));

    JobConf grepJob = new JobConf();

    grepJob.setNumMapTasks(18);
    grepJob.setInputDir(new File(args[0]));

    grepJob.setMapperClass(RegexMapper.class);
    grepJob.set("mapred.mapper.regex", args[2]);
    if (args.length == 4)
      grepJob.set("mapred.mapper.regex.group", args[3]);
   
    grepJob.setCombinerClass(LongSumReducer.class);
    grepJob.setReducerClass(LongSumReducer.class);

    grepJob.setNumReduceTasks(6);
    grepJob.setOutputDir(tempDir);
    grepJob.setOutputFormat(OutputFormats.get("seq"));
    grepJob.setOutputKeyClass(UTF8.class);
    grepJob.setOutputValueClass(LongWritable.class);

    JobClient.runJob(grepJob);

    JobConf sortJob = new JobConf();

    sortJob.setNumMapTasks(6);

    sortJob.setInputDir(tempDir);
    sortJob.setInputFormat(InputFormats.get("seq"));
    sortJob.setInputKeyClass(UTF8.class);
    sortJob.setInputValueClass(LongWritable.class);

    sortJob.setMapperClass(InverseMapper.class);

    sortJob.setNumReduceTasks(1);                 // write a single file
    sortJob.setOutputDir(new File(args[1]));
    sortJob.setOutputKeyComparatorClass           // sort by decreasing freq
      (LongWritable.DecreasingComparator.class);

    JobClient.runJob(sortJob);

    new JobClient().getFs().delete(tempDir);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.