Examples of BSPJob


Examples of org.apache.hama.bsp.BSPJob

          args);
      // get other arguments
      CommandLine results = parser.parse(cli.options,
          genericParser.getRemainingArgs());

      BSPJob job = new BSPJob(getConf());

      if (results.hasOption("input")) {
        FileInputFormat.setInputPaths(job, results.getOptionValue("input"));
      }
      if (results.hasOption("output")) {
        FileOutputFormat.setOutputPath(job,
            new Path(results.getOptionValue("output")));
      }
      if (results.hasOption("jar")) {
        job.setJar(results.getOptionValue("jar"));
      }

      if (results.hasOption("jobname")) {
        job.setJobName(results.getOptionValue("jobname"));
      }

      if (results.hasOption("inputformat")) {
        setIsJavaRecordReader(job.getConfiguration(), true);
        job.setInputFormat(getClass(results, "inputformat", conf,
            InputFormat.class));
      }

      if (results.hasOption("partitioner")) {
        job.setPartitioner(getClass(results, "partitioner", conf,
            Partitioner.class));
      }

      if (results.hasOption("outputformat")) {
        setIsJavaRecordWriter(job.getConfiguration(), true);
        job.setOutputFormat(getClass(results, "outputformat", conf,
            OutputFormat.class));
      }

      if (results.hasOption("streaming")) {
        LOG.info("Streaming enabled!");
        job.set("hama.streaming.enabled", "true");
      }

      if (results.hasOption("jobconf")) {
        LOG.warn("-jobconf option is deprecated, please use -D instead.");
        String options = results.getOptionValue("jobconf");
        StringTokenizer tokenizer = new StringTokenizer(options, ",");
        while (tokenizer.hasMoreTokens()) {
          String keyVal = tokenizer.nextToken().trim();
          String[] keyValSplit = keyVal.split("=", 2);
          job.set(keyValSplit[0], keyValSplit[1]);
        }
      }

      if (results.hasOption("bspTasks")) {
        int optionValue = Integer.parseInt(results.getOptionValue("bspTasks"));
        conf.setInt("bsp.local.tasks.maximum", optionValue);
        conf.setInt("bsp.peers.num", optionValue);
      }

      if (results.hasOption("program")) {
        String executablePath = results.getOptionValue("program");
        setExecutable(job.getConfiguration(), executablePath);
        DistributedCache.addCacheFile(new Path(executablePath).toUri(), conf);
      }

      if (results.hasOption("interpreter")) {
        job.getConfiguration().set("hama.pipes.executable.interpretor",
            results.getOptionValue("interpreter"));
      }

      if (results.hasOption("programArgs")) {
        job.getConfiguration().set("hama.pipes.executable.args",
            Joiner.on(" ").join(results.getOptionValues("programArgs")));
        // job.getConfiguration().set("hama.pipes.resolve.executable.args",
        // "true");
      }

      if (results.hasOption("cachefiles")) {
        FileSystem fs = FileSystem.get(getConf());
        String[] optionValues = results.getOptionValues("cachefiles");
        for (String s : optionValues) {
          Path path = new Path(s);
          FileStatus[] globStatus = fs.globStatus(path);
          for (FileStatus f : globStatus) {
            if (!f.isDir()) {
              DistributedCache.addCacheFile(f.getPath().toUri(),
                  job.getConfiguration());
            } else {
              LOG.info("Ignoring directory " + f.getPath() + " while globbing.");
            }
          }
        }
      }

      // if they gave us a jar file, include it into the class path
      String jarFile = job.getJar();
      if (jarFile != null) {
        @SuppressWarnings("deprecation")
        final URL[] urls = new URL[] { FileSystem.getLocal(conf)
            .pathToFile(new Path(jarFile)).toURL() };
        // FindBugs complains that creating a URLClassLoader should be
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.