Examples of BSPJob

Examples of org.apache.hama.bsp.BSPJob

119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
  public static void main(String[] args) throws InterruptedException,
      IOException, ClassNotFoundException {
    // BSP job configuration
    HamaConfiguration conf = new HamaConfiguration();

    BSPJob bsp = new BSPJob(conf, PiEstimator.class);
    // Set the job name
    bsp.setJobName("Pi Estimation Example");
    bsp.setBspClass(MyEstimator.class);

    BSPJobClient jobClient = new BSPJobClient(conf);
    ClusterStatus cluster = jobClient.getClusterStatus(true);

    if (args.length > 0) {
      bsp.setNumBspTask(Integer.parseInt(args[0]));
    } else {
      // Set to maximum
      bsp.setNumBspTask(cluster.getGroomServers());
    }

    // Choose one as a master
    for (String peerName : cluster.getActiveGroomNames().values()) {
      conf.set(MASTER_TASK, peerName);
      break;
    }

    FileSystem fileSys = FileSystem.get(conf);
    initTempDir(fileSys);

    long startTime = System.currentTimeMillis();

    if (bsp.waitForCompletion(true)) {
      printOutput(fileSys, conf);

      System.out.println("Job Finished in "
          + (double) (System.currentTimeMillis() - startTime) / 1000.0
          + " seconds");
View Full Code Here

Examples of org.apache.hama.bsp.BSPJob

111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
  public static void main(String[] args) throws InterruptedException,
      IOException, ClassNotFoundException {
    // BSP job configuration
    HamaConfiguration conf = new HamaConfiguration();

    BSPJob bsp = new BSPJob(conf, SerializePrinting.class);
    // Set the job name
    bsp.setJobName("Serialize Printing");
    bsp.setBspClass(HelloBSP.class);

    // Set the task size as a number of GroomServer
    BSPJobClient jobClient = new BSPJobClient(conf);
    ClusterStatus cluster = jobClient.getClusterStatus(false);
    bsp.setNumBspTask(cluster.getGroomServers());

    FileSystem fileSys = FileSystem.get(conf);
    initTempDir(fileSys);

    if (bsp.waitForCompletion(true)) {
      printOutput(fileSys, cluster, conf);
    }
  }
View Full Code Here

Examples of org.apache.hama.bsp.BSPJob

101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
    conf.setInt(SIZEOFMSG, Integer.parseInt(args[0]));
    conf.setInt(N_COMMUNICATIONS, Integer.parseInt(args[1]));
    conf.setInt(N_SUPERSTEPS, Integer.parseInt(args[2]));

    BSPJob bsp = new BSPJob(conf, RandBench.class);
    // Set the job name
    bsp.setJobName("Random Communication Benchmark");
    bsp.setBspClass(RandBSP.class);

    // Set the task size as a number of GroomServer
    BSPJobClient jobClient = new BSPJobClient(conf);
    ClusterStatus cluster = jobClient.getClusterStatus(false);
    bsp.setNumBspTask(cluster.getGroomServers());

    long startTime = System.currentTimeMillis();
    bsp.waitForCompletion(true);
    System.out.println("Job Finished in "
        + (double) (System.currentTimeMillis() - startTime) / 1000.0
        + " seconds");
  }
View Full Code Here

Examples of org.apache.hama.bsp.BSPJob

37
38
39
40
41
42
43
44
45
46
47
  @Test
  public void testFileRead() throws Exception {
    VectorDoubleFileInputFormat inputFormat = new VectorDoubleFileInputFormat();
    Path file = new Path("src/test/resources/vd_file_sample.txt");
    InputSplit split = new FileSplit(file, 0, 1000, new String[]{"localhost"});
    BSPJob job = new BSPJob();
    RecordReader<VectorWritable, DoubleWritable> recordReader = inputFormat.getRecordReader(split, job);
    assertNotNull(recordReader);
    VectorWritable key = recordReader.createKey();
    assertNotNull(key);
    DoubleWritable value = recordReader.createValue();
View Full Code Here

Examples of org.apache.hama.bsp.BSPJob

86
87
88
89
90
91
92
93
94
95
    } else {
      KMeansBSP.prepareInputText(k, conf, in, center, out, fs);
      in = new Path(args[0], "textinput/in.seq");
    }

    BSPJob job = KMeansBSP.createJob(conf, in, out, true);

    // just submit the job
    job.waitForCompletion(true);
  }
View Full Code Here

Examples of org.apache.hama.bsp.BSPJob

95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
    conf.setInt(SIZEOFMSG, Integer.parseInt(args[0]));
    conf.setInt(N_COMMUNICATIONS, Integer.parseInt(args[1]));
    conf.setInt(N_SUPERSTEPS, Integer.parseInt(args[2]));

    BSPJob bsp = new BSPJob(conf, RandBench.class);
    // Set the job name
    bsp.setJobName("Random Communication Benchmark");
    bsp.setBspClass(RandBSP.class);
    bsp.setInputFormat(NullInputFormat.class);
    bsp.setOutputFormat(NullOutputFormat.class);

    // Set the task size as a number of GroomServer
    BSPJobClient jobClient = new BSPJobClient(conf);
    ClusterStatus cluster = jobClient.getClusterStatus(false);
    bsp.setNumBspTask(cluster.getMaxTasks());

    long startTime = System.currentTimeMillis();
    if (bsp.waitForCompletion(true)) {
      System.out.println("Job Finished in "
          + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds");
    }
  }
View Full Code Here

Examples of org.apache.hama.bsp.BSPJob

52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
    if (args.length > 1 && args[1] != null && args[1].equals("logistic")) {
      conf.setClass(GradientDescentBSP.REGRESSION_MODEL_CLASS,
          LogisticRegressionModel.class, RegressionModel.class);
    }

    BSPJob bsp = new BSPJob(conf, GradientDescentExample.class);
    // Set the job name
    bsp.setJobName("Gradient Descent Example");
    bsp.setBspClass(GradientDescentBSP.class);
    bsp.setInputFormat(VectorDoubleFileInputFormat.class);
    bsp.setInputPath(new Path(args[0]));
    bsp.setInputKeyClass(VectorWritable.class);
    bsp.setInputValueClass(DoubleWritable.class);
    bsp.setOutputKeyClass(VectorWritable.class);
    bsp.setOutputValueClass(DoubleWritable.class);
    bsp.setOutputFormat(TextOutputFormat.class);
    FileOutputFormat.setOutputPath(bsp, TMP_OUTPUT);

    long startTime = System.currentTimeMillis();
    if (bsp.waitForCompletion(true)) {
      printOutput(conf);
      System.out.println("Job Finished in "
          + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds");
    }
View Full Code Here

Examples of org.apache.hama.bsp.BSPJob

129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
  @SuppressWarnings("unchecked")
  public static void main(String[] args) throws IOException,
      ClassNotFoundException, InterruptedException {
    // BSP job configuration
    HamaConfiguration conf = new HamaConfiguration();
    BSPJob bsp = new BSPJob(conf, SuperstepBSP.class);
    // Set the job name
    bsp.setJobName("Fault Tolerant Pi Estimation Example");

    // set our supersteps, they must be given in execution order
    bsp.setSupersteps(SuperstepPiEstimator.PiEstimatorCalculator.class,
        SuperstepPiEstimator.PiEstimatorAggregator.class);

    bsp.setInputFormat(NullInputFormat.class);
    bsp.setOutputKeyClass(Text.class);
    bsp.setOutputValueClass(DoubleWritable.class);
    bsp.setOutputFormat(TextOutputFormat.class);
    FileOutputFormat.setOutputPath(bsp, TMP_OUTPUT);

    BSPJobClient jobClient = new BSPJobClient(conf);
    ClusterStatus cluster = jobClient.getClusterStatus(true);

    if (args.length > 0) {
      bsp.setNumBspTask(Integer.parseInt(args[0]));
    } else {
      // Set to maximum
      bsp.setNumBspTask(cluster.getMaxTasks());
    }

    long startTime = System.currentTimeMillis();
    if (bsp.waitForCompletion(true)) {
      printOutput(conf);
      System.out.println("Job Finished in "
          + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds");
    }
  }
View Full Code Here

Examples of org.apache.hama.bsp.BSPJob

392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
          args);
      // get other arguments
      CommandLine results = parser.parse(cli.options,
          genericParser.getRemainingArgs());

      BSPJob job = new BSPJob(getConf());

      if (results.hasOption("input")) {
        FileInputFormat.setInputPaths(job, results.getOptionValue("input"));
      }
      if (results.hasOption("output")) {
        FileOutputFormat.setOutputPath(job,
            new Path(results.getOptionValue("output")));
      }
      if (results.hasOption("jar")) {
        job.setJar(results.getOptionValue("jar"));
      }

      if (results.hasOption("jobname")) {
        job.setJobName(results.getOptionValue("jobname"));
      }

      if (results.hasOption("inputformat")) {
        setIsJavaRecordReader(job.getConf(), true);
        job.setInputFormat(getClass(results, "inputformat", conf,
            InputFormat.class));
      }

      if (results.hasOption("partitioner")) {
        job.setPartitioner(getClass(results, "partitioner", conf,
            Partitioner.class));
      }

      if (results.hasOption("outputformat")) {
        setIsJavaRecordWriter(job.getConf(), true);
        job.setOutputFormat(getClass(results, "outputformat", conf,
            OutputFormat.class));
      }

      if (results.hasOption("streaming")) {
        LOG.info("Streaming enabled!");
        job.set("hama.streaming.enabled", "true");
      }

      if (results.hasOption("jobconf")) {
        LOG.warn("-jobconf option is deprecated, please use -D instead.");
        String options = results.getOptionValue("jobconf");
        StringTokenizer tokenizer = new StringTokenizer(options, ",");
        while (tokenizer.hasMoreTokens()) {
          String keyVal = tokenizer.nextToken().trim();
          String[] keyValSplit = keyVal.split("=", 2);
          job.set(keyValSplit[0], keyValSplit[1]);
        }
      }

      if (results.hasOption("bspTasks")) {
        int optionValue = Integer.parseInt(results.getOptionValue("bspTasks"));
        conf.setInt("bsp.local.tasks.maximum", optionValue);
        conf.setInt("bsp.peers.num", optionValue);
      }

      if (results.hasOption("program")) {
        String executablePath = results.getOptionValue("program");
        setExecutable(job.getConf(), executablePath);
        DistributedCache.addCacheFile(new Path(executablePath).toUri(), conf);
      }

      if (results.hasOption("interpreter")) {
        job.getConf().set("hama.pipes.executable.interpretor",
            results.getOptionValue("interpreter"));
      }

      if (results.hasOption("programArgs")) {
        job.getConf().set("hama.pipes.executable.args",
            Joiner.on(" ").join(results.getOptionValues("programArgs")));
        // job.getConf().set("hama.pipes.resolve.executable.args", "true");
      }

      if (results.hasOption("cachefiles")) {
        FileSystem fs = FileSystem.get(getConf());
        String[] optionValues = results.getOptionValues("cachefiles");
        for (String s : optionValues) {
          Path path = new Path(s);
          FileStatus[] globStatus = fs.globStatus(path);
          for (FileStatus f : globStatus) {
            if (!f.isDir()) {
              DistributedCache.addCacheFile(f.getPath().toUri(), job.getConf());
            } else {
              LOG.info("Ignoring directory " + f.getPath() + " while globbing.");
            }
          }
        }
      }

      // if they gave us a jar file, include it into the class path
      String jarFile = job.getJar();
      if (jarFile != null) {
        @SuppressWarnings("deprecation")
        final URL[] urls = new URL[] { FileSystem.getLocal(conf)
            .pathToFile(new Path(jarFile)).toURL() };
        // FindBugs complains that creating a URLClassLoader should be
View Full Code Here

Examples of org.apache.hama.bsp.BSPJob

113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
  public static void main(String[] args) throws InterruptedException,
      IOException, ClassNotFoundException {
    // BSP job configuration
    HamaConfiguration conf = new HamaConfiguration();

    BSPJob bsp = new BSPJob(conf, PiEstimator.class);
    // Set the job name
    bsp.setJobName("Pi Estimation Example");
    bsp.setBspClass(MyEstimator.class);
    bsp.setInputFormat(NullInputFormat.class);
    bsp.setOutputKeyClass(Text.class);
    bsp.setOutputValueClass(DoubleWritable.class);
    bsp.setOutputFormat(TextOutputFormat.class);
    FileOutputFormat.setOutputPath(bsp, TMP_OUTPUT);

    BSPJobClient jobClient = new BSPJobClient(conf);
    ClusterStatus cluster = jobClient.getClusterStatus(true);

    if (args.length > 0) {
      bsp.setNumBspTask(Integer.parseInt(args[0]));
    } else {
      // Set to maximum
      bsp.setNumBspTask(cluster.getMaxTasks());
    }

    long startTime = System.currentTimeMillis();
    if (bsp.waitForCompletion(true)) {
      printOutput(conf);
      System.out.println("Job Finished in "
          + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds");
    }
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.