Package org.apache.hama

Examples of org.apache.hama.HamaConfiguration

415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
        if (numTasks == 0) {
          numTasks = numSplits;
        }

        HamaConfiguration conf = new HamaConfiguration(job.getConfiguration());

        conf.setInt(Constants.RUNTIME_DESIRED_PEERS_COUNT, numTasks);
        if (job.getConfiguration().get(Constants.RUNTIME_PARTITIONING_DIR) != null) {
          conf.set(Constants.RUNTIME_PARTITIONING_DIR, job.getConfiguration()
              .get(Constants.RUNTIME_PARTITIONING_DIR));
        }
        if (job.getConfiguration().get(Constants.RUNTIME_PARTITIONING_CLASS) != null) {
          conf.set(Constants.RUNTIME_PARTITIONING_CLASS,
              job.get(Constants.RUNTIME_PARTITIONING_CLASS));
        }
        BSPJob partitioningJob = new BSPJob(conf);
        LOG.debug("partitioningJob input: " + partitioningJob.get(Constants.JOB_INPUT_DIR));
        partitioningJob.setInputFormat(job.getInputFormat().getClass());
        partitioningJob.setInputKeyClass(job.getInputKeyClass());
        partitioningJob.setInputValueClass(job.getInputValueClass());
        partitioningJob.setOutputFormat(NullOutputFormat.class);
        partitioningJob.setOutputKeyClass(NullWritable.class);
        partitioningJob.setOutputValueClass(NullWritable.class);
        partitioningJob.setBspClass(PartitioningRunner.class);
        partitioningJob.set("bsp.partitioning.runner.job", "true");
        partitioningJob.getConfiguration().setBoolean(
            Constants.ENABLE_RUNTIME_PARTITIONING, false);
        partitioningJob.setOutputPath(partitionDir);

        boolean isPartitioned = false;
        try {
          isPartitioned = partitioningJob.waitForCompletion(true);
        } catch (InterruptedException e) {
          LOG.error("Interrupted partitioning run-time.", e);
        } catch (ClassNotFoundException e) {
          LOG.error("Class not found error partitioning run-time.", e);
        }

        if (isPartitioned) {
          if (job.getConfiguration().get(Constants.RUNTIME_PARTITIONING_DIR) != null) {
            job.setInputPath(new Path(conf
                .get(Constants.RUNTIME_PARTITIONING_DIR)));
          } else {
            job.setInputPath(partitionDir);
          }
          job.setBoolean("input.has.partitioned", true);
View Full Code Here
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
    boolean submitJob = false;
    boolean getStatus = false;
    String submitJobFile = null;
    String jobid = null;

    HamaConfiguration conf = new HamaConfiguration(getConf());
    init(conf);

    if ("-list".equals(cmd)) {
      if (args.length != 1 && !(args.length == 2 && "all".equals(args[1]))) {
        displayUsage(cmd);
        return exitCode;
      }
      if (args.length == 2 && "all".equals(args[1])) {
        listAllJobs = true;
      } else {
        listJobs = true;
      }
    } else if ("-list-active-grooms".equals(cmd)) {
      if (args.length != 1) {
        displayUsage(cmd);
        return exitCode;
      }
      listActiveGrooms = true;
    } else if ("-submit".equals(cmd)) {
      if (args.length == 1) {
        displayUsage(cmd);
        return exitCode;
      }

      submitJob = true;
      submitJobFile = args[1];
    } else if ("-kill".equals(cmd)) {
      if (args.length == 1) {
        displayUsage(cmd);
        return exitCode;
      }
      killJob = true;
      jobid = args[1];

    } else if ("-status".equals(cmd)) {
      if (args.length != 2) {
        displayUsage(cmd);
        return exitCode;
      }
      jobid = args[1];
      getStatus = true;

      // TODO Later, below functions should be implemented
      // with the Fault Tolerant mechanism.
    } else if ("-list-attempt-ids".equals(cmd)) {
      System.out.println("This function is not implemented yet.");
      return exitCode;
    } else if ("-kill-task".equals(cmd)) {
      System.out.println("This function is not implemented yet.");
      return exitCode;
    } else if ("-fail-task".equals(cmd)) {
      System.out.println("This function is not implemented yet.");
      return exitCode;
    }

    BSPJobClient jc = new BSPJobClient(new HamaConfiguration());
    if (listJobs) {
      listJobs();
      exitCode = 0;
    } else if (listAllJobs) {
      listAllJobs();
      exitCode = 0;
    } else if (listActiveGrooms) {
      listActiveGrooms();
      exitCode = 0;
    } else if (submitJob) {
      HamaConfiguration tConf = new HamaConfiguration(new Path(submitJobFile));
      RunningJob job = jc.submitJob(new BSPJob(tConf));
      System.out.println("Created job " + job.getID().toString());
    } else if (killJob) {
      RunningJob job = jc.getJob(BSPJobID.forName(jobid));
      if (job == null) {
View Full Code Here
57
58
59
60
61
62
63
  private PipesApplication<?, ?, ?, ?, ?> pipesApp = null;
  private static final Log LOG = LogFactory.getLog(BSPJob.class);

  public BSPJob() throws IOException {
    this(new HamaConfiguration());
  }
View Full Code Here
350
351
352
353
354
355
356
357
358
359
360
        layerSizeArraySb.append(' ');
      }
      conf.set("layerSizeArray", layerSizeArraySb.toString());
    }

    HamaConfiguration hamaConf = new HamaConfiguration(conf);

    BSPJob job = new BSPJob(hamaConf, SmallMLPTrainer.class);
    job.setJobName("Small scale MLP training");
    job.setJarByClass(SmallMLPTrainer.class);
    job.setBspClass(SmallMLPTrainer.class);
View Full Code Here
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
      return -1;
    }

    public static void main(String[] args) {

      HamaConfiguration hamaConf = new HamaConfiguration();
      hamaConf.setInt(Constants.GROOM_PING_PERIOD, 200);
      hamaConf.setClass("bsp.work.class", FaulTestBSP.class, BSP.class);
      hamaConf.setClass(SyncServiceFactory.SYNC_PEER_CLASS,
          LocalBSPRunner.LocalSyncClient.class, SyncClient.class);

      hamaConf.setInt("bsp.master.port", 610002);

      TaskAttemptID tid = new TaskAttemptID(new TaskID(new BSPJobID(
          "job_201110102255", 1), 1), 1);

      hamaConf.setInt(TEST_POINT, Integer.parseInt(args[0]));
      int port = Integer.parseInt(args[1]);

      try {
        BSPJob job = new BSPJob(hamaConf);
        job.setInputFormat(NullInputFormat.class);
        job.setOutputFormat(NullOutputFormat.class);

        final BSPPeerProtocol proto = (BSPPeerProtocol) RPC.getProxy(
            BSPPeerProtocol.class, HamaRPCProtocolVersion.versionID,
            new InetSocketAddress("127.0.0.1", port), hamaConf);

        BSPTask task = new BSPTask();
        task.setConf(job);

        LOG.info("Testing failure case in process - "
            + hamaConf.getInt(TEST_POINT, 0));

        Runtime.getRuntime().addShutdownHook(new Thread() {
          @Override
          public void run() {
            try {
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
  // these variables are preventing from rebooting the whole stuff again since
  // setup and teardown are called per method.

  public TestPartitioning() {
    configuration = new HamaConfiguration();
    configuration.set("bsp.master.address", "localhost");
    configuration.set("hama.child.redirect.log.console", "true");
    assertEquals("Make sure master addr is set to localhost:", "localhost",
        configuration.get("bsp.master.address"));
    configuration.set("bsp.local.dir", "/tmp/hama-test");
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
  public void testPartitioner() throws Exception {

    Configuration conf = new Configuration();
    conf.set("bsp.local.dir", "/tmp/hama-test/partitioning");
    conf.setBoolean("bsp.input.runtime.partitioning", true);
    BSPJob bsp = new BSPJob(new HamaConfiguration(conf));
    bsp.setJobName("Test partitioning with input");
    bsp.setBspClass(PartionedBSP.class);
    bsp.setNumBspTask(2);
    conf.setInt(Constants.ZOOKEEPER_SESSION_TIMEOUT, 600);
    bsp.setInputFormat(TextInputFormat.class);
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
  // increment is here to solve race conditions in parallel execution to choose
  // other ports.
  public static volatile int increment = 1;

  public void testMemoryMessaging() throws Exception {
    HamaConfiguration conf = new HamaConfiguration();
    conf.setClass(MessageManager.RECEIVE_QUEUE_TYPE_CLASS, MemoryQueue.class,
        MessageQueue.class);
    conf.set(DiskQueue.DISK_QUEUE_PATH_KEY, TMP_OUTPUT_PATH);
    messagingInternal(conf);
  }
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
    conf.set(DiskQueue.DISK_QUEUE_PATH_KEY, TMP_OUTPUT_PATH);
    messagingInternal(conf);
  }

  public void testDiskMessaging() throws Exception {
    HamaConfiguration conf = new HamaConfiguration();
    conf.set(DiskQueue.DISK_QUEUE_PATH_KEY, TMP_OUTPUT_PATH);
    conf.setClass(MessageManager.RECEIVE_QUEUE_TYPE_CLASS, DiskQueue.class,
        MessageQueue.class);
    messagingInternal(conf);
  }
View Full Code Here
152
153
154
155
156
157
158
159
160
161
162
163
      e.printStackTrace();
    }
   
   
    try {
      HamaConfiguration conf = new HamaConfiguration();
      conf.setInt(KeyValueHashPartitionedBSP.TEST_MAX_VALUE, maxValue);
      BSPJob job = new BSPJob(conf, TestKeyValueTextInputFormat.class);
      job.setJobName("Test KeyValueTextInputFormat together with HashPartitioner");
      job.setBspClass(KeyValueHashPartitionedBSP.class);
     
      job.setBoolean(Constants.ENABLE_RUNTIME_PARTITIONING, true);
View Full Code Here
TOP

Related Classes of org.apache.hama.HamaConfiguration

  • org.apache.hama.bsp.BSPApplicationMaster
  • org.apache.hama.bsp.BSPJob
  • org.apache.hama.bsp.BSPJobClient
  • org.apache.hama.bsp.BSPJobContext
  • org.apache.hama.bsp.BSPRunner
  • org.apache.hama.bsp.GroomServer
  • org.apache.hama.bsp.GroomServer$BSPPeerChild
  • org.apache.hama.bsp.GroomServer$Child
  • org.apache.hama.bsp.LocalBSPRunner
  • org.apache.hama.bsp.MesosExecutor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.