Examples of GiraphConfiguration


Examples of org.apache.giraph.conf.GiraphConfiguration

   * @throws InterruptedException
   */
  @Test
  public void testEmptyVertexInputFormat()
      throws IOException, InterruptedException, ClassNotFoundException {
    GiraphConfiguration conf = new GiraphConfiguration();
    conf.setVertexClass(SimpleMsgVertex.class);
    conf.setVertexInputFormatClass(SimpleSuperstepVertexInputFormat.class);
    GiraphJob job = prepareJob(getCallingMethodName(), conf);
    job.getConfiguration().setLong(GeneratedVertexReader.READER_VERTICES, 0);
    assertTrue(job.run(true));
  }
View Full Code Here

Examples of org.apache.giraph.conf.GiraphConfiguration

   * @throws InterruptedException
   */
  @Test
  public void testBspCombiner()
      throws IOException, InterruptedException, ClassNotFoundException {
    GiraphConfiguration conf = new GiraphConfiguration();
    conf.setVertexClass(SimpleCombinerVertex.class);
    conf.setVertexInputFormatClass(SimpleSuperstepVertexInputFormat.class);
    conf.setCombinerClass(SimpleSumCombiner.class);
    GiraphJob job = prepareJob(getCallingMethodName(), conf);
    assertTrue(job.run(true));
  }
View Full Code Here

Examples of org.apache.giraph.conf.GiraphConfiguration

   */
  @Test
  public void testBspShortestPaths()
      throws IOException, InterruptedException, ClassNotFoundException {
    Path outputPath = getTempPath(getCallingMethodName());
    GiraphConfiguration conf = new GiraphConfiguration();
    conf.setVertexClass(SimpleShortestPathsVertex.class);
    conf.setVertexInputFormatClass(SimplePageRankVertexInputFormat.class);
    conf.setVertexOutputFormatClass(
        JsonLongDoubleFloatDoubleVertexOutputFormat.class);
    SimpleShortestPathsVertex.SOURCE_ID.set(conf, 0);
    GiraphJob job = prepareJob(getCallingMethodName(), conf, outputPath);

    assertTrue(job.run(true));
View Full Code Here

Examples of org.apache.giraph.conf.GiraphConfiguration

  @Test
  public void testBspPageRankWithAggregatorWriter()
      throws IOException, InterruptedException, ClassNotFoundException {
    Path outputPath = getTempPath(getCallingMethodName());

    GiraphConfiguration conf = new GiraphConfiguration();
    conf.setVertexClass(SimplePageRankVertex.class);
    conf.setAggregatorWriterClass(TextAggregatorWriter.class);
    conf.setMasterComputeClass(
        SimplePageRankVertex.SimplePageRankVertexMasterCompute.class);
    conf.setVertexInputFormatClass(SimplePageRankVertexInputFormat.class);
    conf.setVertexOutputFormatClass(
        SimplePageRankVertex.SimplePageRankVertexOutputFormat.class);
    conf.setWorkerContextClass(
        SimplePageRankVertex.SimplePageRankVertexWorkerContext.class);
    GiraphJob job = prepareJob(getCallingMethodName(), conf, outputPath);
    GiraphConfiguration configuration = job.getConfiguration();
    Path aggregatorValues = getTempPath("aggregatorValues");
    configuration.setInt(TextAggregatorWriter.FREQUENCY,
        TextAggregatorWriter.ALWAYS);
    configuration.set(TextAggregatorWriter.FILENAME,
        aggregatorValues.toString());

    assertTrue(job.run(true));

    FileSystem fs = FileSystem.get(configuration);
View Full Code Here

Examples of org.apache.giraph.conf.GiraphConfiguration

   * @throws InterruptedException
   */
  @Test
  public void testBspMasterCompute()
      throws IOException, InterruptedException, ClassNotFoundException {
    GiraphConfiguration conf = new GiraphConfiguration();
    conf.setVertexClass(SimpleMasterComputeVertex.class);
    conf.setVertexInputFormatClass(SimplePageRankVertexInputFormat.class);
    conf.setMasterComputeClass(
        SimpleMasterComputeVertex.SimpleMasterCompute.class);
    conf.setWorkerContextClass(
        SimpleMasterComputeVertex.SimpleMasterComputeWorkerContext.class);
    GiraphJob job = prepareJob(getCallingMethodName(), conf);
    assertTrue(job.run(true));
    if (!runningInDistributedMode()) {
      double finalSum =
View Full Code Here

Examples of org.apache.giraph.conf.GiraphConfiguration

  /**
   * Adjust the configuration to the basic test case
   */
  public final Configuration setupConfiguration(GiraphJob job)
      throws IOException {
    GiraphConfiguration conf = job.getConfiguration();
    conf.set("mapred.jar", getJarLocation());

    // Allow this test to be run on a real Hadoop setup
    if (runningInDistributedMode()) {
      System.out.println("setupConfiguration: Sending job to job tracker " +
          jobTracker + " with jar path " + getJarLocation()
          + " for " + getName());
      conf.set("mapred.job.tracker", jobTracker);
      conf.setWorkerConfiguration(getNumWorkers(), getNumWorkers(), 100.0f);
    }
    else {
      System.out.println("setupConfiguration: Using local job runner with " +
          "location " + getJarLocation() + " for " + getName());
      conf.setWorkerConfiguration(1, 1, 100.0f);
      // Single node testing
      GiraphConstants.SPLIT_MASTER_WORKER.set(conf, false);
    }
    conf.setMaxMasterSuperstepWaitMsecs(30 * 1000);
    conf.setEventWaitMsecs(3 * 1000);
    GiraphConstants.ZOOKEEPER_SERVERLIST_POLL_MSECS.set(conf, 500);
    if (getZooKeeperList() != null) {
      conf.setZooKeeperConfiguration(getZooKeeperList());
    }
    // GeneratedInputSplit will generate 5 vertices
    conf.setLong(READER_VERTICES, 5);

    // Setup pathes for temporary files
    Path zookeeperDir = getTempPath("_bspZooKeeper");
    Path zkManagerDir = getTempPath("_defaultZkManagerDir");
    Path checkPointDir = getTempPath("_checkpoints");

    // We might start several jobs per test, so we need to clean up here
    FileUtils.deletePath(conf, zookeeperDir);
    FileUtils.deletePath(conf, zkManagerDir);
    FileUtils.deletePath(conf, checkPointDir);

    conf.set(GiraphConstants.ZOOKEEPER_DIR, zookeeperDir.toString());
    GiraphConstants.ZOOKEEPER_MANAGER_DIRECTORY.set(conf,
        zkManagerDir.toString());
    GiraphConstants.CHECKPOINT_DIRECTORY.set(conf, checkPointDir.toString());

    return conf;
View Full Code Here

Examples of org.apache.giraph.conf.GiraphConfiguration

  @Before
  public void setUp() throws IOException, InterruptedException {
    rr = mock(RecordReader.class);
    when(rr.nextKeyValue()).thenReturn(true).thenReturn(false);
    GiraphConfiguration giraphConf = new GiraphConfiguration();
    giraphConf.setVertexClass(DummyVertex.class);
    conf = new ImmutableClassesGiraphConfiguration<Text, DoubleWritable,
        DoubleWritable, Writable>(giraphConf);
    graphState = mock(GraphState.class);
    tac = mock(TaskAttemptContext.class);
    when(tac.getConfiguration()).thenReturn(conf);
View Full Code Here

Examples of org.apache.giraph.conf.GiraphConfiguration

    if (runningInDistributedMode()) {
      System.out.println(
          "testComputeContext: Ignore this test in distributed mode.");
      return;
    }
    GiraphConfiguration conf = new GiraphConfiguration();
    conf.setVertexClass(PartitionContextTestVertex.class);
    conf.setVertexInputFormatClass(
        SimplePageRankVertex.SimplePageRankVertexInputFormat.class);
    conf.setWorkerContextClass(
        PartitionContextTestVertex.TestPartitionContextWorkerContext.class);
    conf.setPartitionContextClass(
        PartitionContextTestVertex.TestPartitionContextPartitionContext.class);
    GiraphJob job = prepareJob(getCallingMethodName(), conf);
    // Use multithreading
    job.getConfiguration().setNumComputeThreads(
        PartitionContextTestVertex.NUM_COMPUTE_THREADS);
View Full Code Here

Examples of org.apache.giraph.conf.GiraphConfiguration

   * @throws ClassNotFoundException
   * @throws InterruptedException
   */
  private void testPageRank(int numComputeThreads)
      throws IOException, InterruptedException, ClassNotFoundException {
    GiraphConfiguration conf = new GiraphConfiguration();
    conf.setVertexClass(SimplePageRankVertex.class);
    conf.setVertexInputFormatClass(
        SimplePageRankVertex.SimplePageRankVertexInputFormat.class);
    conf.setWorkerContextClass(
        SimplePageRankVertex.SimplePageRankVertexWorkerContext.class);
    conf.setMasterComputeClass(
        SimplePageRankVertex.SimplePageRankVertexMasterCompute.class);
    conf.setNumComputeThreads(numComputeThreads);
    // Set enough partitions to generate randomness on the compute side
    if (numComputeThreads != 1) {
      GiraphConstants.USER_PARTITION_COUNT.set(conf, numComputeThreads * 5);
    }
    GiraphJob job = prepareJob(getCallingMethodName(), conf);
View Full Code Here

Examples of org.apache.giraph.conf.GiraphConfiguration

        "[2,0,[[3,1],[4,10]]]",
        "[3,0,[[4,2]]]",
        "[4,0,[]]"
    };

    GiraphConfiguration conf = new GiraphConfiguration();
    // start from vertex 1
    SOURCE_ID.set(conf, 1);
    conf.setVertexClass(SimpleShortestPathsVertex.class);
    conf.setOutEdgesClass(ByteArrayEdges.class);
    conf.setVertexInputFormatClass(
        JsonLongDoubleFloatDoubleVertexInputFormat.class);
    conf.setVertexOutputFormatClass(
        JsonLongDoubleFloatDoubleVertexOutputFormat.class);

    // run internally
    Iterable<String> results = InternalVertexRunner.run(conf, graph);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.