Examples of GiraphConfiguration

org.apache.giraph.conf.GiraphConfiguration
Adds user methods specific to Giraph. This will be put into an ImmutableClassesGiraphConfiguration that provides the configuration plus the immutable classes.

Examples of org.apache.giraph.conf.GiraphConfiguration

   * @param taskAttemptId the MRv1 TaskAttemptID we constructed from CLI args
   *                      supplied by GiraphApplicationMaster.
   */
  public GiraphYarnTask(final TaskAttemptID taskAttemptId) {
    conf = new ImmutableClassesGiraphConfiguration<I, V, E, M>(
      new GiraphConfiguration());
    bspTaskId = taskAttemptId.getTaskID().getId();
    conf.setInt("mapred.task.partition", bspTaskId);
    proxy = buildProxyMapperContext(taskAttemptId);
    graphTaskManager = new GraphTaskManager<I, V, E, M>(proxy);
  }

View Full Code Here

Examples of org.apache.giraph.conf.GiraphConfiguration

    containerId = cId; // future good stuff will need me to operate.
    appAttemptId = aId;
    progress = 0.0f;
    lastResponseId = new AtomicInteger(0);
    giraphConf =
      new ImmutableClassesGiraphConfiguration(new GiraphConfiguration());
    completedCount = new AtomicInteger(0);
    failedCount = new AtomicInteger(0);
    allocatedCount = new AtomicInteger(0);
    successfulCount = new AtomicInteger(0);
    rpc = YarnRPC.create(giraphConf);

View Full Code Here

Examples of org.apache.giraph.conf.GiraphConfiguration

  }


  @Before
  public void setUp() throws IOException {
    // Setup the conf
    GiraphConfiguration tmpConf = new GiraphConfiguration();
    tmpConf.setVertexClass(TestVertex.class);
    conf = new ImmutableClassesGiraphConfiguration(tmpConf);


    context = mock(Context.class);
    when(context.getConfiguration()).thenReturn(conf);
  }

View Full Code Here

Examples of org.apache.giraph.conf.GiraphConfiguration


  /** Tests if aggregators are handled on a proper way during supersteps */
  @Test
  public void testAggregatorsHandling() throws IOException,
      ClassNotFoundException, InterruptedException {
    GiraphConfiguration conf = new GiraphConfiguration();
    conf.setVertexClass(AggregatorsTestVertex.class);
    conf.setVertexInputFormatClass(
        SimplePageRankVertex.SimplePageRankVertexInputFormat.class);
    GiraphJob job = prepareJob(getCallingMethodName(), conf);
    job.getConfiguration().setMasterComputeClass(
        AggregatorsTestVertex.AggregatorsTestMasterCompute.class);
    // test with aggregators split in a few requests

View Full Code Here

Examples of org.apache.giraph.conf.GiraphConfiguration

  @Test
  public void testAggregatorsCheckpointing() throws ClassNotFoundException,
      IOException, InterruptedException {
    Path checkpointsDir = getTempPath("checkPointsForTesting");
    Path outputPath = getTempPath(getCallingMethodName());
    GiraphConfiguration conf = new GiraphConfiguration();
    conf.setVertexClass(AggregatorsTestVertex.class);
    conf.setMasterComputeClass(
        AggregatorsTestVertex.AggregatorsTestMasterCompute.class);
    conf.setVertexInputFormatClass(
        SimplePageRankVertex.SimplePageRankVertexInputFormat.class);
    GiraphJob job = prepareJob(getCallingMethodName(), conf, outputPath);


    GiraphConfiguration configuration = job.getConfiguration();
    GiraphConstants.CHECKPOINT_DIRECTORY.set(configuration, checkpointsDir.toString());
    GiraphConstants.CLEANUP_CHECKPOINTS_AFTER_SUCCESS.set(configuration, false);
    configuration.setCheckpointFrequency(4);


    assertTrue(job.run(true));


    // Restart the test from superstep 4
    System.out.println("testAggregatorsCheckpointing: Restarting from " +
        "superstep 4 with checkpoint path = " + checkpointsDir);
    outputPath = getTempPath(getCallingMethodName() + "Restarted");
    conf = new GiraphConfiguration();
    conf.setVertexClass(AggregatorsTestVertex.class);
    conf.setMasterComputeClass(
        AggregatorsTestVertex.AggregatorsTestMasterCompute.class);
    conf.setVertexInputFormatClass(
        SimplePageRankVertex.SimplePageRankVertexInputFormat.class);
    GiraphJob restartedJob = prepareJob(getCallingMethodName() + "Restarted",
        conf, outputPath);
    job.getConfiguration().setMasterComputeClass(
        SimpleCheckpointVertex.SimpleCheckpointVertexMasterCompute.class);
    GiraphConfiguration restartedJobConf = restartedJob.getConfiguration();
    GiraphConstants.CHECKPOINT_DIRECTORY.set(restartedJobConf,
        checkpointsDir.toString());
    restartedJobConf.setLong(GiraphConstants.RESTART_SUPERSTEP, 4);


    assertTrue(restartedJob.run(true));
  }

View Full Code Here

Examples of org.apache.giraph.conf.GiraphConfiguration

        AccumuloOutputFormat.setOutputInfo(conf, USER, PASSWORD, true, null);
        AccumuloOutputFormat.setMockInstance(conf, INSTANCE_NAME);


        GiraphJob job = new GiraphJob(conf, getCallingMethodName());
        setupConfiguration(job);
        GiraphConfiguration giraphConf = job.getConfiguration();
        giraphConf.setVertexClass(EdgeNotification.class);
        giraphConf.setVertexInputFormatClass(AccumuloEdgeInputFormat.class);
        giraphConf.setVertexOutputFormatClass(AccumuloEdgeOutputFormat.class);


        HashSet<Pair<Text, Text>> columnsToFetch = new HashSet<Pair<Text,Text>>();
        columnsToFetch.add(new Pair<Text, Text>(FAMILY, CHILDREN));
        AccumuloInputFormat.fetchColumns(job.getConfiguration(), columnsToFetch);

View Full Code Here

Examples of org.apache.giraph.conf.GiraphConfiguration

   * Set up the GiraphConfiguration settings we need to run a no-op Giraph
   * job on a MiniYARNCluster as an integration test. Some YARN-specific
   * flags are set inside GiraphYarnClient and won't need to be set here.
   */
  private void setupYarnConfiguration() throws IOException {
    conf = new GiraphConfiguration();
    conf.setWorkerConfiguration(1, 1, 100.0f);
    conf.setMaxMasterSuperstepWaitMsecs(30 * 1000);
    conf.setEventWaitMsecs(3 * 1000);
    conf.setYarnLibJars(""); // no need
    conf.setYarnTaskHeapMb(256); // small since no work to be done

View Full Code Here

Examples of org.apache.giraph.conf.GiraphConfiguration

  }


  @Before
  public void setUp() throws IOException {
    // Setup the conf
    GiraphConfiguration tmpConf = new GiraphConfiguration();
    GiraphConstants.VERTEX_CLASS.set(tmpConf, TestVertex.class);
    conf = new ImmutableClassesGiraphConfiguration(tmpConf);


    @SuppressWarnings("rawtypes")
    Context context = mock(Context.class);

View Full Code Here

Examples of org.apache.giraph.conf.GiraphConfiguration

    }
  }


  @Before
  public void setUp() {
    GiraphConfiguration tmpConfig = new GiraphConfiguration();
    tmpConfig.setVertexClass(IntVertex.class);
    conf = new ImmutableClassesGiraphConfiguration(tmpConfig);
  }

View Full Code Here

Examples of org.apache.giraph.conf.GiraphConfiguration

    // additional configuration for Hive
    adjustConfigurationForHive();


    // setup GiraphJob
    GiraphJob job = new GiraphJob(getConf(), getClass().getName());
    GiraphConfiguration giraphConf = job.getConfiguration();
    giraphConf.setVertexClass(vertexClass);


    setupHiveInputs(giraphConf);
    setupHiveOutput(giraphConf);


    giraphConf.setWorkerConfiguration(workers, workers, 100.0f);
    initGiraphJob(job);


    logOptions(giraphConf);


    return job.run(isVerbose) ? 0 : -1;

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.