Examples of GiraphJob

org.apache.giraph.graph.GiraphJob
Limits the functions that can be called by the user. Job is too flexible for our needs. For instance, our job should not have any reduce tasks.
org.apache.giraph.job.GiraphJob
Generates an appropriate internal {@link Job} for using Giraph in Hadoop.Uses composition to avoid unwanted {@link Job} methods from exposureto the user.

Examples of org.apache.giraph.graph.GiraphJob

                               "per vertex (-e)");
            return -1;
        }


        int workers = Integer.parseInt(cmd.getOptionValue('w'));
        GiraphJob job = new GiraphJob(getConf(), getClass().getName());
        if (!cmd.hasOption('c') ||
                (Integer.parseInt(cmd.getOptionValue('c')) == 0)) {
            System.out.println("Using " +
                                PageRankHashMapVertex.class.getName());
            job.setVertexClass(PageRankHashMapVertex.class);
        } else {
            System.out.println("Using " +
                                PageRankEdgeListVertex.class.getName());
            job.setVertexClass(PageRankEdgeListVertex.class);
        }
        job.setVertexInputFormatClass(PseudoRandomVertexInputFormat.class);
        job.setWorkerConfiguration(workers, workers, 100.0f);
        job.getConfiguration().setLong(
            PseudoRandomVertexInputFormat.AGGREGATE_VERTICES,
            Long.parseLong(cmd.getOptionValue('V')));
        job.getConfiguration().setLong(
            PseudoRandomVertexInputFormat.EDGES_PER_VERTEX,
            Long.parseLong(cmd.getOptionValue('e')));
        job.getConfiguration().setInt(
            SUPERSTEP_COUNT,
            Integer.parseInt(cmd.getOptionValue('s')));


        boolean isVerbose = false;
        if (cmd.hasOption('v')) {
            isVerbose = true;
        }
        if (cmd.hasOption('s')) {
            getConf().setInt(SUPERSTEP_COUNT,
                             Integer.parseInt(cmd.getOptionValue('s')));
        }
        if (job.run(isVerbose) == true) {
            return 0;
        } else {
            return -1;
        }
    }

View Full Code Here

Examples of org.apache.giraph.graph.GiraphJob

     * @throws ClassNotFoundException
     * @throws InterruptedException
     */
    public void testBspSuperStep()
            throws IOException, InterruptedException, ClassNotFoundException {
        GiraphJob job = new GiraphJob(getCallingMethodName());
        setupConfiguration(job);
        job.getConfiguration().setFloat(GiraphJob.TOTAL_INPUT_SPLIT_MULTIPLIER,
                                        2.0f);
        // GeneratedInputSplit will generate 10 vertices
        job.getConfiguration().setLong(GeneratedVertexReader.READER_VERTICES,
                                       10);
        job.setVertexClass(SimpleSuperstepVertex.class);
        job.setVertexInputFormatClass(SimpleSuperstepVertexInputFormat.class);
        job.setVertexOutputFormatClass(SimpleSuperstepVertexOutputFormat.class);
        Path outputPath = new Path("/tmp/" + getCallingMethodName());
        removeAndSetOutput(job, outputPath);
        assertTrue(job.run(true));
        if (getJobTracker() == null) {
            FileStatus fileStatus = getSinglePartFileStatus(job, outputPath);
            assertTrue(fileStatus.getLen() == 49);
        }
    }

View Full Code Here

Examples of org.apache.giraph.graph.GiraphJob

     * @throws ClassNotFoundException
     * @throws InterruptedException
     */
    public void testBspMsg()
            throws IOException, InterruptedException, ClassNotFoundException {
        GiraphJob job = new GiraphJob(getCallingMethodName());
        setupConfiguration(job);
        job.setVertexClass(SimpleMsgVertex.class);
        job.setVertexInputFormatClass(SimpleSuperstepVertexInputFormat.class);
        assertTrue(job.run(true));
    }

View Full Code Here

Examples of org.apache.giraph.graph.GiraphJob

     * @throws ClassNotFoundException
     * @throws InterruptedException
     */
    public void testEmptyVertexInputFormat()
            throws IOException, InterruptedException, ClassNotFoundException {
        GiraphJob job = new GiraphJob(getCallingMethodName());
        setupConfiguration(job);
        job.getConfiguration().setLong(GeneratedVertexReader.READER_VERTICES,
                                       0);
        job.setVertexClass(SimpleMsgVertex.class);
        job.setVertexInputFormatClass(SimpleSuperstepVertexInputFormat.class);
        assertTrue(job.run(true));
    }

View Full Code Here

Examples of org.apache.giraph.graph.GiraphJob

     * @throws ClassNotFoundException
     * @throws InterruptedException
     */
    public void testBspCombiner()
            throws IOException, InterruptedException, ClassNotFoundException {
        GiraphJob job = new GiraphJob(getCallingMethodName());
        setupConfiguration(job);
        job.setVertexClass(SimpleCombinerVertex.class);
        job.setVertexInputFormatClass(SimpleSuperstepVertexInputFormat.class);
        job.setVertexCombinerClass(SimpleSumCombiner.class);
        assertTrue(job.run(true));
    }

View Full Code Here

Examples of org.apache.giraph.graph.GiraphJob

     * @throws ClassNotFoundException
     * @throws InterruptedException
     */
    public void testBspPageRank()
            throws IOException, InterruptedException, ClassNotFoundException {
        GiraphJob job = new GiraphJob(getCallingMethodName());
        setupConfiguration(job);
        job.setVertexClass(SimplePageRankVertex.class);
        job.setWorkerContextClass(
          SimplePageRankVertex.SimplePageRankVertexWorkerContext.class);
        job.setVertexInputFormatClass(SimplePageRankVertexInputFormat.class);
        assertTrue(job.run(true));
        if (getJobTracker() == null) {
            double maxPageRank =
              SimplePageRankVertex.SimplePageRankVertexWorkerContext.finalMax;
            double minPageRank =
              SimplePageRankVertex.SimplePageRankVertexWorkerContext.finalMin;

View Full Code Here

Examples of org.apache.giraph.graph.GiraphJob

     * @throws ClassNotFoundException
     * @throws InterruptedException
     */
    public void testBspShortestPaths()
            throws IOException, InterruptedException, ClassNotFoundException {
        GiraphJob job = new GiraphJob(getCallingMethodName());
        setupConfiguration(job);
        job.setVertexClass(SimpleShortestPathsVertex.class);
        job.setVertexInputFormatClass(SimplePageRankVertexInputFormat.class);
        job.setVertexOutputFormatClass(
            SimpleShortestPathsVertexOutputFormat.class);
        job.getConfiguration().setLong(SimpleShortestPathsVertex.SOURCE_ID, 0);
        Path outputPath = new Path("/tmp/" + getCallingMethodName());
        removeAndSetOutput(job, outputPath);
        assertTrue(job.run(true));


        job = new GiraphJob(getCallingMethodName());
        setupConfiguration(job);
        job.setVertexClass(SimpleShortestPathsVertex.class);
        job.setVertexInputFormatClass(SimplePageRankVertexInputFormat.class);
        job.setVertexOutputFormatClass(
            SimpleShortestPathsVertexOutputFormat.class);
        job.getConfiguration().setLong(SimpleShortestPathsVertex.SOURCE_ID, 0);
        Path outputPath2 = new Path("/tmp/" + getCallingMethodName() + "2");
        removeAndSetOutput(job, outputPath2);
        assertTrue(job.run(true));
        if (getJobTracker() == null) {
            FileStatus fileStatus = getSinglePartFileStatus(job, outputPath);
            FileStatus fileStatus2 = getSinglePartFileStatus(job, outputPath2);
            assertTrue(fileStatus.getLen() == fileStatus2.getLen());
        }

View Full Code Here

Examples of org.apache.giraph.graph.GiraphJob

     * @throws ClassNotFoundException
     * @throws InterruptedException
     */
    public void testBspPageRankWithAggregatorWriter()
            throws IOException, InterruptedException, ClassNotFoundException {
        GiraphJob job = new GiraphJob(getCallingMethodName());
        setupConfiguration(job);
        job.setVertexClass(SimplePageRankVertex.class);
        job.setWorkerContextClass(
            SimplePageRankVertex.SimplePageRankVertexWorkerContext.class);
        job.setVertexInputFormatClass(SimplePageRankVertexInputFormat.class);
        job.setAggregatorWriterClass(SimpleAggregatorWriter.class);
        Path outputPath = new Path("/tmp/" + getCallingMethodName());
        removeAndSetOutput(job, outputPath);
        assertTrue(job.run(true));
        if (getJobTracker() == null) {
            double maxPageRank =
                SimplePageRankVertex.SimplePageRankVertexWorkerContext.finalMax;
            double minPageRank =
                SimplePageRankVertex.SimplePageRankVertexWorkerContext.finalMin;

View Full Code Here

Examples of org.apache.giraph.graph.GiraphJob

        if (!cmd.hasOption('n')) {
            System.out.println("Need to set the number of messages per edge (-n)");
            return -1;
        }
        int workers = Integer.parseInt(cmd.getOptionValue('w'));
        GiraphJob job = new GiraphJob(getConf(), getClass().getName());
        job.getConfiguration().setInt(GiraphJob.CHECKPOINT_FREQUENCY, 0);
        job.setVertexClass(RandomMessageVertex.class);
        job.setVertexInputFormatClass(PseudoRandomVertexInputFormat.class);
        job.setWorkerContextClass(RandomMessageBenchmarkWorkerContext.class);
        job.setWorkerConfiguration(workers, workers, 100.0f);
        job.getConfiguration().setLong(
            PseudoRandomVertexInputFormat.AGGREGATE_VERTICES,
            Long.parseLong(cmd.getOptionValue('V')));
        job.getConfiguration().setLong(
            PseudoRandomVertexInputFormat.EDGES_PER_VERTEX,
            Long.parseLong(cmd.getOptionValue('e')));
        job.getConfiguration().setInt(
            SUPERSTEP_COUNT,
            Integer.parseInt(cmd.getOptionValue('s')));
        job.getConfiguration().setInt(
            RandomMessageBenchmark.NUM_BYTES_PER_MESSAGE,
            Integer.parseInt(cmd.getOptionValue('b')));
        job.getConfiguration().setInt(
            RandomMessageBenchmark.NUM_MESSAGES_PER_EDGE,
            Integer.parseInt(cmd.getOptionValue('n')));


        boolean isVerbose = false;
        if (cmd.hasOption('v')) {
            isVerbose = true;
        }
        if (cmd.hasOption('s')) {
            getConf().setInt(SUPERSTEP_COUNT,
                             Integer.parseInt(cmd.getOptionValue('s')));
        }
        if (cmd.hasOption('f')) {
            job.getConfiguration().setInt(GiraphJob.MSG_NUM_FLUSH_THREADS,
                Integer.parseInt(cmd.getOptionValue('f')));
        }
        if (job.run(isVerbose) == true) {
            return 0;
        } else {
            return -1;
        }
    }

View Full Code Here

Examples of org.apache.giraph.graph.GiraphJob

        if (getJobTracker() == null) {
            System.out.println(
                "testSingleFault: Ignore this test in local mode.");
            return;
        }
        GiraphJob job = new GiraphJob(getCallingMethodName());
        setupConfiguration(job);
        job.getConfiguration().setBoolean(SimpleCheckpointVertex.ENABLE_FAULT,
                                          true);
        job.getConfiguration().setInt("mapred.map.max.attempts", 4);
        job.getConfiguration().setInt(GiraphJob.POLL_MSECS, 5000);
        job.getConfiguration().set(GiraphJob.CHECKPOINT_DIRECTORY,
                                   HDFS_CHECKPOINT_DIR);
        job.getConfiguration().setBoolean(
            GiraphJob.CLEANUP_CHECKPOINTS_AFTER_SUCCESS, false);
        job.setVertexClass(SimpleCheckpointVertex.class);
        job.setVertexInputFormatClass(SimpleSuperstepVertexInputFormat.class);
        job.setVertexOutputFormatClass(SimpleSuperstepVertexOutputFormat.class);
        job.setWorkerContextClass(
            SimpleCheckpointVertex.SimpleCheckpointVertexWorkerContext.class);
        Path outputPath = new Path("/tmp/" + getCallingMethodName());
        removeAndSetOutput(job, outputPath);
        assertTrue(job.run(true));
    }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.