Package edu.cmu.graphchi.preprocessing

Examples of edu.cmu.graphchi.preprocessing.FastSharder


            Thread chiThread = new Thread(new Runnable() {
                public void run() {
                    try {
                        setStatusString("Preprocessing: reading data from HDFS: " + location);
                        final FastSharder sharder = createSharder(getGraphName(), getNumShards());


                        HDFSGraphLoader hdfsLoader = new HDFSGraphLoader(location, new EdgeProcessor<Float>() {
                            long counter = 0;

                            public Float receiveEdge(int from, int to, String token) {
                                try {
                                    sharder.addEdge(from, to, token);
                                    counter++;
                                    if (counter % 100000 == 0) {
                                        setStatusString("Preprocessing, read " + counter + " edges");
                                    }
                                } catch (IOException e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }
                        });

                        hdfsLoader.load(pigSplit.getConf());

                        setStatusString("Sharding...");
                        sharder.process();

                        logger.info("Starting to run GraphChi");
                        setStatusString("Start GraphChi engine");
                        runGraphChi();
                        logger.info("Ready.");
View Full Code Here


public class IO {

  public static void convert_matrix_market() throws IOException{
      /* Run sharding (preprocessing) if the files do not exist yet */
        FastSharder sharder = createSharder(ProblemSetup.training, ProblemSetup.nShards);
        if (!new File(ChiFilenames.getFilenameIntervals(ProblemSetup.training, ProblemSetup.nShards)).exists() ||
                !new File(ProblemSetup.training + ".matrixinfo").exists()) {
            sharder.shard(new FileInputStream(new File(ProblemSetup.training)), FastSharder.GraphInputFormat.MATRIXMARKET);
        } else {
            ProblemSetup.logger.info("Found shards -- no need to preprocess");
        }
  }
View Full Code Here

        String inputFilePath = inputDirectory + fileName;
       
        /* Preprocessing graph : Making shards */

        FastSharder sharder = createSharder(inputFilePath, nShards);
        if (inputFilePath.equals("pipein")) {     // Allow piping graph in
            sharder.shard(System.in, fileType);
        } else {
            if (!new File(ChiFilenames.getFilenameIntervals(inputFilePath, nShards)).exists()) {
                sharder.shard(new FileInputStream(new File(inputFilePath)), fileType);
            } else {
                logger.info("Found shards -- no need to preprocess");
            }
        }

View Full Code Here

        CompressedIO.disableCompression();

        String inputFilePath = inputDirectory + fileName;
       
        /* Making shards */
        FastSharder sharder = createSharder(inputFilePath, nShards);
        if (inputFilePath.equals("pipein")) {     // Allow piping graph in
            sharder.shard(System.in, fileType);
        } else {
            if (!new File(ChiFilenames.getFilenameIntervals(inputFilePath, nShards)).exists()) {
                sharder.shard(new FileInputStream(new File(inputFilePath)), fileType);
            } else {
                logger.info("Found shards -- no need to preprocess");
            }
        }
       
View Full Code Here

TOP

Related Classes of edu.cmu.graphchi.preprocessing.FastSharder

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.