Package voldemort.store.readonly.checksum.CheckSum

Examples of voldemort.store.readonly.checksum.CheckSum.CheckSumType

340
341
342
343
344
345
346
347
348
349
350
            if(statuses != null) {
                // sort the files so that index files come last. Maybe
                // this will help keep them cached until the swap
                Arrays.sort(statuses, new IndexFileLastComparator());
                byte[] origCheckSum = null;
                CheckSumType checkSumType = CheckSumType.NONE;

                // Do a checksum of checksum - Similar to HDFS
                CheckSum checkSumGenerator = null;
                CheckSum fileCheckSumGenerator = null;
View Full Code Here
379
380
381
382
383
384
385
386
387
388
389
        Path outputDir = conf.getOutputDir();
        Path inputPath = conf.getInputPath();
        Cluster cluster = conf.getCluster();
        List<StoreDefinition> storeDefs = conf.getStoreDefs();
        String storeName = conf.getStoreName();
        CheckSumType checkSumType = conf.getCheckSumType();
        boolean saveKeys = conf.getSaveKeys();
        boolean reducerPerBucket = conf.getReducerPerBucket();

        StoreDefinition storeDef = null;
        for(StoreDefinition def: storeDefs)
View Full Code Here
357
358
359
360
361
362
363
364
365
366
367
            if(statuses != null && statuses.length > 0) {
                // sort the files so that index files come last. Maybe
                // this will help keep them cached until the swap
                Arrays.sort(statuses, new IndexFileLastComparator());
                byte[] origCheckSum = null;
                CheckSumType checkSumType = CheckSumType.NONE;

                // Do a checksum of checksum - Similar to HDFS
                CheckSum checkSumGenerator = null;
                CheckSum fileCheckSumGenerator = null;
View Full Code Here
829
830
831
832
833
834
835
836
837
838
839
                                              props.getString("build.key.selection", null));

        int numChunks = props.getInt("num.chunks." + storeDef.getName(),
                                     props.getInt("num.chunks", -1));

        CheckSumType checkSumType = CheckSum.fromString(props.getString("checksum.type",
                                                                        CheckSum.toString(CheckSumType.MD5)));
        boolean saveKeys = props.getBoolean("save.keys", true);
        boolean reducerPerBucket = props.getBoolean("reducer.per.bucket", false);

        new VoldemortStoreBuilderJob(this.getId() + "-build-store",
View Full Code Here
571
572
573
574
575
576
577
578
579
580
581
        URI uri = new URI(url);
        Path outputDir = new Path(props.getString("build.output.dir"), uri.getHost());
        Path inputPath = getInputPath();
        String keySelection = props.getString("key.selection", null);
        String valSelection = props.getString("value.selection", null);
        CheckSumType checkSumType = CheckSum.fromString(props.getString("checksum.type",
                                                                        CheckSum.toString(CheckSumType.MD5)));
        boolean saveKeys = props.getBoolean("save.keys", true);
        boolean reducerPerBucket = props.getBoolean("reducer.per.bucket", false);
        int numChunks = props.getInt("num.chunks", -1);
View Full Code Here
176
177
178
179
180
181
182
183
184
185
186
        if(options.has("force-overwrite")) {
            FileSystem fs = outputDir.getFileSystem(conf);
            fs.delete(outputDir, true);
        }

        CheckSumType checkSumType = CheckSumType.toType(CmdUtils.valueOf(options, "checksum", ""));

        Class[] deps = new Class[] { ImmutableCollection.class, JDOMException.class,
                VoldemortConfig.class, HadoopStoreJobRunner.class, mapperClass };

        addDepJars(conf, deps, addJars);
View Full Code Here
TOP

Related Classes of voldemort.store.readonly.checksum.CheckSum.CheckSumType

  • voldemort.server.protocol.hadoop.RestHadoopFetcher
  • voldemort.store.readonly.fetcher.HdfsFetcher
  • voldemort.store.readonly.mr.azkaban.VoldemortBuildAndPushJob
  • voldemort.store.readonly.mr.azkaban.VoldemortMultiStoreBuildAndPushJob
  • voldemort.store.readonly.mr.azkaban.VoldemortStoreBuilderJob
  • voldemort.store.readonly.mr.HadoopStoreJobRunner
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.