Package com.netflix.aegisthus.tools

Examples of com.netflix.aegisthus.tools.StorageHelper

198
199
200
201
202
203
204
205
206
207
    job.setMapOutputKeyClass(LongWritable.class);
    job.setMapOutputValueClass(Text.class);
    job.setMapperClass(Map.class);
    job.setReducerClass(Reduce.class);
    job.setPartitionerClass(Partition.class);
    StorageHelper sh = new StorageHelper(job.getConfiguration());
    LOG.info(String.format("temp location for job: %s", sh.getBaseTempLocation()));

    return job;
  }
View Full Code Here
264
265
266
267
268
269
270
271
272
273
274
275
      return 1;
    }

    Job job = initializeJob();
    String outputDir = cl.getOptionValue(OUTPUT);
    StorageHelper helper = new StorageHelper(job.getConfiguration());
    helper.setFinalPath(outputDir);

    checkOutputDirectory(job, outputDir, cl.hasOption(OPT_OVERWRITE));

    job.getConfiguration().setBoolean(CFG_PREFIX + OPT_PRIAM, cl.hasOption(OPT_PRIAM));
    if (cl.hasOption(OPT_RECURSIVE)) {
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
  public static class Reduce extends Reducer<LongWritable, Text, Text, Text> {

    @Override
    protected void reduce(LongWritable key, Iterable<Text> values, Context ctx) throws IOException,
        InterruptedException {
      StorageHelper helper = new StorageHelper(ctx);
      boolean snappy = ctx.getConfiguration().getBoolean(CFG_PREFIX + OPT_PRIAM, false);
      String base = ctx.getConfiguration().get(CFG_PREFIX + OPT_RECURSIVE);
      if (base != null && !base.endsWith("/")) {
        base = base + "/";
      }
      for (Text file : values) {
        String fileString = cleanS3(file.toString().trim());
        if (base == null) {
          helper.copyToTemp(fileString, snappy);
        } else {
          String prefix = fileString.substring(base.length());
          prefix = prefix.replaceAll("/[^/]+$", "");
          helper.copyToTemp(fileString, prefix, snappy);
        }

      }
    }
View Full Code Here
23
24
25
26
27
28
29
30
31
32
33
34
35
36
  }

  @Override
  public void abortJob(JobContext job, State state) throws IOException {
    LOG.info("aborting job");
    StorageHelper sh = new StorageHelper(job.getConfiguration());
    try {
      LOG.info("deleting committed files");
      sh.deleteCommitted();
    } finally {
      LOG.info("deleting temp files");
      sh.deleteBaseTempLocation();
    }
  }
View Full Code Here
36
37
38
39
40
41
42
43
  }

  @Override
  public void commitJob(JobContext job) throws IOException {
    LOG.info("committing job");
    StorageHelper sh = new StorageHelper(job.getConfiguration());
    sh.deleteBaseTempLocation();
  }
View Full Code Here
43
44
45
46
47
48
49
50
  }

  @Override
  public void commitTask(TaskAttemptContext ctx) throws IOException {
    LOG.info("committing task");
    StorageHelper sh = new StorageHelper(ctx);
    sh.moveTaskOutputToFinal();
  }
View Full Code Here
57
58
59
60
61
62
63
64
    return true;
  }

  @Override
  public void setupJob(JobContext job) throws IOException {
    StorageHelper sh = new StorageHelper(job.getConfiguration());
    LOG.info(String.format("temp location for job: %s", sh.getBaseTempLocation()));
  }
View Full Code Here
63
64
65
66
67
68
69
70
    LOG.info(String.format("temp location for job: %s", sh.getBaseTempLocation()));
  }

  @Override
  public void setupTask(TaskAttemptContext ctx) throws IOException {
    StorageHelper sh = new StorageHelper(ctx);
    LOG.info(String.format("temp location for task: %s", sh.getBaseTaskAttemptTempLocation()));
  }
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
78
        ctx.getCounter("aegisthus", "rows_written").increment(1L);
    }

    @Override
    protected void setup(Context context) throws IOException, InterruptedException {
        StorageHelper sh = new StorageHelper(context);
        Path outputDir = new Path(sh.getBaseTaskAttemptTempLocation());
        FileSystem fs = outputDir.getFileSystem(context.getConfiguration());
        String filename = String.format("%s-%s-%05d0%04d-Data.db",
                context.getConfiguration().get("aegisthus.dataset", "keyspace-dataset"), Version.current_version,
                context.getTaskAttemptID().getTaskID().getId(), context.getTaskAttemptID().getId());
        Path outputFile = new Path(outputDir, filename);
View Full Code Here
247
248
249
250
251
252
253
254
255
256
257
258
259
        }
        TextInputFormat.setInputPaths(job, paths.toArray(new Path[paths.size()]));
        Path temp = new Path("/tmp/" + UUID.randomUUID());
        TextOutputFormat.setOutputPath(job, temp);

        StorageHelper sh = new StorageHelper(job.getConfiguration());
        sh.setFinalPath(cl.getOptionValue(OPT_OUTPUT));
        LOG.info("temp location for job: {}", sh.getBaseTempLocation());

        job.submit();
        System.out.println(job.getJobID());
        System.out.println(job.getTrackingURL());
        boolean success = job.waitForCompletion(true);
View Full Code Here
TOP

Related Classes of com.netflix.aegisthus.tools.StorageHelper

  • com.netflix.Aegisthus
  • com.netflix.aegisthus.mapred.reduce.CassSSTableReducer
  • com.netflix.hadoop.output.CleanOutputCommitter
  • Distcp
  • Distcp$Reduce
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.Path
  • java.io.InputStreamReader
  • java.io.BufferedReader
  • java.io.DataOutputStream
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.