Examples of StorageHelper

Examples of com.google.api.ads.adwords.awreporting.kratubackend.data.StorageHelper

67
68
69
70
71
72
73
74
75
76
77
    account1.setTopAccountId(456L);
    account1.setIsCanManageClients(false);
    account1.setExternalCustomerId(123L);
    accountList = ImmutableList.of(account1, account1, account1);

    storageHelper = new StorageHelper();

    mockedRunnableKratu = new RunnableKratu(456L, accountList, storageHelper, dateStart, dateEnd);

    MockitoAnnotations.initMocks(this);
View Full Code Here

Examples of com.google.api.ads.adwords.awreporting.kratubackend.data.StorageHelper

63
64
65
66
67
68
69
70
71
72
73
  private final Date dateStart = DateUtil.parseDateTime("20140101").toDate();
  private final Date dateEnd = DateUtil.parseDateTime("20140131").toDate();

  @Before
  public void setUp() throws Exception {
    storageHelper = new StorageHelper(mockedEntitiesPersister);

    kratuProcessor = new KratuProcessor();

    MockitoAnnotations.initMocks(this);
View Full Code Here

Examples of com.netflix.aegisthus.tools.StorageHelper

198
199
200
201
202
203
204
205
206
207
    job.setMapOutputKeyClass(LongWritable.class);
    job.setMapOutputValueClass(Text.class);
    job.setMapperClass(Map.class);
    job.setReducerClass(Reduce.class);
    job.setPartitionerClass(Partition.class);
    StorageHelper sh = new StorageHelper(job.getConfiguration());
    LOG.info(String.format("temp location for job: %s", sh.getBaseTempLocation()));

    return job;
  }
View Full Code Here

Examples of com.netflix.aegisthus.tools.StorageHelper

264
265
266
267
268
269
270
271
272
273
274
275
      return 1;
    }

    Job job = initializeJob();
    String outputDir = cl.getOptionValue(OUTPUT);
    StorageHelper helper = new StorageHelper(job.getConfiguration());
    helper.setFinalPath(outputDir);

    checkOutputDirectory(job, outputDir, cl.hasOption(OPT_OVERWRITE));

    job.getConfiguration().setBoolean(CFG_PREFIX + OPT_PRIAM, cl.hasOption(OPT_PRIAM));
    if (cl.hasOption(OPT_RECURSIVE)) {
View Full Code Here

Examples of com.netflix.aegisthus.tools.StorageHelper

59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
  public static class Reduce extends Reducer<LongWritable, Text, Text, Text> {

    @Override
    protected void reduce(LongWritable key, Iterable<Text> values, Context ctx) throws IOException,
        InterruptedException {
      StorageHelper helper = new StorageHelper(ctx);
      boolean snappy = ctx.getConfiguration().getBoolean(CFG_PREFIX + OPT_PRIAM, false);
      String base = ctx.getConfiguration().get(CFG_PREFIX + OPT_RECURSIVE);
      if (base != null && !base.endsWith("/")) {
        base = base + "/";
      }
      for (Text file : values) {
        String fileString = cleanS3(file.toString().trim());
        if (base == null) {
          helper.copyToTemp(fileString, snappy);
        } else {
          String prefix = fileString.substring(base.length());
          prefix = prefix.replaceAll("/[^/]+$", "");
          helper.copyToTemp(fileString, prefix, snappy);
        }

      }
    }
View Full Code Here

Examples of com.netflix.aegisthus.tools.StorageHelper

23
24
25
26
27
28
29
30
31
32
33
34
35
36
  }

  @Override
  public void abortJob(JobContext job, State state) throws IOException {
    LOG.info("aborting job");
    StorageHelper sh = new StorageHelper(job.getConfiguration());
    try {
      LOG.info("deleting committed files");
      sh.deleteCommitted();
    } finally {
      LOG.info("deleting temp files");
      sh.deleteBaseTempLocation();
    }
  }
View Full Code Here

Examples of com.netflix.aegisthus.tools.StorageHelper

36
37
38
39
40
41
42
43
  }

  @Override
  public void commitJob(JobContext job) throws IOException {
    LOG.info("committing job");
    StorageHelper sh = new StorageHelper(job.getConfiguration());
    sh.deleteBaseTempLocation();
  }
View Full Code Here

Examples of com.netflix.aegisthus.tools.StorageHelper

43
44
45
46
47
48
49
50
  }

  @Override
  public void commitTask(TaskAttemptContext ctx) throws IOException {
    LOG.info("committing task");
    StorageHelper sh = new StorageHelper(ctx);
    sh.moveTaskOutputToFinal();
  }
View Full Code Here

Examples of com.netflix.aegisthus.tools.StorageHelper

57
58
59
60
61
62
63
64
    return true;
  }

  @Override
  public void setupJob(JobContext job) throws IOException {
    StorageHelper sh = new StorageHelper(job.getConfiguration());
    LOG.info(String.format("temp location for job: %s", sh.getBaseTempLocation()));
  }
View Full Code Here

Examples of com.netflix.aegisthus.tools.StorageHelper

63
64
65
66
67
68
69
70
    LOG.info(String.format("temp location for job: %s", sh.getBaseTempLocation()));
  }

  @Override
  public void setupTask(TaskAttemptContext ctx) throws IOException {
    StorageHelper sh = new StorageHelper(ctx);
    LOG.info(String.format("temp location for task: %s", sh.getBaseTaskAttemptTempLocation()));
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.