Package com.linkedin.camus.etl.kafka.common

Examples of com.linkedin.camus.etl.kafka.common.EtlCounts


    private Logger log;

    public void addCounts(EtlKey key) throws IOException {
        String workingFileName = EtlMultiOutputFormat.getWorkingFileName(context, key);
        if (!counts.containsKey(workingFileName))
            counts.put(workingFileName, new EtlCounts(key.getTopic(),
            EtlMultiOutputFormat.getMonitorTimeGranularityMs(context)));
        counts.get(workingFileName).incrementMonitorCount(key);
        addOffset(key);
    }
View Full Code Here


            for (FileStatus f : fs.listStatus(workPath)) {
                String file = f.getPath().getName();
                log.info("work file: " + file);
                if (file.startsWith("data")) {
                    String workingFileName = file.substring(0, file.lastIndexOf("-m"));
                    EtlCounts count = counts.get(workingFileName);

                    String partitionedFile = getPartitionedPath(context, file,
                            count.getEventCount(), count.getLastKey().getOffset());

                    Path dest = new Path(baseOutDir, partitionedFile);

                    if (!fs.exists(dest.getParent())) {
                            fs.mkdirs(dest.getParent());
                        }

                    commitFile(context, f.getPath(), dest);
                    log.info("Moved file from: " + f.getPath() + " to: " + dest);

                    if (EtlMultiOutputFormat.isRunTrackingPost(context)) {
                            count.writeCountsToMap(allCountObject, fs, new Path(workPath, EtlMultiOutputFormat.COUNTS_PREFIX + "."
                                    + dest.getName().replace(recordWriterProvider.getFilenameExtension(), "")));
                    }
                }
            }
View Full Code Here

            });

        for (EtlCounts count : countsObjects) {
          String topic = count.getTopic();
          if (allCounts.containsKey(topic)) {
            EtlCounts existingCounts = allCounts.get(topic);
            existingCounts
                .setEndTime(Math.max(
                    existingCounts.getEndTime(),
                    count.getEndTime()));
            existingCounts.setLastTimestamp(Math.max(
                existingCounts.getLastTimestamp(),
                count.getLastTimestamp()));
            existingCounts.setStartTime(Math.min(
                existingCounts.getStartTime(),
                count.getStartTime()));
            existingCounts.setFirstTimestamp(Math.min(
                existingCounts.getFirstTimestamp(),
                count.getFirstTimestamp()));
            existingCounts.setErrorCount(existingCounts
                .getErrorCount() + count.getErrorCount());
            existingCounts.setGranularity(count.getGranularity());
            existingCounts.setTopic(count.getTopic());
            for (Entry<String, Source> entry : count.getCounts()
                .entrySet()) {
              Source source = entry.getValue();
              if (existingCounts.getCounts().containsKey(
                  source.toString())) {
                Source old = existingCounts.getCounts().get(
                    source.toString());
                old.setCount(old.getCount() + source.getCount());
                existingCounts.getCounts().put(old.toString(),
                    old);
              } else {
                existingCounts.getCounts().put(
                    source.toString(), source);
              }
              allCounts.put(topic, existingCounts);
            }
          } else {
View Full Code Here

TOP

Related Classes of com.linkedin.camus.etl.kafka.common.EtlCounts

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.