Examples of AegSplit


Examples of com.netflix.aegisthus.columnar_input.splits.AegSplit

public class AegisthusInputFormat extends FileInputFormat<CompositeKey, AtomWritable> {
    private static final Logger LOG = LoggerFactory.getLogger(AegisthusInputFormat.class);

    @Override
    public RecordReader<CompositeKey, AtomWritable> createRecordReader(InputSplit inputSplit, TaskAttemptContext context) {
        AegSplit split = null;
        if (inputSplit instanceof AegCombinedSplit) {
            return new CombineSSTableReader();
        }
        split = (AegSplit) inputSplit;
        RecordReader<CompositeKey, AtomWritable> reader = null;
        switch (split.getType()) {
        case sstable:
            reader = new SSTableRecordReader();
            break;
        case commitlog:
            reader = new CommitLogRecordReader();
View Full Code Here

Examples of com.netflix.aegisthus.columnar_input.splits.AegSplit

                        newIndexOffset = pair.right;

                    }
                    int blkIndex = getBlockIndex(blkLocations, splitStart + (splitSize / 2));
                    LOG.debug("split path: {}:{}:{}", path.getName(), splitStart, splitSize);
                    splits.add(new AegSplit(path, splitStart, splitSize, blkLocations[blkIndex].getHosts()));
                    indexOffset = newIndexOffset;
                    bytesRemaining -= splitSize;
                    splitStart += splitSize;
                }
                if (scanner != null) {
                    scanner.close();
                }
            }

            if (bytesRemaining != 0) {
                LOG.debug("end path: {}:{}:{}", path.getName(), length - bytesRemaining, bytesRemaining);
                if (fs.exists(compressionPath)) {
                    splits.add(new AegCompressedSplit(path, length - bytesRemaining, bytesRemaining,
                            blkLocations[blkLocations.length - 1].getHosts(), compressionPath));
                } else {
                    splits.add(new AegSplit(path, length - bytesRemaining, bytesRemaining,
                            blkLocations[blkLocations.length - 1].getHosts()));
                }
            }
        } else {
            LOG.info("skipping zero length file: {}", path.toString());
View Full Code Here

Examples of com.netflix.aegisthus.columnar_input.splits.AegSplit

            } else if (name.startsWith("CommitLog")) {
                LOG.info(String.format("adding %s as a CommitLog split", file.getPath().toUri().toString()));
                BlockLocation[] blkLocations = file.getPath()
                        .getFileSystem(job.getConfiguration())
                        .getFileBlockLocations(file, 0, file.getLen());
                splits.add(new AegSplit(file.getPath(), 0, file.getLen(), blkLocations[0].getHosts(), Type.commitlog));
            }
        }
        return splits;
    }
View Full Code Here

Examples of com.netflix.aegisthus.columnar_input.splits.AegSplit

    @SuppressWarnings("rawtypes")
    @Override
    public void initialize(InputSplit inputSplit, final TaskAttemptContext ctx) throws IOException,
            InterruptedException {
        AegSplit split = (AegSplit) inputSplit;

        start = split.getStart();
        end = split.getEnd();
        final Path file = split.getPath();

        try {
            cfId = ctx.getConfiguration().getInt("commitlog.cfid", -1000);
            if (cfId == -1000) {
                throw new IOException("commitlog.cfid must be set");
            }
            // open the file and seek to the start of the split
            FileSystem fs = file.getFileSystem(ctx.getConfiguration());
            FSDataInputStream fileIn = fs.open(split.getPath());
            InputStream dis = new BufferedInputStream(fileIn);
            scanner = new CommitLogColumnarScanner(new DataInputStream(dis),
                    Descriptor.fromFilename(split.getPath().getName()).version, cfId);
            this.pos = start;
            iterator = scanner.observable()
                    .onErrorFlatMap(new Func1<OnErrorThrowable, Observable<? extends AtomWritable>>() {
                        @Override
                        public Observable<? extends AtomWritable> call(OnErrorThrowable onErrorThrowable) {
View Full Code Here

Examples of com.netflix.aegisthus.columnar_input.splits.AegSplit

    }

    @Override
    public void initialize(InputSplit inputSplit, final TaskAttemptContext ctx) throws IOException,
            InterruptedException {
        AegSplit split = (AegSplit) inputSplit;

        start = split.getStart();
        InputStream is = split.getInput(ctx.getConfiguration());
        end = split.getDataEnd();
        String filename = split.getPath().toUri().toString();

        LOG.info(String.format("File: %s", split.getPath().toUri().getPath()));
        LOG.info("Start: " + start);
        LOG.info("End: " + end);

        try {
            DataInput indexInput;
View Full Code Here

Examples of com.netflix.aegisthus.columnar_input.splits.AegSplit

        int cnt = 0;
        int cntAdded = 0;

        top:
        for (InputSplit split : splits) {
            AegSplit aegSplit = (AegSplit) split;
            switch (aegSplit.getType()) {
            case sstable:
                cnt++;
                if (aegSplit.getLength() >= MAX_SPLIT_SIZE) {
                    cntAdded++;
                    combinedSplits.add(aegSplit);
                    continue;
                }
                try {
                    String lastLocation = null;
                    for (String location : aegSplit.getLocations()) {
                        lastLocation = location;
                        if (map.containsKey(location)) {
                            AegCombinedSplit temp = map.get(location);
                            cntAdded++;
                            temp.getSplits().add(aegSplit);
View Full Code Here

Examples of com.netflix.aegisthus.input.AegSplit

    }
  }

  @Override
  public void initialize(InputSplit inputSplit, TaskAttemptContext ctx) throws IOException, InterruptedException {
    AegSplit split = (AegSplit) inputSplit;
    InputStream is = split.getInput(ctx.getConfiguration());
    start = split.getStart();
    end = split.getEnd();
    pos = start;
    is.skip(split.getStart());
    if (split.getPath().getName().endsWith(".gz")) {
      reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(is)));
    } else {
      reader = new BufferedReader(new InputStreamReader(is));
    }
  }
View Full Code Here

Examples of com.netflix.aegisthus.input.AegSplit

        }
    }

    @Override
    public void initialize(InputSplit inputSplit, TaskAttemptContext ctx) throws IOException, InterruptedException {
        AegSplit split = (AegSplit) inputSplit;

        start = split.getStart();
        end = split.getEnd();
        final Path file = split.getPath();

        try {
            cfId = ctx.getConfiguration().getInt("commitlog.cfid", -1000);
            if (cfId == -1000) {
                throw new IOException("commitlog.cfid must be set");
            }
            // open the file and seek to the start of the split
            FileSystem fs = file.getFileSystem(ctx.getConfiguration());
            FSDataInputStream fileIn = fs.open(split.getPath());
            InputStream dis = new BufferedInputStream(fileIn);
            scanner = new CommitLogScanner(new DataInputStream(dis), split.getConvertors(),
                    Descriptor.fromFilename(split.getPath().getName()).version);
            this.pos = start;
        } catch (IOException e) {
            throw new IOError(e);
        }
    }
View Full Code Here

Examples of com.netflix.aegisthus.input.AegSplit

  }

  @SuppressWarnings("rawtypes")
  @Override
  public void initialize(InputSplit inputSplit, TaskAttemptContext ctx) throws IOException, InterruptedException {
    AegSplit split = (AegSplit) inputSplit;

    start = split.getStart();
    //TODO: This has a side effect of setting compressionmetadata. remove this.
    InputStream is = split.getInput(ctx.getConfiguration());
    if (split.isCompressed()) {
      end = split.getCompressionMetadata().getDataLength();
    } else {
      end = split.getEnd();
    }
    outputFile = ctx.getConfiguration().getBoolean("aegsithus.debug.file", false);
    filename = split.getPath().toUri().toString();

    LOG.info(String.format("File: %s", split.getPath().toUri().getPath()));
    LOG.info("Start: " + start);
    LOG.info("End: " + end);
    if (ctx instanceof TaskInputOutputContext) {
      context = (TaskInputOutputContext) ctx;
    }

    try {
      scanner = new SSTableScanner(new DataInputStream(is),
          split.getConvertors(), end, Descriptor.fromFilename(filename).version);
      if (ctx.getConfiguration().get("aegisthus.maxcolsize") != null) {
        scanner.setMaxColSize(ctx.getConfiguration().getLong("aegisthus.maxcolsize", -1L));
        LOG.info(String.format("aegisthus.maxcolsize - %d",
            ctx.getConfiguration().getLong("aegisthus.maxcolsize", -1L)));
      }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.