Examples of FileSplit

com.cloudera.iterativereduce.yarn.avro.generated.FileSplit
edu.uci.ics.hyracks.dataflow.std.file.FileSplit
org.apache.hadoop.mapred.FileSplit
A section of an input file. Returned by {@link InputFormat#getSplits(JobConf,int)} and passed to{@link InputFormat#getRecordReader(InputSplit,JobConf,Reporter)}.
org.apache.hadoop.mapreduce.lib.input.FileSplit
A section of an input file. Returned by {@link InputFormat#getSplits(JobContext)} and passed to{@link InputFormat#createRecordReader(InputSplit,TaskAttemptContext)}.
org.apache.hama.bsp.FileSplit

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit

    }


    private static FileSplit getSplit(BlockMap blockMap, Path path, long start, long end) {
        DirectInputFragment f = blockMap.get(start, end);
        List<String> owners = f.getOwnerNodeNames();
        FileSplit split = new FileSplit(
                path, start, end - start,
                owners.toArray(new String[owners.size()]));
        return split;
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit


    @Override
    public RecordReader<NullWritable, T> createRecordReader(
            InputSplit split,
            TaskAttemptContext context) throws IOException, InterruptedException {
        FileSplit s = (FileSplit) split;
        assert s.getStart() % TemporaryFile.BLOCK_SIZE == 0;
        assert s.getStart() > 0 || s.getLength() > 0;
        return createRecordReader();
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit

        }


        @SuppressWarnings("unchecked")
        @Override
        public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
            FileSplit s = (FileSplit) split;
            this.size = s.getLength();
            Path path = s.getPath();
            FileSystem fs = path.getFileSystem(context.getConfiguration());
            int blocks = computeBlocks(s);
            FSDataInputStream stream = fs.open(path);
            boolean succeed = false;
            try {
                if (s.getStart() != 0) {
                    assert s.getStart() % TemporaryFile.BLOCK_SIZE == 0;
                    stream.seek(s.getStart());
                }
                this.input = (TemporaryFileInput<T>) new TemporaryFileInput<Writable>(stream, blocks);
                Class<?> aClass = context.getConfiguration().getClassByName(input.getDataTypeName());
                this.value = (T) ReflectionUtils.newInstance(aClass, context.getConfiguration());
                succeed = true;

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit

    public void splits_simple() {
        BlockMap blocks = blocks("testing", m(10));
        List<FileSplit> splits = TemporaryInputFormat.computeSplits(new Path("testing"), blocks, m(64));


        assertThat(splits, hasSize(1));
        FileSplit s0 = find(splits, 0);
        assertThat(s0.getLength(), is(m(10)));
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit

        BlockMap blocks = blocks("testing", TemporaryFile.BLOCK_SIZE, TemporaryFile.BLOCK_SIZE);
        List<FileSplit> splits = TemporaryInputFormat.computeSplits(new Path("testing"), blocks, m(64));


        assertThat(splits, hasSize(2));


        FileSplit s0 = find(splits, 0);
        assertThat(s0.getLength(), is((long) TemporaryFile.BLOCK_SIZE));


        FileSplit s1 = find(splits, TemporaryFile.BLOCK_SIZE);
        assertThat(s1.getLength(), is((long) TemporaryFile.BLOCK_SIZE));
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit

        BlockMap blocks = blocks("testing", TemporaryFile.BLOCK_SIZE - 10, TemporaryFile.BLOCK_SIZE);
        List<FileSplit> splits = TemporaryInputFormat.computeSplits(new Path("testing"), blocks, m(128));


        assertThat(splits, hasSize(2));


        FileSplit s0 = find(splits, 0);
        assertThat(s0.getLength(), is((long) TemporaryFile.BLOCK_SIZE));


        FileSplit s1 = find(splits, TemporaryFile.BLOCK_SIZE);
        assertThat(s1.getLength(), is((long) TemporaryFile.BLOCK_SIZE - 10));
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit

        BlockMap blocks = blocks("testing", TemporaryFile.BLOCK_SIZE, TemporaryFile.BLOCK_SIZE + 10);
        List<FileSplit> splits = TemporaryInputFormat.computeSplits(new Path("testing"), blocks, m(64));


        assertThat(splits, hasSize(2));


        FileSplit s0 = find(splits, 0);
        assertThat(s0.getLength(), is((long) TemporaryFile.BLOCK_SIZE));


        FileSplit s1 = find(splits, TemporaryFile.BLOCK_SIZE);
        assertThat(s1.getLength(), is((long) TemporaryFile.BLOCK_SIZE + 10));
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit

        List<FileSplit> splits = TemporaryInputFormat.computeSplits(
                new Path("testing"), blocks, TemporaryFile.BLOCK_SIZE + 1);


        assertThat(splits, hasSize(5));


        FileSplit s0 = find(splits, TemporaryFile.BLOCK_SIZE * 0);
        assertThat(s0.getLength(), is((long) TemporaryFile.BLOCK_SIZE * 2));
        FileSplit s1 = find(splits, TemporaryFile.BLOCK_SIZE * 2);
        assertThat(s1.getLength(), is((long) TemporaryFile.BLOCK_SIZE * 2));
        FileSplit s2 = find(splits, TemporaryFile.BLOCK_SIZE * 4);
        assertThat(s2.getLength(), is((long) TemporaryFile.BLOCK_SIZE * 2));
        FileSplit s3 = find(splits, TemporaryFile.BLOCK_SIZE * 6);
        assertThat(s3.getLength(), is((long) TemporaryFile.BLOCK_SIZE * 2));
        FileSplit s4 = find(splits, TemporaryFile.BLOCK_SIZE * 8);
        assertThat(s4.getLength(), is((long) TemporaryFile.BLOCK_SIZE * 2));
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit

    public void splits_suppress() {
        BlockMap blocks = blocks("testing", TemporaryFile.BLOCK_SIZE * 10);
        List<FileSplit> splits = TemporaryInputFormat.computeSplits(new Path("testing"), blocks, 0);


        assertThat(splits, hasSize(1));
        FileSplit s0 = find(splits, 0);
        assertThat(s0.getLength(), is((long) TemporaryFile.BLOCK_SIZE * 10));
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit

        Configuration conf = new ConfigurationProvider().newInstance();
        FileStatus stat = write(conf, 1);
        RecordReader<NullWritable, Text> reader = TemporaryInputFormat.createRecordReader();
        try {
            reader.initialize(
                    new FileSplit(stat.getPath(), 0, stat.getLen(), null),
                    JobCompatibility.newTaskAttemptContext(conf, id()));


            assertThat(reader.nextKeyValue(), is(true));
            assertThat(reader.getCurrentValue(), is(new Text("Hello, world!")));

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.