Examples of LocalFileSystem


Examples of org.apache.hadoop.fs.LocalFileSystem

     * @throws Exception if failed
     */
    @Test
    public void input() throws Exception {
        final int count = 10000;
        LocalFileSystem fs = FileSystem.getLocal(conf);
        Path path = new Path(folder.newFile("testing").toURI());
        SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path, LongWritable.class, Text.class);
        try {
            LongWritable k = new LongWritable();
            Text v = new Text();
            for (int i = 0; i < count; i++) {
                k.set(i);
                v.set("Hello, world at " + i);
                writer.append(k, v);
            }
        } finally {
            writer.close();
        }

        ModelInput<StringOption> in = format.createInput(
                StringOption.class,
                fs,
                path,
                0,
                fs.getFileStatus(path).getLen(),
                new Counter());
        try {
            StringOption value = new StringOption();
            for (int i = 0; i < count; i++) {
                String answer = "Hello, world at " + i;
View Full Code Here

Examples of org.apache.hadoop.fs.LocalFileSystem

     */
    @Test
    public void input_fragment() throws Exception {
        final int count = 30000;
        Random rand = new Random();
        LocalFileSystem fs = FileSystem.getLocal(conf);
        Path path = new Path(folder.newFile("testing").toURI());
        SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path, LongWritable.class, Text.class);
        try {
            LongWritable k = new LongWritable();
            Text v = new Text();
            for (int i = 0; i < count; i++) {
                k.set(i);
                v.set("Hello, world at " + i);
                writer.append(k, v);
            }
        } finally {
            writer.close();
        }

        long fileLen = fs.getFileStatus(path).getLen();
        StringOption value = new StringOption();
        for (int attempt = 0; attempt < 5; attempt++) {
            int index = 0;
            long offset = 0;
            while (offset < fileLen) {
View Full Code Here

Examples of org.apache.hadoop.fs.LocalFileSystem

            buf.append("Hello, world!");
        }
        Text record = new Text(buf.toString());

        final int count = 5;
        LocalFileSystem fs = FileSystem.getLocal(conf);
        Path path = new Path(folder.newFile("testing").toURI());
        SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path, LongWritable.class, Text.class);
        try {
            LongWritable k = new LongWritable();
            Text v = new Text();
            for (int i = 0; i < count; i++) {
                k.set(i);
                v.set(record);
                writer.append(k, v);
            }
        } finally {
            writer.close();
        }

        long fileLen = fs.getFileStatus(path).getLen();
        StringOption value = new StringOption();
        int index = 0;
        long offset = 0;
        while (offset < fileLen) {
            long length = SequenceFile.SYNC_INTERVAL * 2;
 
View Full Code Here

Examples of org.apache.hadoop.fs.LocalFileSystem

     * Test for input empty file.
     * @throws Exception if failed
     */
    @Test
    public void input_empty() throws Exception {
        LocalFileSystem fs = FileSystem.getLocal(conf);
        Path path = new Path(folder.newFile("testing").toURI());
        fs.create(path).close();
        ModelInput<StringOption> in = format.createInput(
                StringOption.class,
                fs,
                path,
                0,
                fs.getFileStatus(path).getLen(),
                new Counter());
        try {
            assertThat("eof", in.readTo(new StringOption()), is(false));
        } finally {
            in.close();
View Full Code Here

Examples of org.apache.hadoop.fs.LocalFileSystem

     * Test for input invalid file.
     * @throws Exception if failed
     */
    @Test(expected = IOException.class)
    public void input_invalid() throws Exception {
        LocalFileSystem fs = FileSystem.getLocal(conf);
        Path path = new Path(folder.newFile("testing").toURI());
        FSDataOutputStream output = fs.create(path);
        try {
            output.writeUTF("Hello, world!");
        } finally {
            output.close();
        }
        ModelInput<StringOption> in = format.createInput(
                StringOption.class,
                fs,
                path,
                0,
                fs.getFileStatus(path).getLen(),
                new Counter());
        in.close();
    }
View Full Code Here

Examples of org.apache.hadoop.fs.LocalFileSystem

     */
    @SuppressWarnings("deprecation")
    @Test
    public void output() throws Exception {
        final int count = 10000;
        LocalFileSystem fs = FileSystem.getLocal(conf);
        Path path = new Path(folder.newFile("testing").toURI());
        ModelOutput<StringOption> out = format.createOutput(StringOption.class, fs, path, new Counter());
        try {
            StringOption value = new StringOption();
            for (int i = 0; i < count; i++) {
View Full Code Here

Examples of org.apache.hadoop.fs.LocalFileSystem

     * compressed output.
     * @throws Exception if failed
     */
    @Test
    public void output_compressed() throws Exception {
        LocalFileSystem fs = FileSystem.getLocal(conf);
        Path path = new Path(folder.newFile("testing").toURI());
        ModelOutput<StringOption> out = format.codec(new DefaultCodec())
            .createOutput(StringOption.class, fs, path, new Counter());
        try {
            out.write(new StringOption("Hello, world!"));
View Full Code Here

Examples of org.apache.hadoop.fs.LocalFileSystem

     * compressed output.
     * @throws Exception if failed
     */
    @Test
    public void output_no_compressed() throws Exception {
        LocalFileSystem fs = FileSystem.getLocal(conf);
        Path path = new Path(folder.newFile("testing.gz").toURI());
        ModelOutput<StringOption> out = format.codec(null)
            .createOutput(StringOption.class, fs, path, new Counter());
        try {
            out.write(new StringOption("Hello, world!"));
View Full Code Here

Examples of org.apache.hadoop.fs.LocalFileSystem

     * compressed output.
     * @throws Exception if failed
     */
    @Test
    public void output_compressed_conf() throws Exception {
        LocalFileSystem fs = FileSystem.getLocal(conf);
        Path path = new Path(folder.newFile("testing").toURI());
        format.getConf().set(SequenceFileFormat.KEY_COMPRESSION_CODEC, DefaultCodec.class.getName());
        ModelOutput<StringOption> out = format.createOutput(StringOption.class, fs, path, new Counter());
        try {
            out.write(new StringOption("Hello, world!"));
View Full Code Here

Examples of org.apache.hadoop.fs.LocalFileSystem

     * invalid compressed output.
     * @throws Exception if failed
     */
    @Test
    public void output_compressed_invalid() throws Exception {
        LocalFileSystem fs = FileSystem.getLocal(conf);
        Path path = new Path(folder.newFile("testing").toURI());
        format.getConf().set(SequenceFileFormat.KEY_COMPRESSION_CODEC, "__INVALID__");
        ModelOutput<StringOption> out = format.createOutput(StringOption.class, fs, path, new Counter());
        try {
            out.write(new StringOption("Hello, world!"));
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.