Examples of RecordOutputStream

Examples of com.backtype.hadoop.formats.RecordOutputStream

107
108
109
110
111
112
113
114
115
116
117
    private void writeStrings(Pail pail, String userfile, String... strs) throws IOException {
        writeStrings(pail, userfile, Arrays.asList(strs));
    }

    private void writeStrings(Pail pail, String userfile, Collection<String> strs) throws IOException {
        RecordOutputStream os = pail.openWrite(userfile);
        for(String s: strs) {
            os.writeRaw(s.getBytes());
        }
        os.close();
    }
View Full Code Here

Examples of com.backtype.hadoop.formats.RecordOutputStream

141
142
143
144
145
146
147
148
149
150
151
    }

    public void testAtomicity() throws Exception {
        String path = getTmpPath(local, "pail");
        Pail pail = Pail.create(local, path);
        RecordOutputStream os = pail.openWrite("aaa");
        assertEquals(0, pail.getUserFileNames().size());
        os.writeRaw(new byte[] {1, 2, 3});
        os.close();
        assertEquals(1, pail.getUserFileNames().size());
    }
View Full Code Here

Examples of com.backtype.hadoop.formats.RecordOutputStream

98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
        RecordStreamFactory factout;

        @Override
        protected void copyFile(FileSystem fsSource, Path source, FileSystem fsDest, Path target, Reporter reporter) throws IOException {
            RecordInputStream fin = factin.getInputStream(fsSource, source);
            RecordOutputStream fout = factout.getOutputStream(fsDest, target);

            try {
                byte[] record;
                int bytes = 0;
                while((record = fin.readRawRecord()) != null) {
                    fout.writeRaw(record);
                    bytes+=record.length;
                    if(bytes >= 1000000) { //every 1 MB of data report progress so we don't time out on large files
                        bytes = 0;
                        reporter.progress();
                    }
                }
            } finally {
                fin.close();
            }
            //don't complete files that aren't done yet. prevents partial files from being written
            fout.close();
        }
View Full Code Here

Examples of com.backtype.hadoop.formats.RecordOutputStream

43
44
45
46
47
48
49
50
                    throw new IllegalArgumentException("Cannot write object " + obj.toString() + " to " + p.toString() +
                            ". Conflicts with the structure of the datastore.");
                }
                _workers.put(targetDir, Pail.super.openWrite(p.toString(), _overwrite));
            }
            RecordOutputStream os = _workers.get(targetDir);
            os.writeRaw(structure.serialize(obj));
        }
View Full Code Here

Examples of org.openquark.cal.internal.serialization.RecordOutputStream

2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
     * @return TypeExpr
     * @throws IOException
     */
    private TypeExpr serializeDeserialize(TypeExpr inTypeExpr) throws IOException {
        NakedByteArrayOutputStream bos = new NakedByteArrayOutputStream(128);
        RecordOutputStream ros = new RecordOutputStream(bos);
        inTypeExpr.write(ros);
        ros.close();
       
        ByteArrayInputStream bis = new ByteArrayInputStream(bos.getByteArray());
        RecordInputStream ris = new RecordInputStream(bis);

        CompilerMessageLogger msgLogger = new MessageLogger();
View Full Code Here

Examples of org.openquark.cal.internal.serialization.RecordOutputStream

680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
        // Get the info file within that folder.
        ProgramResourceLocator.File compileModuleInfoFileLocator = moduleFolder.extendFile(module.getName() + "." + Module.COMPILED_MODULE_SUFFIX);

        // Put the module info into a byte array.
        NakedByteArrayOutputStream bos = new NakedByteArrayOutputStream(8192);
        RecordOutputStream ros = new RecordOutputStream(bos);

        Exception exception = null;
        try {
            module.write(ros);
        } catch (IOException saveException) {
            exception = saveException;
        } finally {
            try {
                ros.close();
            } catch (IOException ioe) {
                exception = ioe;
            }
        }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.