Package org.apache.hadoop.typedbytes

Examples of org.apache.hadoop.typedbytes.TypedBytesOutput

241
242
243
244
245
246
247
248
249
250
251
252
253
        }
       
        protected TypedBytesWritable randomKey() throws IOException {
            ByteArrayOutputStream bytes = new ByteArrayOutputStream();
           
            TypedBytesOutput out =
                new TypedBytesOutput(new DataOutputStream(bytes));
            out.writeInt(rand.nextInt(2000000000));
           
            TypedBytesWritable val =
                new TypedBytesWritable(bytes.toByteArray());
           
            return val;
View Full Code Here
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
       
        protected TypedBytesWritable encodeTypedBytes(double array[])
            throws IOException {
            ByteArrayOutputStream bytes = new ByteArrayOutputStream();
           
            TypedBytesOutput out =
                new TypedBytesOutput(new DataOutputStream(bytes));
                   
            out.writeVectorHeader(array.length);
            for (int i=0; i<array.length; ++i) {
                out.writeDouble(array[i]);
            }
           
            TypedBytesWritable val =
                new TypedBytesWritable(bytes.toByteArray());
           
View Full Code Here
41
42
43
44
45
46
47
48
49
50
51
52
53
54
    Configuration conf = new Configuration();
    MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null);
    FileSystem fs = cluster.getFileSystem();
   
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    TypedBytesOutput tboutput = new TypedBytesOutput(new DataOutputStream(out));
    for (int i = 0; i < 100; i++) {
      tboutput.write(new Long(i)); // key
      tboutput.write("" + (10 * i)); // value
    }
    InputStream isBackup = System.in;
    ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
    System.setIn(in);
    LoadTypedBytes loadtb = new LoadTypedBytes(conf);
View Full Code Here
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
    this.find = find;
  }

  public void go() throws IOException {
    TypedBytesInput tbinput = new TypedBytesInput(new DataInputStream(System.in));
    TypedBytesOutput tboutput = new TypedBytesOutput(new DataOutputStream(System.out));

    Object key = tbinput.readRaw();
    while (key != null) {
      Object value = tbinput.read();
      for (String part : value.toString().split(find)) {
        tboutput.write(part);  // write key
        tboutput.write(1);     // write value
      }
      System.err.println("reporter:counter:UserCounters,InputLines,1");
      key = tbinput.readRaw();
    }
   
View Full Code Here
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
public class TypedBytesReduceApp {

  public void go() throws IOException {
    TypedBytesInput tbinput = new TypedBytesInput(new DataInputStream(System.in));
    TypedBytesOutput tboutput = new TypedBytesOutput(new DataOutputStream(System.out));
   
    Object prevKey = null;
    int sum = 0;
    Object key = tbinput.read();
    while (key != null) {
      if (prevKey != null && !key.equals(prevKey)) {
        tboutput.write(prevKey);  // write key
        tboutput.write(sum);      // write value
        sum = 0;
      }
      sum += (Integer) tbinput.read();
      prevKey = key;
      key = tbinput.read();
    }
    tboutput.write(prevKey);
    tboutput.write(sum);
   
    System.out.flush();
  }
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
53
54
55
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2)
        .build();
    FileSystem fs = cluster.getFileSystem();
   
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    TypedBytesOutput tboutput = new TypedBytesOutput(new DataOutputStream(out));
    for (int i = 0; i < 100; i++) {
      tboutput.write(new Long(i)); // key
      tboutput.write("" + (10 * i)); // value
    }
    InputStream isBackup = System.in;
    ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
    System.setIn(in);
    LoadTypedBytes loadtb = new LoadTypedBytes(conf);
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
50
51
52
    Configuration conf = new Configuration();
    MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null);
    FileSystem fs = cluster.getFileSystem();
   
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    TypedBytesOutput tboutput = new TypedBytesOutput(new DataOutputStream(out));
    for (int i = 0; i < 100; i++) {
      tboutput.write(new Long(i)); // key
      tboutput.write("" + (10 * i)); // value
    }
    InputStream isBackup = System.in;
    ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
    System.setIn(in);
    LoadTypedBytes loadtb = new LoadTypedBytes(conf);
View Full Code Here
36
37
38
39
40
41
42
43
  @Override
  public void initialize(PipeMapRed pipeMapRed) throws IOException {
    super.initialize(pipeMapRed);
    DataOutput clientOut = pipeMapRed.getClientOutput();
    tbOut = new TypedBytesOutput(clientOut);
    tbwOut = new TypedBytesWritableOutput(clientOut);
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.typedbytes.TypedBytesOutput

  • gov.sandia.dfgleic.TSQR$TSQRIteration
  • org.apache.hadoop.streaming.io.TypedBytesInputWriter
  • org.apache.hadoop.streaming.TestLoadTypedBytes
  • org.apache.hadoop.streaming.TypedBytesMapApp
  • org.apache.hadoop.streaming.TypedBytesReduceApp
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.