Package com.cloudera.flume.handlers.hdfs

Examples of com.cloudera.flume.handlers.hdfs.WriteableEvent

77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
   
    FlumeConfiguration conf = FlumeConfiguration.get();
    FileSystem fs = FileSystem.getLocal(conf);
    SequenceFile.Reader r = new SequenceFile.Reader(fs, new Path(f.toURI()), conf);
    WriteableEventKey k = new WriteableEventKey();
    WriteableEvent evt = new WriteableEvent();
    while (r.next(k, evt)) {
      Event expected = mem.next();
      assertEquals(evt.getTimestamp(), expected.getTimestamp());
      assertEquals(evt.getNanos(), expected.getNanos());
      assertEquals(evt.getPriority(), expected.getPriority());
      assertTrue(Arrays.equals(evt.getBody(), expected.getBody()));
    }

  }
View Full Code Here
55
56
57
58
59
60
61
62
63
64
    ByteArrayInputStream bais = new ByteArrayInputStream(bs);
    GZIPInputStream gzis = new GZIPInputStream(bais);
    DataInputStream dis = new DataInputStream(gzis);

    WriteableEvent out = new WriteableEvent();
    out.readFields(dis);
    dis.close();
    super.append(out);
  }
View Full Code Here
55
56
57
58
59
60
61
62
63
64
    int sz = ByteBuffer.wrap(e.get(BatchingDecorator.BATCH_SIZE)).getInt();
    byte[] data = e.get(BatchingDecorator.BATCH_DATA);
    DataInput in = new DataInputStream(new ByteArrayInputStream(data));
    for (int i = 0; i < sz; i++) {
      WriteableEvent we = new WriteableEvent();
      we.readFields(in);
      super.append(we);
    }
  }
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
157
158
    GunzipDecorator<EventSink> gunz = new GunzipDecorator<EventSink>(mem3);
    gunz.open();
    gunz.append(gzbe);
    Event gunze = mem3.next();

    int origsz = new WriteableEvent(be).toBytes().length;
    int gzipsz = new WriteableEvent(gzbe).toBytes().length;
    int ungzsz = new WriteableEvent(gunze).toBytes().length;

    LOG.info(String.format("before: %d  gzip: %d  gunzip: %d", origsz, gzipsz,
        ungzsz));

    Assert.assertTrue(origsz > gzipsz); // got some benefit for compressing?
View Full Code Here
101
102
103
104
105
106
107
108
109
110
111
    b.mark("hdfs_fileopen_started");

    Event e = null;
    while ((e = mem.next()) != null) {
      // writing
      w.append(new WriteableEventKey(e), new WriteableEvent(e));
    }
    w.close();
    b.mark("seqfile_hdfs_write");

    hdfs.close();
View Full Code Here
38
39
40
41
42
43
44
45
46
47
48
49
    super(s);
  }

  @Override
  public void append(Event e) throws IOException {
    WriteableEvent we = new WriteableEvent(e);
    byte[] bs = we.toBytes();

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    GZIPOutputStream gzos = new GZIPOutputStream(baos);
    gzos.write(bs);
    gzos.close();
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
68
69
  }

  @Override
  public void rawAppend(RawEvent evt) throws TException {
    try {
      WriteableEvent e = WriteableEvent.create(evt.getRaw().array());
      sink.append(e);
    } catch (IOException e) {
      e.printStackTrace();
      throw new TException("Caught IO exception " + e);
    }

  }
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
94
  Event batchevent(List<Event> evts) throws IOException {
    ByteArrayOutputStream baos = new ByteArrayOutputStream(2 >> 15);
    DataOutput out = new DataOutputStream(baos);
    for (Event evt : events) {
      WriteableEvent we = new WriteableEvent(evt);
      we.write(out);
    }

    Event be = new EventImpl(new byte[0]);
    ByteBuffer b = ByteBuffer.allocate(4);
    b.putInt(events.size());
View Full Code Here
TOP

Related Classes of com.cloudera.flume.handlers.hdfs.WriteableEvent

  • com.cloudera.flume.collector.SequenceFileMerger
  • com.cloudera.flume.core.EventImpl
  • com.cloudera.flume.handlers.batch.BatchingDecorator
  • com.cloudera.flume.handlers.batch.GunzipDecorator
  • com.cloudera.flume.handlers.batch.GzipDecorator
  • com.cloudera.flume.handlers.batch.TestBatching
  • com.cloudera.flume.handlers.batch.UnbatchingDecorator
  • com.cloudera.flume.handlers.seqfile.TestSequenceFileOutputFormat
  • com.cloudera.flume.handlers.thrift.ThriftFlumeEventServerImpl
  • com.cloudera.flume.handlers.thrift.ThriftRawEventSink
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.