Package com.cloudera.flume.core

Examples of com.cloudera.flume.core.EventSource


    File f = File.createTempFile("prepend", "bar");
    f.deleteOnExit();
    FileWriter fw = new FileWriter(f);

    ExecNioSource.Builder builder = new ExecNioSource.Builder();
    EventSource source = builder.build("tail -F " + f.getAbsolutePath());
    source.open();

    fw.write("foo\n");
    fw.flush();
    Event e = source.next();
    assertTrue(Arrays.equals("foo".getBytes(), e.getBody()));

    fw.write("bar\n");
    fw.flush();
    e = source.next();
    assertTrue(Arrays.equals("bar".getBytes(), e.getBody()));

    fw.write("baz\n");
    fw.flush();
    e = source.next();
    assertTrue(Arrays.equals("baz".getBytes(), e.getBody()));

    source.close();
  }
View Full Code Here


    File f = File.createTempFile("truncate", ".bar");
    f.deleteOnExit();
    FileWriter fw = new FileWriter(f);

    ExecNioSource.Builder builder = new ExecNioSource.Builder();
    EventSource source = builder.build("tail -F " + f.getAbsolutePath());
    source.open();

    fw.write("foo\n");
    fw.flush();
    Event e = source.next();
    assertTrue(Arrays.equals("foo".getBytes(), e.getBody()));

    // this case inserts an extremely long line.
    int max = (int) FlumeConfiguration.get().getEventMaxSizeBytes();
    int tooBig = max * 10;
    byte[] data = new byte[tooBig];
    for (int i = 0; i < data.length; i++) {
      data[i] = 'a';
    }
    fw.write(new String(data) + "\n");
    fw.flush();
    e = source.next();
    assertEquals(max, e.getBody().length);

    // back to previous test
    fw.write("baz\n");
    fw.flush();
    e = source.next();
    assertTrue(Arrays.equals("baz".getBytes(), e.getBody()));

    source.close();
  }
View Full Code Here

   *
   * @throws InterruptedException
   */
  @Test
  public void checkSynth() throws IOException, InterruptedException {
    EventSource src = new SynthSource(5, 10, 1337);
    Event e = null;
    EventSink snk = new ConsoleEventSink(new AvroJsonOutputFormat());
    MemorySinkSource mem = new MemorySinkSource();
    while ((e = src.next()) != null) {
      snk.append(e); // visual inspection
      mem.append(e); // testing
    }

    mem.open();
View Full Code Here

   *
   * @throws InterruptedException
   */
  @Test
  public void checkAttrSynth() throws IOException, InterruptedException {
    EventSource src = new AttrSynthSource(5, 10, 20, 15, 1337);
    Event e = null;
    EventSink snk = new ConsoleEventSink(new AvroJsonOutputFormat());
    MemorySinkSource mem = new MemorySinkSource();
    while ((e = src.next()) != null) {
      snk.append(e); // visual inspection
      mem.append(e); // testing
    }

    mem.open();
View Full Code Here

    LOG.info("opening new file for " + sendingTag);
    changeState(sendingTag, State.LOGGED, State.SENDING);
    sendingCount.incrementAndGet();
    File curFile = getFile(sendingTag);
    EventSource curSource = new SeqfileEventSource(curFile.getAbsolutePath());
    return new StateChangeDeco(curSource, sendingTag);
  }
View Full Code Here

   *
   * @throws InterruptedException
   */
  @Test
  public void testThriftSend() throws IOException, InterruptedException {
    EventSource txt = new NoNlASCIISynthSource(25, 100);
    txt.open();
    MemorySinkSource mem = new MemorySinkSource();
    mem.open();
    EventUtil.dumpAll(txt, mem);
    txt.close();

    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(
        conf.getCollectorPort() + 1); // this is a slight
    // tweak to avoid port conflicts
View Full Code Here

   * @throws InterruptedException
   *
   */
  @Test
  public void testThriftRawSend() throws IOException, InterruptedException {
    EventSource txt = new NoNlASCIISynthSource(25, 100);
    txt.open();
    MemorySinkSource mem = new MemorySinkSource();
    mem.open();
    EventUtil.dumpAll(txt, mem);
    txt.close();

    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(conf.getCollectorPort());
    tes.open();

View Full Code Here

      Thread th = new Thread() {
        public void run() {
          try {
            // TODO (jon) this may have different sizes due to the host it is
            // running on . Needs to be fixed.
            EventSource txt = new NoNlASCIISynthSource(25, 100);

            txt.open();
            MemorySinkSource mem = new MemorySinkSource();
            mem.open();
            EventUtil.dumpAll(txt, mem);
            txt.close();

            // mem -> ThriftEventSink
            ThriftEventSink snk = new ThriftEventSink("0.0.0.0", conf
                .getCollectorPort() + 1);
            snk.open();
View Full Code Here

public class TestBenchmarkDeco implements ExampleData {

  @Test
  public void testSimpleMem() throws IOException, InterruptedException {

    EventSource src = new NoNlASCIISynthSource(25, 100, 1);
    EventSink snk = new ConsoleEventSink();
    EventSink snk2 = new BenchmarkReportDecorator<EventSink>("report", snk);
    EventSink snk3 = new BenchmarkInjectDecorator<EventSink>(snk2);
    EventSink snk4 = new InMemoryDecorator<EventSink>(snk3);

    DirectDriver connect = new DirectDriver(src, snk4);
    src.open();
    snk4.open();
    connect.start();
    connect.join(Long.MAX_VALUE);
    src.close();
    snk4.close();
    snk2.getMetrics().toText(new OutputStreamWriter(System.err));
  }
View Full Code Here

    snk2.getMetrics().toText(new OutputStreamWriter(System.err));
  }

  @Test
  public void testSimple() throws IOException, InterruptedException {
    EventSource src = new NoNlASCIISynthSource(25, 100, 1);
    EventSink snk = new ConsoleEventSink();
    EventSink snk2 = new BenchmarkReportDecorator<EventSink>("report", snk);
    EventSink snk3 = new BenchmarkInjectDecorator<EventSink>(snk2);

    DirectDriver connect = new DirectDriver(src, snk3);
    src.open();
    snk3.open();
    connect.start();
    connect.join(Long.MAX_VALUE);
    src.close();
    snk3.close();
    snk2.getMetrics().toText(new OutputStreamWriter(System.err));
  }
View Full Code Here

TOP

Related Classes of com.cloudera.flume.core.EventSource

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.