Package com.cloudera.flume.core

Examples of com.cloudera.flume.core.EventSource


    // EventSource src = FlumeBuilder.buildSource("");
    EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(
        LogicalNodeContext.testingContext()),
        "{ ackedWriteAhead => { ackChecker => counter(\"count\") } }");
    EventSource src = MemorySinkSource.cannedData("foo foo foo ", 5);
    snk.open();
    src.open();
    EventUtil.dumpAll(src, snk);
    src.close();
    snk.close(); // this should block until recovery complete.

    // agent checks for ack registrations.
    BenchmarkHarness.node.getAckChecker().checkAcks();
View Full Code Here


    // /////////////////////

    EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(
        LogicalNodeContext.testingContext()),
        "{ ackedWriteAhead => { ackChecker => counter(\"count\") } }");
    EventSource src = MemorySinkSource.cannedData("foo foo foo ", 5);
    snk.open();
    src.open();
    EventUtil.dumpAll(src, snk);
    src.close();
    snk.close(); // this should block until recovery complete.

    // agent checks for ack registrations.
    BenchmarkHarness.node.getAckChecker().checkAcks();
View Full Code Here

    FileUtil.dumbfilecopy(truncated, new File(writing, truncated.getName()));

    EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(
        LogicalNodeContext.testingContext()),
        "{ ackedWriteAhead => { ackChecker => counter(\"count\") } }");
    EventSource src = MemorySinkSource.cannedData("foo foo foo ", 5);
    snk.open();
    src.open();
    EventUtil.dumpAll(src, snk);
    src.close();
    snk.close(); // this should block until recovery complete.

    CounterSink cnt = (CounterSink) ReportManager.get().getReportable("count");
    // 461 in file before truncated + 5 from silly driver
    assertEquals(466, cnt.getCount());
View Full Code Here

   */
  @Test
  public void testOpenCloseSyslogTcpSourceThreads() throws IOException,
      InterruptedException {
    LogicalNode node = new LogicalNode(new Context(), "test");
    EventSource prev = new SyslogTcpSourceThreads(6789);
    node.openLoadNode(prev, new NullSink());
    for (int i = 0; i < 20; i++) {
      EventSource cur = new SyslogTcpSourceThreads(6789);
      node.openLoadNode(cur, new NullSink());
      prev = cur;

    }
  }
View Full Code Here

      new Thread() {
        @Override
        public void run() {
          start.countDown();
          try {
            EventSource src = new NoNlASCIISynthSource(count, 100);
            start.await();
            src.open();
            EventUtil.dumpAll(src, snk);
            src.close();
          } catch (Exception e) {
            LOG.error("failure", e);
            // fail("e");
          } finally {
            done.countDown();
View Full Code Here

      new Thread() {
        @Override
        public void run() {
          start.countDown();
          try {
            EventSource src = new NoNlASCIISynthSource(count, 100);
            Context ctx = new ReportTestingContext();
            EventSink snk = new CompositeSink(ctx, "counter(\"total." + idx
                + "\")");
            // use the same wal, but different counter.
            snk = new NaiveFileWALDeco(ctx, snk, wal, new TimeTrigger(
                new ProcessTagger(), 1000000), new AckListener.Empty(), 1000000);
            src.open();
            snk.open();

            start.await();

            EventUtil.dumpAll(src, snk);
            src.close();
            snk.close();
          } catch (Exception e) {
            LOG.error("failure", e);
            // fail("e");
          } finally {
View Full Code Here

      new Thread() {
        @Override
        public void run() {
          start.countDown();
          try {
            EventSource src = new NoNlASCIISynthSource(count, 100);
            Context ctx = new ReportTestingContext();
            EventSink snk = new CompositeSink(ctx, "counter(\"total." + idx
                + "\")");
            // use the same wal, but different counter.

            snk = new NaiveFileWALDeco(ctx, snk, wal, new TimeTrigger(
                new ProcessTagger(), 1000000), new AckListener.Empty(), 1000000);

            start.await();

            // allow for contention on the open call.
            src.open();
            snk.open();

            EventUtil.dumpAll(src, snk);
            src.close();
            snk.close();
          } catch (Exception e) {
            LOG.error("failure", e);
            // fail("e");
          } finally {
View Full Code Here

    corrupt.deleteOnExit();

    // check now, and any age is too old.
    File commit = new File(tmpdir, "committed");
    commit.deleteOnExit();
    EventSource src = new SeqfileEventSource(corrupt.getAbsolutePath());

    try {
      src.open();
    } catch (IOException e) {
      return;
    }

    Assert.fail("should have failed with io exception");
View Full Code Here

    sink.close();
    assertEquals(0, wal.getWritingTags().size());
    assertEquals(1, wal.getLoggedTags().size());

    // logged values can transition to the sending state by getting a sender.
    EventSource curSource = wal.getUnackedSource();
    // no state change
    assertEquals(0, wal.getWritingTags().size());
    assertEquals(0, wal.getLoggedTags().size());
    assertEquals(1, wal.getSendingTags().size());

    // open changes state
    curSource.open();
    assertEquals(0, wal.getLoggedTags().size());
    assertEquals(1, wal.getSendingTags().size());

    // read next event
    Event e = null;
    ConsoleEventSink console = new ConsoleEventSink();
    while ((e = curSource.next()) != null) {
      console.append(e);
    }
    curSource.close();
    assertEquals(0, wal.getSendingTags().size());
    assertEquals(1, wal.getSentTags().size());

    // trigger to make it move from sent to acked/done
    String tag = wal.getSentTags().iterator().next();
View Full Code Here

  @Test
  public void testBuildTextSource() throws IOException, FlumeSpecException,
      InterruptedException {
    Context ctx = LogicalNodeContext.testingContext();
    LOG.info("Working Dir path: " + new File(".").getAbsolutePath());
    EventSource src = FlumeBuilder.buildSource(ctx, SOURCE);
    src.open();
    Event e = null;
    int cnt = 0;
    while ((e = src.next()) != null) {
      LOG.info(e);
      cnt++;
    }
    src.close();
    assertEquals(LINES, cnt);
  }
View Full Code Here

TOP

Related Classes of com.cloudera.flume.core.EventSource

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.