Examples of com.cloudera.flume.core.EventSource

com.cloudera.flume.core.EventSource
This provides a synchronous interface to getting events. An state/event-based architecture will likely be more efficient than a thread based one because there are many sources of information and only one output.

    WALManager mockWalMan = mock(WALManager.class);
    BenchmarkHarness.setupFlumeNode(null, mockWalMan, null, null, null);
    FlumeNode node = FlumeNode.getInstance();
    File tmpdir = FileUtil.mktempdir();


    EventSource ackedmem = setupAckRoll();
    Pair<RollSink, EventSink> p = setupSink(node, tmpdir);
    EventSink snk = p.getRight();
    RollSink roll = p.getLeft();
    snk.open();


    String tag1 = roll.getCurrentTag();
    LOG.info(tag1);
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    // don't rotate the first one.
    assertEquals(1, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();
    // still one ack pending.
    assertEquals(1, node.getAckChecker().getPendingAckTags().size());


    String tag2 = roll.getCurrentTag();
    LOG.info(tag2);
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    roll.rotate();
    // two acks pending.
    assertEquals(2, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();
    // no more acks pending.
    assertEquals(0, node.getAckChecker().getPendingAckTags().size());


    String tag3 = roll.getCurrentTag();
    LOG.info(tag3);
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    roll.rotate();
    // one ack pending
    assertEquals(1, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();

View Full Code Here

 */
public class TestHadoopLogData implements ExampleData {


  @Test
  public void testLineCount() throws IOException, InterruptedException {
    EventSource src = new NoNlASCIISynthSource(25, 100, 1);
    src.open();


    SimpleRegexReporterBuilder b = new SimpleRegexReporterBuilder(
        HADOOP_REGEXES);
    Collection<RegexGroupHistogramSink> sinks = b.load();
    MultiReporter mr = new MultiReporter("apache_sinks", sinks);

View Full Code Here

    WALManager mockWalMan = mock(WALManager.class);
    BenchmarkHarness.setupFlumeNode(null, mockWalMan, null, null, null);
    FlumeNode node = FlumeNode.getInstance();
    File tmpdir = FileUtil.mktempdir();


    EventSource ackedmem = setupAckRoll();
    Pair<RollSink, EventSink> p = setupSink(node, tmpdir);
    EventSink snk = p.getRight();
    RollSink roll = p.getLeft();
    snk.open();


    String tag1 = roll.getCurrentTag();
    LOG.info(tag1);
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    roll.rotate(); // we should have the first batch and part of the second
    // one ack pending
    assertEquals(1, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();
    // no acks pending
    assertEquals(0, node.getAckChecker().getPendingAckTags().size());


    // note, we still are checking state for the 2nd batch of messages


    String tag2 = roll.getCurrentTag();
    LOG.info(tag2);
    // This is the end msg closes the 2nd batch
    snk.append(ackedmem.next()); // ack end
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    roll.rotate();
    // now 2nd batch and 3rd batch are pending.
    assertEquals(2, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();

View Full Code Here

    WALManager mockWalMan = mock(WALManager.class);
    BenchmarkHarness.setupFlumeNode(null, mockWalMan, null, null, null);
    FlumeNode node = FlumeNode.getInstance();
    File tmpdir = FileUtil.mktempdir();


    EventSource ackedmem = setupAckRoll();
    Pair<RollSink, EventSink> p = setupSink(node, tmpdir);
    EventSink snk = p.getRight();
    RollSink roll = p.getLeft();
    snk.open();


    String tag1 = roll.getCurrentTag();
    LOG.info(tag1);
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    snk.append(ackedmem.next()); // ack beg
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    roll.rotate(); // we should have the first batch and part of the second
    // one ack pending
    assertEquals(1, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();
    // no acks pending
    assertEquals(0, node.getAckChecker().getPendingAckTags().size());


    // we are partially through the second batch, at a different split point


    String tag2 = roll.getCurrentTag();
    LOG.info(tag2);
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    roll.rotate();
    // now we have closed off group2 and group3
    assertEquals(2, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();

View Full Code Here

      final int ROLLS = 1000;


      // setup a source of data that will shove a lot of ack laden data into the
      // stream.
      MemorySinkSource mem = new MemorySinkSource();
      EventSource src = new NoNlASCIISynthSource(COUNT, 10);
      src.open();
      Event e;
      while ((e = src.next()) != null) {
        AckChecksumInjector<EventSink> acks = new AckChecksumInjector<EventSink>(
            mem);
        acks.open(); // memory in sink never resets, and just keeps appending
        acks.append(e);
        acks.close();

View Full Code Here


    MultiGrepReporterSink<String> snk = new MultiGrepReporterSink<String>(
        "multi grep", aho);
    snk.open();


    EventSource src = new NoNlASCIISynthSource(25, 100, 1);
    src.open();


    EventUtil.dumpAll(src, snk);


    snk.append(new EventImpl(NPE.getBytes()));
    snk.append(new EventImpl(LOST.getBytes()));
    snk.append(new EventImpl(LOST.getBytes()));


    Histogram<String> h = snk.getHistogram();
    System.out.println(h);


    Assert.assertEquals(3, h.total());
    Assert.assertEquals(2, h.get(LOST));
    Assert.assertEquals(1, h.get(NPE));


    snk.close();
    src.close();


  }

View Full Code Here

    Assert.assertEquals(1, c.size());


    MultiGrepReporterSink<String> snk = c.iterator().next();
    snk.open();


    EventSource src = new NoNlASCIISynthSource(25, 100, 1);


    src.open();


    EventUtil.dumpAll(src, snk);
    snk.append(new EventImpl(NPE.getBytes()));
    snk.append(new EventImpl(LOST.getBytes()));
    snk.append(new EventImpl(LOST.getBytes()));


    Histogram<String> h = snk.getHistogram();
    System.out.println(h);


    Assert.assertEquals(3, h.total());
    Assert.assertEquals(2, h.get(LOST));
    Assert.assertEquals(1, h.get(NPE));


    snk.close();
    src.close();


  }

View Full Code Here

   * @throws InterruptedException
   */
  @Test
  public void testCollectorSource() throws FlumeSpecException, IOException,
      InterruptedException {
    EventSource src = FlumeBuilder.buildSource(LogicalNodeContext
        .testingContext(), "collectorSource(34568)");
    EventSink snk = FlumeBuilder.buildSink(new Context(),
        "rpcSink(\"localhost\", 34568)");
    src.open();
    snk.open();
    snk.append(new EventImpl("foo".getBytes()));
    src.next();
    snk.close();
    src.close();


  }

View Full Code Here

  @Test
  public void testAgentSink() throws FlumeSpecException, IOException,
      InterruptedException {
    String snkcfg = "agentSink(\"localhost\", 12345)";


    EventSource src = FlumeBuilder.buildSource(LogicalNodeContext
        .testingContext(), "collectorSource(12345)");
    src.open();


    for (int i = 0; i < 100; i++) {
      EventSink snk = FlumeBuilder.buildSink(LogicalNodeContext
          .testingContext(), snkcfg);
      snk.open();

View Full Code Here

                new TimeTrigger(1000000), new AckListener.Empty(), 100);


            ReportManager.get().add(cnt1);
            // make each parallel instance send a slightly different number of
            // messages.
            EventSource src = new NoNlASCIISynthSource(events + idx, 100);


            src.open();
            snk.open();


            started.countDown();


            EventUtil.dumpAll(src, snk);
            src.close();
            snk.close();
            FileUtil.rmr(f1);
          } catch (Exception e) {
            LOG.error(e, e);
          } finally {

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.cloudera.flume.core.EventSource

com.cloudera.flume.agent.diskfailover.NaiveFileFailoverManager

com.cloudera.flume.agent.diskfailover.TestConcurrentDFOMan

com.cloudera.flume.agent.diskfailover.TestDiskFailoverBenchmarking

com.cloudera.flume.agent.diskfailover.TestDiskFailoverDeco

com.cloudera.flume.agent.diskfailover.TestDiskFailoverManager

com.cloudera.flume.agent.diskfailover.TestDiskFailoverSource

com.cloudera.flume.agent.durability.NaiveFileWALManager

com.cloudera.flume.agent.durability.TestConcurrentWALMan

com.cloudera.flume.agent.durability.TestNaiveFileWALDeco

com.cloudera.flume.agent.durability.TestNaiveFileWALManager

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.