Examples of com.cloudera.flume.core.EventSource

com.cloudera.flume.core.EventSource
This provides a synchronous interface to getting events. An state/event-based architecture will likely be more efficient than a thread based one because there are many sources of information and only one output.

    for (int i = 0; i < threads; i++) {
      final int id = i;
      Thread th = new Thread() {
        public void run() {
          try {
            EventSource txt = new NoNlASCIISynthSource(25, 100);
            txt.open();
            MemorySinkSource mem = new MemorySinkSource();
            mem.open();
            EventUtil.dumpAll(txt, mem);
            txt.close();


            // mem -> AvroEventSink
            AvroEventSink snk = new AvroEventSink("0.0.0.0", conf
                .getCollectorPort() + 1);
            snk.open();

View Full Code Here

    // failover sink replaces each names


    LOG.info(snk.getMetrics().toText());


    snk.open();
    EventSource src = MemorySinkSource.cannedData("test is a test", 31);
    src.open();
    EventUtil.dumpAll(src, snk);


    int[] ans = { 16, 8, 4, 2, 1 };
    for (int i = 0; i < ans.length; i++) {
      Reportable rptable = ReportManager.get().getReportable(names.get(i));
      long val = rptable.getMetrics().getLongMetric(names.get(i));
      assertEquals(ans[i], val);
    }


    src.open();
    try {
      // here we finally have all failovers triggered to fail
      snk.append(src.next());
    } catch (IOException ioe) {
      // this should be thrown and caught.
      src.close();
      snk.close();
      return;
    }


    fail("Expected exception");

View Full Code Here

    EventSink snk = new CompositeSink(new ReportTestingContext(), spec);


    LOG.info(snk.getMetrics().toText());


    snk.open();
    EventSource src = MemorySinkSource.cannedData("test is a test", 31);
    src.open();
    EventUtil.dumpAll(src, snk);


    int[] ans = { 16, 8, 4, 2, 1 };
    for (int i = 0; i < ans.length; i++) {
      Reportable rptable = ReportManager.get().getReportable(names.get(i));

View Full Code Here

      setName(name + "-" + getId());
    }


    public void run() {
      EventSink sink = null;
      EventSource source = null;
      synchronized (DirectDriver.this) {
        sink = DirectDriver.this.sink;
        source = DirectDriver.this.source;
      }
      try {
        stopped = false;
        error = null;
        state = NodeState.ACTIVE;
        LOG.debug("Starting driver " + DirectDriver.this);
        fireStart();


        while (!stopped) {
          Event e = source.next();
          if (e == null)
            break;


          sink.append(e);
        }

View Full Code Here

      Context ctx = new Context(context);
      Pair<String, List<String>> idArgs = handleArgs(t, ctx);
      String sourceType = idArgs.getLeft();
      List<String> args = idArgs.getRight();


      EventSource src = srcFactory.getSource(ctx, sourceType, args
          .toArray(new String[0]));
      if (src == null) {
        throw new FlumeIdException("Invalid source: "
            + FlumeSpecGen.genEventSource(t));
      }

View Full Code Here


  @Test
  public void testTailPermissionDenied() throws IOException,
      FlumeSpecException, InterruptedException {
    File f;
    final EventSource eventSource;
    final CompositeSink eventSink;
    final AtomicBoolean workerFailed;
    Thread workerThread;
    FileWriter writer;
    long sleepTime;
    long eventCount;


    f = File.createTempFile("temp", ".tmp");
    f.setReadable(false, false);


    f.deleteOnExit();


    eventSource = TailSource.builder().build(f.getAbsolutePath());
    eventSink = new CompositeSink(new ReportTestingContext(),
        "{ delay(50) => counter(\"count\") }");
    workerFailed = new AtomicBoolean(false);
    workerThread = new Thread() {


      @Override
      public void run() {
        try {
          eventSource.open();
          eventSink.open();


          EventUtil.dumpN(10, eventSource, eventSink);
          Clock.sleep(500);
          eventSource.close();
          eventSink.close();
        } catch (Exception e) {
          LOG.error("Unexpected exception", e);
        }
      }

View Full Code Here

      InterruptedException {
    File f = File.createTempFile("temp", ".tmp");
    f.deleteOnExit();
    final CompositeSink snk = new CompositeSink(new ReportTestingContext(),
        "{ delay(50) => counter(\"count\") }");
    final EventSource src = TailSource.builder().build(f.getAbsolutePath());
    final CountDownLatch done = new CountDownLatch(1);
    final int count = 30;
    runDriver(src, snk, done, count);
    FileWriter fw = new FileWriter(f);
    for (int i = 0; i < count; i++) {

View Full Code Here

    File f2 = File.createTempFile("moved", ".tmp");
    f2.delete();
    f2.deleteOnExit();
    final CompositeSink snk = new CompositeSink(new ReportTestingContext(),
        "{ delay(50) => counter(\"count\") }");
    final EventSource src = TailSource.builder().build(f.getAbsolutePath());
    final CountDownLatch done = new CountDownLatch(1);
    final int count = 30;
    runDriver(src, snk, done, count);


    // Need to make sure the first file shows up

View Full Code Here

    f.deleteOnExit();
    File f2 = File.createTempFile("multitemp2", ".tmp");
    f2.deleteOnExit();
    final CompositeSink snk = new CompositeSink(new ReportTestingContext(),
        "{ delay(50) => counter(\"count\") }");
    final EventSource src = TailSource.multiTailBuilder().build(
        f.getAbsolutePath(), f2.getAbsolutePath());
    final CountDownLatch done = new CountDownLatch(1);
    final int count = 60;
    runDriver(src, snk, done, count);

View Full Code Here

    File f = File.createTempFile("multitemp1", ".tmp");
    f.deleteOnExit();
    File f2 = File.createTempFile("multitemp2", ".tmp");
    f2.deleteOnExit();
    final MemorySinkSource snk = new MemorySinkSource();
    final EventSource src = TailSource.multiTailBuilder().build(
        f.getAbsolutePath(), f2.getAbsolutePath());
    final CountDownLatch done = new CountDownLatch(1);
    final int count = 60;
    Thread t = new Thread() {
      @Override
      public void run() {
        try {
          src.open();
          snk.open();
          EventUtil.dumpN(count, src, snk);
          src.close();
          snk.close();
          done.countDown();
        } catch (IOException e) {
          e.printStackTrace();
        } catch (InterruptedException e) {

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.cloudera.flume.core.EventSource

com.cloudera.flume.agent.diskfailover.NaiveFileFailoverManager

com.cloudera.flume.agent.diskfailover.TestConcurrentDFOMan

com.cloudera.flume.agent.diskfailover.TestDiskFailoverBenchmarking

com.cloudera.flume.agent.diskfailover.TestDiskFailoverDeco

com.cloudera.flume.agent.diskfailover.TestDiskFailoverManager

com.cloudera.flume.agent.diskfailover.TestDiskFailoverSource

com.cloudera.flume.agent.durability.NaiveFileWALManager

com.cloudera.flume.agent.durability.TestConcurrentWALMan

com.cloudera.flume.agent.durability.TestNaiveFileWALDeco

com.cloudera.flume.agent.durability.TestNaiveFileWALManager

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.