Package com.cloudera.flume.core

Examples of com.cloudera.flume.core.EventSink


    // /////////////////////
    // This illustrates the problems from the previous test.
    FileUtil.dumbfilecopy(acked, new File(writing, acked.getName()));
    // /////////////////////

    EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(
        LogicalNodeContext.testingContext()),
        "{ ackedWriteAhead => { ackChecker => counter(\"count\") } }");
    EventSource src = MemorySinkSource.cannedData("foo foo foo ", 5);
    snk.open();
    src.open();
    EventUtil.dumpAll(src, snk);
    src.close();
    snk.close(); // this should block until recovery complete.

    // agent checks for ack registrations.
    BenchmarkHarness.node.getAckChecker().checkAcks();

    CounterSink cnt = (CounterSink) ReportManager.get().getReportable("count");
View Full Code Here


        .getPhysicalNodeName()), "writing");

    writing.mkdirs();
    FileUtil.dumbfilecopy(truncated, new File(writing, truncated.getName()));

    EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(
        LogicalNodeContext.testingContext()),
        "{ ackedWriteAhead => { ackChecker => counter(\"count\") } }");
    EventSource src = MemorySinkSource.cannedData("foo foo foo ", 5);
    snk.open();
    src.open();
    EventUtil.dumpAll(src, snk);
    src.close();
    snk.close(); // this should block until recovery complete.

    CounterSink cnt = (CounterSink) ReportManager.get().getReportable("count");
    // 461 in file before truncated + 5 from silly driver
    assertEquals(466, cnt.getCount());
View Full Code Here

  public void testExceptionThreadHandoff() throws IOException,
      InterruptedException {
    try {
      BenchmarkHarness.setupLocalWriteDir();
      Event e = new EventImpl(new byte[0]);
      EventSink snk = new EventSink.Base() {
        @Override
        public void append(Event e) throws IOException {
          throw new IOException("mock ioe");
        }
      };
View Full Code Here

    final CountDownLatch done = new CountDownLatch(threads);
    final NaiveFileWALManager wal = new NaiveFileWALManager(dir);
    wal.open();

    Context ctx = new ReportTestingContext();
    EventSink cntsnk = new CompositeSink(ctx, "counter(\"total\")");
    // use the same wal, but different counter.
    final EventSink snk = new NaiveFileWALDeco(ctx, cntsnk, wal,
        new TimeTrigger(new ProcessTagger(), 1000000), new AckListener.Empty(),
        1000000);
    snk.open();

    for (int i = 0; i < threads; i++) {
      new Thread() {
        @Override
        public void run() {
          start.countDown();
          try {
            EventSource src = new NoNlASCIISynthSource(count, 100);
            start.await();
            src.open();
            EventUtil.dumpAll(src, snk);
            src.close();
          } catch (Exception e) {
            LOG.error("failure", e);
            // fail("e");
          } finally {
            done.countDown();
          }
        }

      }.start();
    }

    boolean ok = done.await(30, TimeUnit.SECONDS);
    assertTrue("Test timed out!", ok);
    Thread.sleep(1000);
    snk.close();

    CounterSink cnt = (CounterSink) ReportManager.get().getReportable("total");
    long ci = cnt.getCount();
    LOG.info("count : " + ci);
    assertEquals((count * threads) + 2, ci);
View Full Code Here

        public void run() {
          start.countDown();
          try {
            EventSource src = new NoNlASCIISynthSource(count, 100);
            Context ctx = new ReportTestingContext();
            EventSink snk = new CompositeSink(ctx, "counter(\"total." + idx
                + "\")");
            // use the same wal, but different counter.
            snk = new NaiveFileWALDeco(ctx, snk, wal, new TimeTrigger(
                new ProcessTagger(), 1000000), new AckListener.Empty(), 1000000);
            src.open();
            snk.open();

            start.await();

            EventUtil.dumpAll(src, snk);
            src.close();
            snk.close();
          } catch (Exception e) {
            LOG.error("failure", e);
            // fail("e");
          } finally {
            done.countDown();
View Full Code Here

        public void run() {
          start.countDown();
          try {
            EventSource src = new NoNlASCIISynthSource(count, 100);
            Context ctx = new ReportTestingContext();
            EventSink snk = new CompositeSink(ctx, "counter(\"total." + idx
                + "\")");
            // use the same wal, but different counter.

            snk = new NaiveFileWALDeco(ctx, snk, wal, new TimeTrigger(
                new ProcessTagger(), 1000000), new AckListener.Empty(), 1000000);

            start.await();

            // allow for contention on the open call.
            src.open();
            snk.open();

            EventUtil.dumpAll(src, snk);
            src.close();
            snk.close();
          } catch (Exception e) {
            LOG.error("failure", e);
            // fail("e");
          } finally {
            done.countDown();
View Full Code Here

    // since using udp doesn't matter if it gets there or not.

    // default ganglia gmond multicast destination ip.
    String svrs = "239.2.11.71";

    EventSink lsnk = new GangliaSink(svrs, ATTR_LONG, "bytes", Type.LONG);
    EventSink isnk = new GangliaSink(svrs, ATTR_INT, "bytes", Type.INT);
    EventSink dsnk = new GangliaSink(svrs, ATTR_DOUBLE, "bytes", Type.DOUBLE);
    EventSink snk = new FanOutSink<EventSink>(lsnk, isnk, dsnk);
    snk.open();
    // This is enough for the data to register.
    for (int i = 0; i < 60; i++) {
      Event e = new EventImpl("".getBytes());
      Attributes.setLong(e, ATTR_LONG, i * 1000000);
      Attributes.setInt(e, ATTR_INT, (int) (i * 1000000));
      Attributes.setDouble(e, ATTR_DOUBLE, (double) (1.0 / (i % 20)));
      snk.append(e);
      Clock.sleep(1000);
    }
    snk.close();

  }
View Full Code Here

  }

  @Test
  public void testBuilder() throws IOException, InterruptedException {
    EventSink snk = GangliaSink.builder().build(new Context(), "localhost",
        "foo", "int");
    for (int i = 0; i < 10; i++) {
      snk.open();
      snk.append(new EventImpl("".getBytes()));
      snk.close();
    }

    EventSink snk4 = GangliaSink.builder().build(new Context(), "localhost",
        "foo", "int", FlumeConfiguration.get().getGangliaServers());
    for (int i = 0; i < 10; i++) {
      snk4.open();
      snk4.append(new EventImpl("".getBytes()));
      snk4.close();
    }

    try {
      GangliaSink.builder().build(new Context(), "localhost", "foo", "bar");
    } catch (IllegalArgumentException e) {
View Full Code Here

  }

  @Test
  public void testFactoryBuild() throws FlumeSpecException, IOException,
      InterruptedException {
    EventSink snk = new CompositeSink(new Context(),
        "ganglia(\"localhost\", \"foo\", \"int\")");
    for (int i = 0; i < 10; i++) {
      snk.open();
      snk.append(new EventImpl("".getBytes()));
      snk.close();
    }

  }
View Full Code Here

    assertTrue("Took too long to bind to a port", listener.listening.await(5,
        TimeUnit.SECONDS));
    LOG.info("Listening to port " + listener.getPort());

    // setup and send some ganglia data.
    EventSink ganglia = new GangliaSink(hostName + ":" + listener.getPort(),
        "foo", "bars", Type.INT);
    ganglia.open();
    Event e = new EventImpl("baz".getBytes());
    Attributes.setInt(e, "foo", 1337);
    ganglia.append(e);
    ganglia.close();

    // did the other thread get the data?
    assertTrue("Took too long to recieve a packet", listener.received.await(5,
        TimeUnit.SECONDS));
View Full Code Here

TOP

Related Classes of com.cloudera.flume.core.EventSink

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.