Examples of com.cloudera.flume.core.EventSource

com.cloudera.flume.core.EventSource
This provides a synchronous interface to getting events. An state/event-based architecture will likely be more efficient than a thread based one because there are many sources of information and only one output.

   */
  @Test
  public void testTooManyOpens() throws IOException, FlumeSpecException, InterruptedException {
    Benchmark b = new Benchmark("connection exhaust");


    EventSource src = FlumeBuilder.buildSource(LogicalNodeContext.testingContext(),"thrift(31337)");


    EventSink snk = FlumeBuilder.buildSink("thrift(\"0.0.0.0\",31337)");


    src.open();


    // iterate until an exception is thrown
    int i = 0;
    try {
      for (i = 0; true; i++) {


        snk.open();
        System.out.println(i + " connections...");
      }
    } catch (IOException io) {
      System.out.println(io);
      b.mark("conns", i);
    }
    src.close();
    b.done();
  }

View Full Code Here

  @Test
  public void testSurviveManyOpens() throws IOException, FlumeSpecException,
      InterruptedException {
    Benchmark b = new Benchmark("connection exhaust");


    EventSource src = FlumeBuilder.buildSource(LogicalNodeContext.testingContext(), "thrift(31337)");
    src.open();


    // iterate until an exception is thrown
    for (int i = 0; i < 10000; i++) { // previous fails at 1000, make sure ok at
      // an order of magnitude bigger.


      EventSink snk = FlumeBuilder.buildSink(LogicalNodeContext.testingContext(),
          "thrift(\"0.0.0.0\",31337)");
      snk.open();
      System.out.println(i + " connections...");
      snk.close();
    }


    src.close();
    b.done();
  }

View Full Code Here

  /**
   * Test process creation and reading from stdout; assumes 'yes' is available.
   */
  public void testSimpleExec() throws IOException {
    try {
      EventSource source = new ExecNioSource.Builder().build("yes");
      source.open();
      Event e = source.next();
      String body = new String(e.getBody());
      assertEquals("Expected event body to be 'y', but got " + body, body, "y");
      source.close();
    } catch (Exception e) {
      LOG.error(e.getMessage(), e);
      fail(e.getMessage());
    }
  }

View Full Code Here

   */
  @Test
  public void testEmptyCommand() throws IOException, InterruptedException {
    Exception e = null;
    try {
      EventSource source = new ExecNioSource.Builder().build("");
      source.open();
      source.close();
    } catch (IllegalArgumentException i) {
      e = i;
    }
    assertNotNull("testEmptyCommand expected IllegalArgumentException", e);
  }

View Full Code Here

    BufferedWriter out = new BufferedWriter(new FileWriter(temp));
    out.write("#!/bin/bash\necho \"Hello world!\" >&2 \n");
    out.close();
    String cmd = temp.getAbsolutePath();


    EventSource source = new ExecNioSource.Builder().build("/bin/bash " + cmd);
    source.open();
    Event e = source.next();
    String body = new String(e.getBody());
    String src = new String(e.get(ExecNioSource.A_PROC_SOURCE));
    source.close();
    assertEquals("Expected event body to be 'Hello World!', but got '" + body
        + "'", body, "Hello world!");
    assertEquals("Expected to receive event from STDERR, but got " + src, src,
        "STDERR");
  }

View Full Code Here

    BufferedWriter out = new BufferedWriter(new FileWriter(temp));
    out.write("#!/bin/bash\n while true; do sleep 1; done; >&2 \n");
    out.close();
    String cmd = temp.getAbsolutePath();


    final EventSource source = new ExecNioSource.Builder().build("/bin/bash "
        + cmd);
    final CountDownLatch started = new CountDownLatch(1);
    final CountDownLatch latch = new CountDownLatch(1);
    Thread t = new Thread() {
      public void run() {
        try {
          source.open();
          started.countDown();
          source.next();
          latch.countDown();
        } catch (Exception e) {
          LOG.warn("Event consumption thread caught exception, test will fail",
              e);
        }
      }
    };
    t.start();
    started.await(1000, TimeUnit.MILLISECONDS);
    Clock.sleep(100); // give next a chance to start.
    source.close();
    assertTrue("source.next did not exit after close within 5 seconds",
        latch.await(5000, TimeUnit.MILLISECONDS));
  }

View Full Code Here

      RuntimeException, FlumeSpecException {


    // got a newer configuration
    LOG.debug("Attempt to load config " + cfg);
    EventSink newSnk = null;
    EventSource newSrc = null;
    try {
      String errMsg = null;
      if (cfg.sinkConfig == null || cfg.sinkConfig.length() == 0) {
        errMsg = this.getName() + " - empty sink";
      }

View Full Code Here

    BufferedWriter out = new BufferedWriter(new FileWriter(temp));
    out.write("#!/bin/bash\n echo \"Hello\"\necho \"World!\"\n");
    out.close();
    String cmd = temp.getAbsolutePath();
    ExecNioSource.Builder builder = new ExecNioSource.Builder();
    EventSource source = builder.build("/bin/bash " + cmd, "true");
    source.open();


    Clock.sleep(250); // need to sleep to let things percolate through threads.
    Event e = source.next();
    source.close();
    String output = new String(e.getBody());
    assertEquals("Expected aggregate exec output to be 'Hello\nWorld!', "
        + "but got '" + output + "'", output, "Hello\nWorld!\n");


  }

View Full Code Here

    out.write("#!/bin/bash\n date +%s");
    out.close();
    String cmd = temp.getAbsolutePath();
    ExecNioSource.Builder builder = new ExecNioSource.Builder();
    // aggregate = false, restart=true, restart after 1000ms
    EventSource source = builder.build("/bin/bash " + cmd, "false", "true",
        "1000");
    source.open();
    Event e1 = source.next();
    Event e2 = source.next();
    source.close();
    String t1 = new String(e1.getBody());
    String t2 = new String(e2.getBody());
    long time1 = Long.parseLong(t1);
    long time2 = Long.parseLong(t2);
    assertTrue("Time difference with repeated exec should be >= 1s", time2

View Full Code Here

   * this test can give false successes, but no false failures.
   */
  @Test
  public void testRepeatedOpenClose() {
    ExecNioSource.Builder builder = new ExecNioSource.Builder();
    EventSource source = builder.build("date");
    for (int i = 0; i < 1024; ++i) {
      try {
        source = builder.build("date");
        source.open();
        source.close();
      } catch (Exception e) {
        assertTrue("Exec open / close failed on iteration " + i
            + " with failure " + e, false);
      }
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.cloudera.flume.core.EventSource

com.cloudera.flume.agent.diskfailover.NaiveFileFailoverManager

com.cloudera.flume.agent.diskfailover.TestConcurrentDFOMan

com.cloudera.flume.agent.diskfailover.TestDiskFailoverBenchmarking

com.cloudera.flume.agent.diskfailover.TestDiskFailoverDeco

com.cloudera.flume.agent.diskfailover.TestDiskFailoverManager

com.cloudera.flume.agent.diskfailover.TestDiskFailoverSource

com.cloudera.flume.agent.durability.NaiveFileWALManager

com.cloudera.flume.agent.durability.TestConcurrentWALMan

com.cloudera.flume.agent.durability.TestNaiveFileWALDeco

com.cloudera.flume.agent.durability.TestNaiveFileWALManager

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.