Package com.cloudera.flume.handlers.debug

Examples of com.cloudera.flume.handlers.debug.MemorySinkSource


  @Test
  public void testAvroDataFileWriteRead() throws IOException,
      FlumeSpecException, InterruptedException {

    MemorySinkSource mem = MemorySinkSource.cannedData("test ", 5);

    // setup sink.
    File f = File.createTempFile("avrodata", ".avro");
    f.deleteOnExit();
    LOG.info("filename before escaping: " + f.getAbsolutePath());
    String custom = "text(\""
        + StringEscapeUtils.escapeJava(f.getAbsolutePath())
        + "\", \"avrodata\")";
    LOG.info("sink to parse: " + custom);
    EventSink snk = FlumeBuilder.buildSink(new Context(), custom);
    snk.open();
    mem.open();
    EventUtil.dumpAll(mem, snk);
    snk.close();

    mem.open();
    DatumReader<EventImpl> dtm = new ReflectDatumReader<EventImpl>(
        EventImpl.class);
    DataFileReader<EventImpl> dr = new DataFileReader<EventImpl>(f, dtm);

    EventImpl eout = null;
    for (Object o : dr) {
      eout = (EventImpl) o; // TODO (jon) fix AVRO -- this is gross
      Event expected = mem.next();
      Assert.assertTrue(Arrays.equals(eout.getBody(), expected.getBody()));
    }

  }
View Full Code Here


   * @throws InterruptedException
   */
  @Test
  public void testCheckChecker() throws IOException, InterruptedException {
    int msgs = 100;
    MemorySinkSource mss = new MemorySinkSource();
    AckChecksumChecker<EventSink> acc = new AckChecksumChecker<EventSink>(mss);
    AckChecksumInjector<EventSink> aci = new AckChecksumInjector<EventSink>(acc);

    aci.open();
    for (int i = 0; i < msgs; i++) {
      Event e = new EventImpl(("this is a test " + i).getBytes());
      aci.append(e);
    }
    aci.close(); // will throw exception if checksum doesn't match

    Event eo = null;
    int count = 0;
    while ((eo = mss.next()) != null) {
      System.out.println(eo);
      count++;
    }

    assertEquals(msgs, count); // extra ack messages should have been consumed
View Full Code Here

   *
   * @throws InterruptedException
   */
  @Test
  public void testReorderedChecker() throws IOException, InterruptedException {
    MemorySinkSource mss = new MemorySinkSource();
    AckChecksumChecker<EventSink> cc = new AckChecksumChecker<EventSink>(mss);
    ReorderDecorator<EventSink> ro = new ReorderDecorator<EventSink>(cc, .5,
        .5, 0);
    AckChecksumInjector<EventSink> cp = new AckChecksumInjector<EventSink>(ro);

    cp.open();
    for (int i = 0; i < 100; i++) {
      Event e = new EventImpl(("this is a test " + i).getBytes());
      cp.append(e);
    }
    cp.close();

    Event eo = null;
    while ((eo = mss.next()) != null) {
      System.out.println(eo);
    }
  }
View Full Code Here

   * @throws InterruptedException
   */
  @Test
  public void testNoStart() throws IOException, InterruptedException {
    final int msgs = 100;
    MemorySinkSource mss = new MemorySinkSource();
    AckChecksumChecker<EventSink> cc = new AckChecksumChecker<EventSink>(mss);
    EventSinkDecorator<EventSink> dropFirst = new EventSinkDecorator<EventSink>(
        cc) {
      int count = 0;

View Full Code Here

   * @throws InterruptedException
   */
  @Test
  public void testNoStop() throws IOException, InterruptedException {
    final int msgs = 100;
    MemorySinkSource mss = new MemorySinkSource();
    AckChecksumChecker<EventSink> cc = new AckChecksumChecker<EventSink>(mss);
    EventSinkDecorator<EventSink> dropStop = new EventSinkDecorator<EventSink>(
        cc) {
      int drop = msgs + 1; // intial + messages, the drop the last
      int count = 0;
View Full Code Here

   * @throws InterruptedException
   */
  @Test
  public void testDupe() throws IOException, InterruptedException {
    final int msgs = 100;
    MemorySinkSource mss = new MemorySinkSource();
    AckChecksumChecker<EventSink> cc = new AckChecksumChecker<EventSink>(mss,
        new AckListener.Empty() {
          @Override
          public void err(String group) throws IOException {
            throw new IOException("Fail");
View Full Code Here

    Benchmark b = new Benchmark("seqfile write");
    b.mark("begin");

    TextFileSource txt = new TextFileSource(HADOOP_DATA[0]);
    txt.open();
    MemorySinkSource mem = new MemorySinkSource();
    mem.open();
    EventUtil.dumpAll(txt, mem);
    txt.close();
    b.mark("disk_loaded");

    File tmp = File.createTempFile("test", "tmp");
    tmp.deleteOnExit();
    SeqfileEventSink sink = new SeqfileEventSink(tmp);
    sink.open();
    b.mark("receiver_started");

    EventUtil.dumpAll(mem, sink);

    b.mark("seqfile_disk_write");

    sink.close();
    b.mark("seqfile size", tmp.length());
    b.done();

    // //////// second phase using the file written in previous phase.
    Benchmark b2 = new Benchmark("seqfile_disk_read");
    b2.mark("begin");

    SeqfileEventSource seq = new SeqfileEventSource(tmp.getAbsolutePath());
    seq.open();
    MemorySinkSource mem2 = new MemorySinkSource();
    EventUtil.dumpAll(seq, mem2);
    seq.close();
    b2.mark("seqfile_loaded");

    b2.done();
View Full Code Here

    Benchmark b = new Benchmark("log4j format read");
    b.mark("begin");

    Log4jTextFileSource txt = new Log4jTextFileSource(HADOOP_DATA[0]);
    txt.open();
    MemorySinkSource mem = new MemorySinkSource();
    mem.open();
    EventUtil.dumpAll(txt, mem);
    txt.close();
    b.mark("log4j_disk_loaded");
    b.done();
  }
View Full Code Here

    MockMasterRPC mock = new MockMasterRPC();
    FlumeNode node = new FlumeNode(mock, false /* starthttp */, false /* onshot */);
    AckListener pending = node.getAckChecker().getAgentAckQueuer();

    // initial source of data.
    MemorySinkSource mem = new MemorySinkSource();
    byte[] tag = "my tag".getBytes();
    AckChecksumInjector<EventSink> ackinj = new AckChecksumInjector<EventSink>(
        mem, tag, pending);
    ackinj.open();
    for (int i = 0; i < 5; i++) {
      ackinj.append(new EventImpl(("Event " + i).getBytes()));
    }
    ackinj.close();

    // there now are now 7 events in the mem
    // consume data.
    EventSink es1 = new ConsoleEventSink();
    EventUtil.dumpAll(mem, es1);
    System.out.println("---");

    String rpt = "foo";
    String snk = "  { intervalDroppyAppend(5)  => { ackChecker => [console, counter(\""
        + rpt + "\") ] } }  ";
    EventSink es = FlumeBuilder.buildSink(new Context(), snk);
    mem.open(); // resets index.
    es.open();
    EventUtil.dumpAll(mem, es);
    node.getAckChecker().checkAcks();
    assertEquals(1, node.getAckChecker().pending.size());

    mem.open(); // resets index.
    // send to collector, collector updates master state (bypassing flakeyness)
    EventUtil.dumpAll(mem, ((EventSinkDecorator<EventSink>) es).getSink());
    // agent gets state from master, updates it ack states
    try {
      node.getAckChecker().checkAcks();
View Full Code Here

    Benchmark b = new Benchmark("hdfs seqfile copy");
    b.mark("begin");

    TextFileSource txt = new TextFileSource(HADOOP_DATA[0]);
    txt.open();
    MemorySinkSource mem = new MemorySinkSource();
    mem.open();
    EventUtil.dumpAll(txt, mem);
    txt.close();
    b.mark("disk_loaded");

    File tmp = File.createTempFile("test", "tmp");
View Full Code Here

TOP

Related Classes of com.cloudera.flume.handlers.debug.MemorySinkSource

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.