Package com.cloudera.flume.core

Examples of com.cloudera.flume.core.EventSink


   *
   * @throws InterruptedException
   */
  @Test
  public void testStubbornNew() throws IOException, InterruptedException {
    EventSink failAppend = mock(EventSink.class);
    Event e = new EventImpl("test".getBytes());

    // for mocking void void returning calls, we use this mockito
    // syntax (it is kinda gross but still cleaner than the version above)
    // the first two calls "succeed", the third throws io exn, and then the
View Full Code Here


      InterruptedException {

    // count resets on open and close, this one does not.
    final AtomicInteger ok = new AtomicInteger();

    EventSink cnt = new EventSink.Base() {
      @Override
      public void append(Event e) throws IOException {
        ok.incrementAndGet();
      }
    };
View Full Code Here

  }

  @Test
  public void testExceptionFallthrough() throws IOException,
      InterruptedException {
    EventSink mock = mock(EventSink.class);
    // two ok, and then two exception throwing cases
    doNothing().doNothing().doThrow(new IOException()).doThrow(
        new IOException()).when(mock).append(Mockito.<Event> anyObject());
    doReturn(new ReportEvent("stub")).when(mock).getMetrics();
View Full Code Here

  @Test
  public void testStubbornAppendMetrics() throws JSONException,
      FlumeSpecException, IOException, InterruptedException {
    ReportTestUtils.setupSinkFactory();

    EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(),
        "stubbornAppend one");
    ReportEvent rpt = ReportUtil.getFlattenedReport(snk);
    LOG.info(ReportUtil.toJSONObject(rpt).toString());
    assertNotNull(rpt.getLongMetric(StubbornAppendSink.A_FAILS));
    assertNotNull(rpt.getLongMetric(StubbornAppendSink.A_RECOVERS));
View Full Code Here

      setupLocalWriteDir();
      ReportManager.get().clear();

      // copy all events into memory
      EventSource src = MemorySinkSource.bufferize(ent.getValue());
      EventSink snk = createDecoratorBenchmarkSink(ent.getKey() + "," + deco,
          deco);
      src.open();
      snk.open();
      EventUtil.dumpAll(src, snk);
      src.close();
      snk.close();
      dumpReports();
      cleanupLocalWriteDir();
    }
  }
View Full Code Here

    FlumeNode node = FlumeNode.getInstance();
    File tmpdir = FileUtil.mktempdir();

    EventSource ackedmem = setupAckRoll();
    Pair<RollSink, EventSink> p = setupSink(node, tmpdir);
    EventSink snk = p.getRight();
    RollSink roll = p.getLeft();
    snk.open();

    String tag1 = roll.getCurrentTag();
    LOG.info(tag1);
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    // don't rotate the first one.
    assertEquals(1, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();
    // still one ack pending.
    assertEquals(1, node.getAckChecker().getPendingAckTags().size());

    String tag2 = roll.getCurrentTag();
    LOG.info(tag2);
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    roll.rotate();
    // two acks pending.
    assertEquals(2, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();
    // no more acks pending.
    assertEquals(0, node.getAckChecker().getPendingAckTags().size());

    String tag3 = roll.getCurrentTag();
    LOG.info(tag3);
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    roll.rotate();
    // one ack pending
    assertEquals(1, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();
    // no more acks pending.
    assertEquals(0, node.getAckChecker().getPendingAckTags().size());

    snk.close();

    FileUtil.rmr(tmpdir);
    BenchmarkHarness.cleanupLocalWriteDir();
  }
View Full Code Here

      InterruptedException {
    String rpt = "foo";
    String snk = " { ackedWriteAhead(100) => [console,  counter(\"" + rpt
        + "\") ] } ";
    for (int i = 0; i < 100; i++) {
      EventSink es = FlumeBuilder.buildSink(
          LogicalNodeContext.testingContext(), snk);
      es.open();
      es.close();
    }

  }
View Full Code Here

    FlumeNode node = FlumeNode.getInstance();
    File tmpdir = FileUtil.mktempdir();

    EventSource ackedmem = setupAckRoll();
    Pair<RollSink, EventSink> p = setupSink(node, tmpdir);
    EventSink snk = p.getRight();
    RollSink roll = p.getLeft();
    snk.open();

    String tag1 = roll.getCurrentTag();
    LOG.info(tag1);
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    roll.rotate(); // we should have the first batch and part of the second
    // one ack pending
    assertEquals(1, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();
    // no acks pending
    assertEquals(0, node.getAckChecker().getPendingAckTags().size());

    // note, we still are checking state for the 2nd batch of messages

    String tag2 = roll.getCurrentTag();
    LOG.info(tag2);
    // This is the end msg closes the 2nd batch
    snk.append(ackedmem.next()); // ack end
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    roll.rotate();
    // now 2nd batch and 3rd batch are pending.
    assertEquals(2, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();

    // no more acks out standing
    LOG.info("pending ack tags: " + node.getAckChecker().getPendingAckTags());
    assertEquals(0, node.getAckChecker().getPendingAckTags().size());

    snk.close();

    FileUtil.rmr(tmpdir);
    BenchmarkHarness.cleanupLocalWriteDir();
  }
View Full Code Here

    // in a real situation, there would be a agent sink after the
    // ackedWriteAhead and a collectorSource before the ackChecker block.
    String snk = " { ackedWriteAhead(1000) => { ackChecker => counter(\"" + rpt
        + "\") } }  ";

    EventSink es = FlumeBuilder.buildSink(new ReportTestingContext(
        LogicalNodeContext.testingContext()), snk);
    es.open();
    for (int i = 0; i < count; i++) {
      Event e = new EventImpl(("test message " + i).getBytes());
      es.append(e);
    }

    // last batch doesn't flush automatically unless time passes or closed-- one
    // pending
    Set<String> pending = mock.ackman.getPending();
    assertEquals(0, pending.size());

    // close and flush
    es.close();
    node.getAckChecker().checkAcks();

    CounterSink ctr = (CounterSink) ReportManager.get().getReportable(rpt);
    assertEquals(count, ctr.getCount());
View Full Code Here

    }
    ackinj.close();

    // there now are now 7 events in the mem
    // consume data.
    EventSink es1 = new ConsoleEventSink();
    EventUtil.dumpAll(mem, es1);
    System.out.println("---");

    String rpt = "foo";
    String snk = "  { intervalDroppyAppend(5)  => { ackChecker => [console, counter(\""
        + rpt + "\") ] } }  ";
    EventSink es = FlumeBuilder.buildSink(new Context(), snk);
    mem.open(); // resets index.
    es.open();
    EventUtil.dumpAll(mem, es);
    node.getAckChecker().checkAcks();
    assertEquals(1, node.getAckChecker().pending.size());

    mem.open(); // resets index.
View Full Code Here

TOP

Related Classes of com.cloudera.flume.core.EventSink

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.