Examples of NaiveFileWALDeco

com.cloudera.flume.agent.durability.NaiveFileWALDeco
This decorator does write ahead logging using the NaiveFileWAL mechanism. It has a subordinate thread that drains the events that have been written to disk. Latches are used to maintain open and close semantics.

Examples of com.cloudera.flume.agent.durability.NaiveFileWALDeco

    String rpt = "foo";
    String snk = "{ ackChecker => [ console, counter(\"" + rpt + "\") ] }";
    Context ctx = new ReportTestingContext();
    EventSink es = FlumeBuilder.buildSink(ctx, snk);
    // Excessively large timeout because I am manually triggering it.
    NaiveFileWALDeco wal = new NaiveFileWALDeco(ctx, es, node.getWalManager(),
        new TimeTrigger(new ProcessTagger(), 100000), node.getAckChecker()
            .getAgentAckQueuer(), 1000);


    // open and send data.
    wal.open();
    for (int i = 0; i < count; i++) {
      Event e = new EventImpl(("test message " + i).getBytes());
      wal.append(e);
      if (i % 5 == 4) {
        wal.rotate();
      }
    }


    CounterSink ctr = (CounterSink) ReportManager.get().getReportable(rpt);
    LOG.info("Last batch should be missing. " + count + " > " + ctr.getCount()
        + " sent");


    wal.close();
    LOG.info("All should make it through here. " + count + " =="
        + ctr.getCount() + " sent");
    assertEquals(20, ctr.getCount());


    // check master state -- there should be 5 pending acks.

View Full Code Here

Examples of com.cloudera.flume.agent.durability.NaiveFileWALDeco

    String snk = "  { intervalDroppyAppend(15)  => { ackChecker => [ console, counter(\""
        + rpt + "\") ] } }  ";
    Context ctx = new ReportTestingContext();
    EventSink es = FlumeBuilder.buildSink(ctx, snk);


    NaiveFileWALDeco wal = new NaiveFileWALDeco(ctx, es, node.getWalManager(),
        new TimeTrigger(new ProcessTagger(), 100000), node.getAckChecker()
            .getAgentAckQueuer(), 1000);
    wal.open();


    // // Phase I - (20 events arrive, 3 groups make it)
    for (int i = 0; i < count; i++) {
      Event e = new EventImpl(("test message " + i).getBytes());
      wal.append(e);
      if (i % 5 == 4) {
        wal.rotate();
      }
    }


    // arbitrary sleep for now.
    Clock.sleep(1000);


    // three blocks should have successfully made it.
    assertEquals(3, master.getAckMan().getPending().size()); // (g0, g1, g3)
    CounterSink ctr = (CounterSink) ReportManager.get().getReportable(rpt);
    LOG.info(" Ack begin dropped, but all true messages went through. 20 == "
        + ctr.getCount() + " sent");
    // We dropped an ack begin message. Messages still make it through and there
    // should be some sort of ack warning on the master.
    assertEquals(20, ctr.getCount());


    // check master state
    master.getAckMan().dumpLog();
    Set<String> pending = master.getAckMan().getPending();
    assertEquals(3, pending.size());


    // check to make sure wal files are gone.
    node.getAckChecker().checkAcks();
    node.getAckChecker().checkRetry();
    master.getAckMan().dumpLog();
    pending = master.getAckMan().getPending();
    LOG.info("Number of pending acks (retry had failure)" + pending.size());
    assertEquals(0, pending.size());
    LOG.info("Event count 20  (one message missing/lost) ==" + ctr.getCount());
    assertEquals(20, ctr.getCount());


    Clock.sleep(1000); // somehow force other thread to go.


    // // Phase II - force message resent retry but one is dropped. (only 4 make
    // it)
    node.getAckChecker().checkAcks();
    node.getAckChecker().forceRetry();


    Clock.sleep(500); // somehow force other thread to go.


    LOG.info("Was stuff resent? " + " 24 == " + ctr.getCount() + " sent");
    // This retry succeeds sending 4 more messages
    assertEquals(24, ctr.getCount());
    assertEquals(0, master.getAckMan().getPending().size());


    // // Phase III - force another retry
    node.getAckChecker().checkAcks();
    node.getAckChecker().forceRetry();
    LOG
        .info("After another check. " + count + " < " + ctr.getCount()
            + " sent");
    Clock.sleep(500);


    // suceeded sending 5 more messages.
    assertEquals(29, ctr.getCount());
    assertEquals(1, master.getAckMan().getPending().size());


    // check the ack.
    node.getAckChecker().checkAcks();


    // we are clean.
    assertEquals(29, ctr.getCount());
    assertEquals(0, master.getAckMan().getPending().size());


    wal.close();
  }

View Full Code Here

Examples of com.cloudera.flume.agent.durability.NaiveFileWALDeco

    String snk = "  { intervalDroppyAppend(16)  => { ackChecker => [ console, counter(\""
        + rpt + "\") ] } }  ";
    Context ctx = new ReportTestingContext();
    EventSink es = FlumeBuilder.buildSink(ctx, snk);


    NaiveFileWALDeco wal = new NaiveFileWALDeco(ctx, es, node.getWalManager(),
        new TimeTrigger(new ProcessTagger(), 100000), node.getAckChecker()
            .getAgentAckQueuer(), 1000);
    wal.open();


    // // Phase I - (19 events arrive, 3 groups make it)
    for (int i = 0; i < count; i++) {
      Event e = new EventImpl(("test message " + i).getBytes());
      wal.append(e);
      if (i % 5 == 4) {
        wal.rotate();
      }
    }


    Clock.sleep(1000); // need other thread to make progress
    // 3 blocks should have successfully made it.
    assertEquals(3, master.getAckMan().getPending().size()); // (g0, g1, g3)


    CounterSink ctr = (CounterSink) ReportManager.get().getReportable(rpt);
    LOG.info(" Ack begin dropped, but all true message went through. 20 == "
        + ctr.getCount() + " sent");
    // We dropped a event message, so we should be down one, 19 of 20 events.
    assertEquals(19, ctr.getCount());


    // check master state
    master.getAckMan().dumpLog();
    Set<String> pending = master.getAckMan().getPending();
    assertEquals(3, pending.size());


    // // Phase II (4 more messages, 23 total, still only 3 groups)
    // check to make sure wal files are gone.
    node.getAckChecker().checkAcks();
    node.getAckChecker().forceRetry();
    Clock.sleep(1000);
    master.getAckMan().dumpLog();
    pending = master.getAckMan().getPending();
    LOG.info("Number of pending acks (retry had failure)" + pending.size());
    assertEquals(0, pending.size());
    LOG.info("Event count 23  (one message missing/lost) ==" + ctr.getCount());
    assertEquals(23, ctr.getCount());


    // //Phase III (5 more messages, 28 total, 4 groups good)
    node.getAckChecker().checkAcks();
    node.getAckChecker().forceRetry();
    Clock.sleep(1000); // somehow force other thread to go.


    LOG.info("Was stuff resent? " + " 28 == " + ctr.getCount() + " sent");
    // This retry succeeds sending 5 more messages
    assertEquals(28, ctr.getCount());
    Clock.sleep(1000); // somehow force other thread to go.
    node.getAckChecker().checkAcks();
    node.getAckChecker().forceRetry();
    LOG.info("After another check. " + " 28 == " + ctr.getCount() + " sent");


    // Nothing happened
    assertEquals(28, ctr.getCount());
    assertEquals(0, master.getAckMan().getPending().size());
    wal.close();


  }

View Full Code Here

Examples of com.cloudera.flume.agent.durability.NaiveFileWALDeco

        + rpt + "\") ] } }  ";
    Context ctx = new ReportTestingContext();
    EventSink es = FlumeBuilder.buildSink(ctx, snk);


    // Big delay values so that test has to force different actions.
    NaiveFileWALDeco wal = new NaiveFileWALDeco(ctx, es, node.getWalManager(),
        new TimeTrigger(new ProcessTagger(), 100000), node.getAckChecker()
            .getAgentAckQueuer(), 1000);
    wal.open();


    // Phase I (20 message sent, 3 groups good, 1 group bad)
    for (int i = 0; i < count; i++) {
      Event e = new EventImpl(("test message " + i).getBytes());
      wal.append(e);
      if (i % 5 == 4) {
        wal.rotate();
      }
    }


    Clock.sleep(1000); // need other thread to make progress
    // three blocks should have successfully made it.
    assertEquals(3, master.getAckMan().getPending().size()); // (g0, g1, g3)


    CounterSink ctr = (CounterSink) ReportManager.get().getReportable(rpt);
    LOG.info(" Ack begin dropped, but all true message went through. 20 == "
        + ctr.getCount() + " sent");
    // We dropped a event message, so we should be down one.
    assertEquals(20, ctr.getCount());


    // check master state
    Clock.sleep(1000); // somehow force other thread to go.
    master.getAckMan().dumpLog();
    Set<String> pending = master.getAckMan().getPending();
    assertEquals(3, pending.size());


    // Phase II (5 more messeage, retry succeeds).
    node.getAckChecker().checkAcks();
    node.getAckChecker().forceRetry();
    Clock.sleep(1000);
    master.getAckMan().dumpLog();
    pending = master.getAckMan().getPending();
    LOG.info("Number of pending acks (retry is success)" + pending.size());
    assertEquals(1, pending.size());
    LOG.info("Event count 25 (one message missing/lost) ==" + ctr.getCount());
    assertEquals(25, ctr.getCount());


    node.getAckChecker().checkAcks();
    Clock.sleep(1000); // somehow force other thread to go.


    LOG.info("Nothing new happened " + " 25 == " + ctr.getCount() + " sent");
    // This retry succeeds sending 5 more messages
    assertEquals(25, ctr.getCount());
    pending = master.getAckMan().getPending();
    assertEquals(0, pending.size());


    wal.close();
  }

View Full Code Here

Examples of com.cloudera.flume.agent.durability.NaiveFileWALDeco

    int count = 20;
    String rpt = "foo";
    String snk = "{ ackChecker => [ console, counter(\"" + rpt + "\") ] }";
    Context ctx = new ReportTestingContext();
    EventSink es = FlumeBuilder.buildSink(ctx, snk);
    NaiveFileWALDeco wal = new NaiveFileWALDeco(ctx, es, node.getWalManager(),
        new TimeTrigger(new ProcessTagger(), 100), node.getAckChecker()
            .getAgentAckQueuer(), 10000);


    // open and send data.
    wal.open();
    for (int i = 0; i < count; i++) {
      Event e = new EventImpl(("test message " + i).getBytes());
      wal.append(e);
      if (i % 5 == 4) {
        wal.rotate();
      }
    }


    Clock.sleep(1000); // somehow force other thread to go.
    CounterSink ctr = (CounterSink) ReportManager.get().getReportable(rpt);
    LOG.info(" All should make it through here. " + count + " =="
        + ctr.getCount() + " sent");
    assertEquals(20, ctr.getCount());


    // check master state
    Clock.sleep(1000);
    master.getAckMan().dumpLog();


    Set<String> pending = master.getAckMan().getPending();
    assertEquals(4, pending.size());
    // acks should be at the master, but not check by the agent yet, thus 4
    // acks remaining.


    // To simulate the master not getting an ack message, we clear the tag on
    // the master. This will eventually force the child to retry
    // Artificially ack one so the agent doesn't get the ack.
    master.getAckMan().check(pending.iterator().next());


    node.getAckChecker().checkAcks();
    master.getAckMan().dumpLog();
    Set<String> pending2 = master.getAckMan().getPending();
    Clock.sleep(1000);
    // acks are checked so masters can drop state information
    assertEquals(0, pending2.size());
    assertEquals(1, node.getAckChecker().pending.size());


    // Do retry stuff.
    node.getAckChecker().forceRetry();
    Clock.sleep(1000);
    LOG.info("Expected 1 ack pending on master but instead had "
        + master.getAckMan().getPending().size());
    assertEquals(1, master.getAckMan().getPending().size());
    assertEquals(1, node.getAckChecker().pending.size());
    node.getAckChecker().checkAcks();
    assertEquals(0, master.getAckMan().getPending().size());
    assertEquals(0, node.getAckChecker().pending.size());
    wal.close();
  }

View Full Code Here

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.