Examples of com.cloudera.flume.handlers.debug.MemorySinkSource

Package com.cloudera.flume.handlers.debug

Examples of com.cloudera.flume.handlers.debug.MemorySinkSource

com.cloudera.flume.handlers.debug.MemorySinkSource
This is for throughput testing. Here we put a set of events into memory and then stream them through the system at max rate. It is both a sink and a source. The idea is to use it as a sink where everything is streamed in and then use it as a source for some downstream sink. This will let us determine what network and disk bandwidth limiters are.


  @Test
  public void testAvroDataFileWriteRead() throws IOException,
      FlumeSpecException {


    MemorySinkSource mem = MemorySinkSource.cannedData("test ", 5);


    // setup sink.
    File f = File.createTempFile("avrodata", ".avro");
    f.deleteOnExit();
    LOG.info("filename before escaping: " + f.getAbsolutePath());
    String custom = "text(\""
        + StringEscapeUtils.escapeJava(f.getAbsolutePath())
        + "\", \"avrodata\")";
    LOG.info("sink to parse: " + custom);
    EventSink snk = FlumeBuilder.buildSink(new Context(), custom);
    snk.open();
    mem.open();
    EventUtil.dumpAll(mem, snk);
    snk.close();


    mem.open();
    DatumReader<EventImpl> dtm = new ReflectDatumReader<EventImpl>(
        EventImpl.class);
    DataFileReader<EventImpl> dr = new DataFileReader<EventImpl>(f, dtm);


    EventImpl eout = null;
    for (Object o : dr) {
      eout = (EventImpl) o; // TODO (jon) fix AVRO -- this is gross
      Event expected = mem.next();
      Assert.assertTrue(Arrays.equals(eout.getBody(), expected.getBody()));
    }


  }

View Full Code Here

  }


  // this version checks to make sure it remains the same.
  @Test
  public void testReorderDecoratorCheck() throws IOException {
    MemorySinkSource mss = new MemorySinkSource();
    ReorderDecorator<EventSink> reorder =
        new ReorderDecorator<EventSink>(mss, .5, .5, 0);
    reorder.open();
    for (int i = 0; i < 10; i++) {
      Event e = new EventImpl(new byte[0]);
      e.set("order", ByteBuffer.allocate(4).putInt(i).array());
      reorder.append(e);
    }
    System.out.println("closing");
    reorder.close();
    int[] order = { 0, 2, 1, 4, 3, 5, 7, 6, 8, 9 };
    for (int i = 0; i < order.length; i++) {
      Event e = mss.next();
      int j = ByteBuffer.wrap(e.get("order")).asIntBuffer().get();
      assertEquals(j, order[i]);
    }


  }

View Full Code Here

   * Send messages in order and make sure they check out
   */
  @Test
  public void testCheckChecker() throws IOException {
    int msgs = 100;
    MemorySinkSource mss = new MemorySinkSource();
    AckChecksumChecker<EventSink> acc = new AckChecksumChecker<EventSink>(mss);
    AckChecksumInjector<EventSink> aci =
        new AckChecksumInjector<EventSink>(acc);


    aci.open();
    for (int i = 0; i < msgs; i++) {
      Event e = new EventImpl(("this is a test " + i).getBytes());
      aci.append(e);
    }
    aci.close(); // will throw exception if checksum doesn't match


    Event eo = null;
    int count = 0;
    while ((eo = mss.next()) != null) {
      System.out.println(eo);
      count++;
    }


    assertEquals(msgs, count); // extra ack messages should have been consumed

View Full Code Here

  /**
   * Send messages when some message are reordered and make sure they check out
   */
  @Test
  public void testReorderedChecker() throws IOException {
    MemorySinkSource mss = new MemorySinkSource();
    AckChecksumChecker<EventSink> cc = new AckChecksumChecker<EventSink>(mss);
    ReorderDecorator<EventSink> ro =
        new ReorderDecorator<EventSink>(cc, .5, .5, 0);
    AckChecksumInjector<EventSink> cp = new AckChecksumInjector<EventSink>(ro);


    cp.open();
    for (int i = 0; i < 100; i++) {
      Event e = new EventImpl(("this is a test " + i).getBytes());
      cp.append(e);
    }
    cp.close();


    Event eo = null;
    while ((eo = mss.next()) != null) {
      System.out.println(eo);
    }
  }

View Full Code Here

   * error case: no start message
   */
  @Test
  public void testNoStart() throws IOException {
    final int msgs = 100;
    MemorySinkSource mss = new MemorySinkSource();
    AckChecksumChecker<EventSink> cc = new AckChecksumChecker<EventSink>(mss);
    EventSinkDecorator<EventSink> dropFirst =
        new EventSinkDecorator<EventSink>(cc) {
          int count = 0;

View Full Code Here

   * the long term.
   */
  @Test
  public void testNoStop() throws IOException {
    final int msgs = 100;
    MemorySinkSource mss = new MemorySinkSource();
    AckChecksumChecker<EventSink> cc = new AckChecksumChecker<EventSink>(mss);
    EventSinkDecorator<EventSink> dropStop =
        new EventSinkDecorator<EventSink>(cc) {
          int drop = msgs + 1; // intial + messages, the drop the last
          int count = 0;

View Full Code Here

   * error case: duplicate/dropped message (bad checksum)
   */
  @Test
  public void testDupe() throws IOException {
    final int msgs = 100;
    MemorySinkSource mss = new MemorySinkSource();
    AckChecksumChecker<EventSink> cc =
        new AckChecksumChecker<EventSink>(mss, new AckListener.Empty() {
          @Override
          public void err(String group) throws IOException {
            throw new IOException("Fail");

View Full Code Here

      FlumeSpecException, InterruptedException {
    File f = File.createTempFile("multitemp1", ".tmp");
    f.deleteOnExit();
    File f2 = File.createTempFile("multitemp2", ".tmp");
    f2.deleteOnExit();
    final MemorySinkSource snk = new MemorySinkSource();
    final EventSource src = TailSource.multiTailBuilder().build(
        f.getAbsolutePath(), f2.getAbsolutePath());
    final CountDownLatch done = new CountDownLatch(1);
    final int count = 60;
    Thread t = new Thread() {
      @Override
      public void run() {
        try {
          src.open();
          snk.open();
          EventUtil.dumpN(count, src, snk);
          src.close();
          snk.close();
          done.countDown();
        } catch (IOException e) {
          e.printStackTrace();
        }
      }
    };
    t.start();


    int log1 = 0, log2 = 0;
    FileWriter fw = new FileWriter(f);
    FileWriter fw2 = new FileWriter(f2);
    for (int i = 0; i < count; i++) {
      if (Math.random() > 0.5) {
        fw.append("Line " + i + "\n");
        fw.flush();
        log1++;
      } else {
        fw2.append("Line " + i + "\n");
        fw2.flush();
        log2++;
      }
    }
    fw.close();
    fw2.close();


    assertTrue(done.await(15, TimeUnit.SECONDS));


    Event e = null;
    while ((e = snk.next()) != null) {
      byte[] fn = e.get(TailSource.A_TAILSRCFILE);
      String sfn = new String(fn);
      if (!sfn.equals(f.getName()) && !sfn.equals(f2.getName())) {
        Assert.fail("didn't have tail src file metadata! " + sfn + " != "
            + f.getName() + " or " + f2.getName());

View Full Code Here

    Attributes.setString(e, "duped", "second");
  }


  @Test
  public void testMaskNoConflict() throws IOException {
    MemorySinkSource mem = new MemorySinkSource();
    EventSink s1 = new ValueDecorator<EventSink>(mem, "duped", "second"
        .getBytes());
    EventSink s2 = new MaskDecorator<EventSink>(s1, "duped");
    EventSink snk = new ValueDecorator<EventSink>(s2, "duped", "first"
        .getBytes());
    snk.open();


    Event e = new EventImpl("foo".getBytes());
    snk.append(e);
    snk.close();


    Event e2 = mem.next();
    assertEquals("second", Attributes.readString(e2, "duped"));
  }

View Full Code Here

  public void testProbabilitySampler() throws IOException {
    Benchmark b = new Benchmark("Reservoir sampler + nullsink");
    b.mark("begin");
    TextFileSource txt = new TextFileSource(HADOOP_DATA[0]);
    txt.open();
    MemorySinkSource mem = new MemorySinkSource();
    mem.open();
    EventUtil.dumpAll(txt, mem);


    b.mark("disk_loaded");


    NullSink nullsnk = new NullSink();
    EventUtil.dumpAll(mem, nullsnk);
    b.mark("warmup");


    double probs[] = { .00001, .0001, .001, .01 };
    for (int i = 0; i < probs.length; i++) {
      mem.open();
      double prob = probs[i];
      EventSink res = new ProbabilitySampler<NullSink>(new NullSink(), prob);
      EventUtil.dumpAll(mem, res);
      b.mark("probability" + prob + " sampling done", prob);


      res.close();
      b.mark("sample dump done");
    }


    for (int i = 0; i < probs.length; i++) {
      mem.open();
      double prob = probs[i];
      CounterSink cnt = new CounterSink("null");
      EventSink res = new ProbabilitySampler<CounterSink>(cnt, prob);
      EventUtil.dumpAll(mem, res);
      b.mark("probability", prob);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.cloudera.flume.handlers.debug.MemorySinkSource

com.cloudera.flume.agent.durability.NaiveFileWALManager

com.cloudera.flume.agent.TestAckedWALDecorator

com.cloudera.flume.collector.TestCollectorSink

com.cloudera.flume.core.Event

com.cloudera.flume.core.EventImpl

com.cloudera.flume.core.extractors.TestExtractors

com.cloudera.flume.core.TestDigestDecorator

com.cloudera.flume.core.TestFormatterDecorator

com.cloudera.flume.core.TestSelectMaskDecos

com.cloudera.flume.FlumeBenchmarkHarness

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.