Examples of com.cloudera.flume.reporter.aggregator.AccumulatorSink

Package com.cloudera.flume.reporter.aggregator

Examples of com.cloudera.flume.reporter.aggregator.AccumulatorSink

com.cloudera.flume.reporter.aggregator.AccumulatorSink
This just counts the number of entries appended. Unlike the CounterSink, this sink does not reset the count when opened/closed

    src.open();
    snk.open();
    EventUtil.dumpAll(src, snk);
    src.close();
    snk.close();
    AccumulatorSink cnt = (AccumulatorSink) ReportManager.get().getReportable(
        "count");
    assertEquals(LINES, cnt.getCount());
    LOG.info("== failover stop");
  }

View Full Code Here

      snk.open();
      EventUtil.dumpAll(src, snk);
      src.close();
      snk.close();
    }
    AccumulatorSink cnt = (AccumulatorSink) ReportManager.get().getReportable(
        "count");
    assertEquals(LINES, cnt.getCount());
    LOG.info("== node stop");
  }

View Full Code Here

      new Thread("Concurrent-" + i) {
        @Override
        public void run() {
          try {
            File f1 = FileUtil.mktempdir();
            AccumulatorSink cnt1 = new AccumulatorSink("count." + idx);
            DiskFailoverManager dfoMan = new NaiveFileFailoverManager(f1);
            dfos[idx] = dfoMan; // save for checking.


            // short trigger causes lots of rolls
            EventSink snk = new DiskFailoverDeco(cnt1, LogicalNodeContext
                .testingContext(), dfoMan, new TimeTrigger(100), 50);


            ReportManager.get().add(cnt1);
            // make each parallel instance send a slightly different number of
            // messages.
            EventSource src = new NoNlASCIISynthSource(events + idx, 100);


            src.open();
            snk.open();


            started.countDown();


            EventUtil.dumpAll(src, snk);
            src.close();
            snk.close(); // this triggers a flush of current file!?
            FileUtil.rmr(f1);
          } catch (Exception e) {
            LOG.error(e, e);
          } finally {
            done.countDown();
          }


        }
      }.start();


    }


    started.await();
    boolean ok = done.await(timeout, TimeUnit.MILLISECONDS);
    assertTrue("Test timed out", ok);


    for (int i = 0; i < threads; i++) {
      AccumulatorSink cnt = (AccumulatorSink) ReportManager.get()
          .getReportable("count." + i);
      // check for the slightly different counts based on thread.
      int exp = events + i;
      LOG
          .info("count." + i + " expected " + exp + " and got "
              + cnt.getCount());
      assertEquals(exp, (int) cnt.getCount());


      // check dfo reports to see if they are sane.
      ReportEvent rpt = dfos[i].getMetrics();
      LOG.info(rpt);
      long failovered = rpt.getLongMetric(DiskFailoverManager.A_MSG_WRITING);

View Full Code Here

  public void testExistingDir() throws IOException, InterruptedException {
    File tmpdir = FileUtil.mktempdir();
    File subDir = new File(tmpdir, "subdir");
    subDir.mkdirs();
    TailDirSource src = new TailDirSource(tmpdir, ".*");
    AccumulatorSink cnt = new AccumulatorSink("tailcount");
    src.open();
    cnt.open();
    DirectDriver drv = new DirectDriver(src, cnt);


    genFiles(tmpdir, "foo", 10, 100);


    drv.start();
    Clock.sleep(1000);
    assertEquals(1000, cnt.getCount());


    drv.stop();
    src.close();
    cnt.close();
    FileUtil.rmr(tmpdir);


    // only did 10 files, ignored the dir.
    assertEquals(Long.valueOf(10), src.getMetrics().getLongMetric(
        TailDirSource.A_FILESADDED));

View Full Code Here

   */
  @Test
  public void testExistingAddFiles() throws IOException, InterruptedException {
    File tmpdir = FileUtil.mktempdir();
    TailDirSource src = new TailDirSource(tmpdir, ".*");
    AccumulatorSink cnt = new AccumulatorSink("tailcount");
    src.open();
    cnt.open();
    DirectDriver drv = new DirectDriver(src, cnt);


    genFiles(tmpdir, "foo", 10, 100);


    drv.start();
    Clock.sleep(1000);
    assertEquals(1000, cnt.getCount());


    // more files show up
    genFiles(tmpdir, "foo", 10, 100);
    Clock.sleep(1000);
    assertEquals(2000, cnt.getCount());


    drv.stop();
    src.close();
    cnt.close();
    FileUtil.rmr(tmpdir);


  }

View Full Code Here

  @Test
  public void testExistingRemoveFiles() throws IOException,
      InterruptedException {
    File tmpdir = FileUtil.mktempdir();
    TailDirSource src = new TailDirSource(tmpdir, ".*");
    AccumulatorSink cnt = new AccumulatorSink("tailcount");
    src.open();
    cnt.open();
    DirectDriver drv = new DirectDriver(src, cnt);


    genFiles(tmpdir, "foo", 10, 100);


    drv.start();
    Clock.sleep(1000);
    assertEquals(1000, cnt.getCount());


    FileUtil.rmr(tmpdir);


    Clock.sleep(1000);
    assertEquals(1000, cnt.getCount());


    drv.stop();
    src.close();
    cnt.close();
    FileUtil.rmr(tmpdir);
  }

View Full Code Here

    File tmpdir = FileUtil.mktempdir();
    // generating files: emulating their existence prior to sink opening
    genFiles(tmpdir, "foo", 10, 100);


    TailDirSource src = new TailDirSource(tmpdir, ".*", true);
    AccumulatorSink cnt = new AccumulatorSink("tailcount");
    src.open();
    cnt.open();
    DirectDriver drv = new DirectDriver(src, cnt);


    drv.start();
    Clock.sleep(1000);
    assertEquals(0, cnt.getCount());


    // adding lines to existing files
    addLinesToExistingFiles(tmpdir, 10);


    Clock.sleep(1000);
    assertEquals(10 * 10, cnt.getCount());


    // generating new files
    genFiles(tmpdir, "bar", 10, 100);


    Clock.sleep(1000);
    assertEquals(10 * 10 + 1000, cnt.getCount());


    drv.stop();
    src.close();
    cnt.close();
    FileUtil.rmr(tmpdir);


    // in total 20 files were added
    assertEquals(Long.valueOf(20), src.getMetrics().getLongMetric(
        TailDirSource.A_FILESADDED));

View Full Code Here

    File subDirL3 = new File(subDirL2, "subdir-level3");
    subDirL3.mkdirs();
    genFiles(subDirL3, "2tail-old3", 10, 30);


    TailDirSource src = new TailDirSource(tmpdir, ".(2tail)?.*", true, 2);
    AccumulatorSink cnt = new AccumulatorSink("tailcount");
    src.open();
    cnt.open();
    DirectDriver drv = new DirectDriver(src, cnt);


    drv.start();
    Clock.sleep(1000);
    assertEquals(0, cnt.getCount());


    // adding lines to existing files
    addLinesToExistingFiles(tmpdir, 10);
    addLinesToExistingFiles(subDirL1, 20);
    addLinesToExistingFiles(subDirL2, 30);
    addLinesToExistingFiles(subDirL3, 40);


    Clock.sleep(1000);
    int expEventsCount = 10 * 10 + 10 * 20 + 10 * 30;
    assertEquals(expEventsCount, cnt.getCount());


    // generating new files
    genFiles(tmpdir, "2tail-new", 10, 40);
    genFiles(subDirL1, "2tail-new1", 10, 30);
    genFiles(subDirL2, "2tail-new2", 10, 20);
    genFiles(subDirL3, "2tail-new3", 10, 10);


    Clock.sleep(1000);
    expEventsCount += 10 * 40 + 10 * 30 + 10 * 20;
    assertEquals(expEventsCount, cnt.getCount());


    // creating new subdir on level 2
    File newSubDirL2 = new File(subDirL1, "subdir-level2-new");
    newSubDirL2.mkdirs();
    genFiles(newSubDirL2, "2tail-new2-new", 10, 100);


    Clock.sleep(1000);
    expEventsCount += 10 * 100;
    assertEquals(expEventsCount, cnt.getCount());


    // deleting some dirs
    FileUtil.rmr(subDirL3);
    FileUtil.rmr(subDirL2);


    Clock.sleep(1000);
    assertEquals(expEventsCount, cnt.getCount());


    // adding back deleted dir, checking that it is caught up
    subDirL2.mkdirs();
    genFiles(subDirL2, "2tail-new2", 10, 100);
    Clock.sleep(1000);
    expEventsCount += 10 * 100;
    assertEquals(expEventsCount, cnt.getCount());


    drv.stop();
    src.close();
    cnt.close();
    FileUtil.rmr(tmpdir);


    ReportEvent report = src.getMetrics();
    assertEquals(Long.valueOf(80), report
        .getLongMetric(TailDirSource.A_FILESADDED));

View Full Code Here

    // Windows semantics for rm different from unix
    Assume.assumeTrue(!OSUtils.isWindowsOS());


    File tmpdir = FileUtil.mktempdir();
    TailDirSource src = new TailDirSource(tmpdir, ".*");
    AccumulatorSink cnt = new AccumulatorSink("tailcount");
    src.open();
    cnt.open();
    DirectDriver drv = new DirectDriver(src, cnt);


    drv.start();


    // This blows up when there are ~2000 files
    genFiles(tmpdir, "foo", 200, 10);
    Clock.sleep(1000);
    assertEquals(2000, cnt.getCount());


    ReportEvent rpt1 = src.getMetrics();
    assertEquals(Long.valueOf(200), rpt1
        .getLongMetric(TailDirSource.A_FILESPRESENT));


    FileUtil.rmr(tmpdir); // This fails in windows because taildir keeps file
    // open
    tmpdir.mkdirs();
    Clock.sleep(1000);
    assertEquals(2000, cnt.getCount());


    ReportEvent rpt = src.getMetrics();
    assertEquals(rpt.getLongMetric(TailDirSource.A_FILESADDED), rpt
        .getLongMetric(TailDirSource.A_FILESDELETED));
    assertEquals(Long.valueOf(0), rpt
        .getLongMetric(TailDirSource.A_FILESPRESENT));


    drv.stop();
    src.close();
    cnt.close();
    FileUtil.rmr(tmpdir);
  }

View Full Code Here

   */
  @Test
  public void testNoneToNewFiles() throws IOException, InterruptedException {
    File tmpdir = FileUtil.mktempdir();
    TailDirSource src = new TailDirSource(tmpdir, ".*");
    AccumulatorSink cnt = new AccumulatorSink("tailcount");
    src.open();
    cnt.open();
    DirectDriver drv = new DirectDriver(src, cnt);


    drv.start();
    Clock.sleep(1000);


    genFiles(tmpdir, "foo", 10, 100);


    Clock.sleep(1000);


    assertEquals(1000, cnt.getCount());


    drv.stop();
    src.close();
    cnt.close();
    FileUtil.rmr(tmpdir);
  }

View Full Code Here

0 1 2 3 4 5

TOP

Related Classes of com.cloudera.flume.reporter.aggregator.AccumulatorSink

com.cloudera.flume.agent.diskfailover.TestConcurrentDFOMan

com.cloudera.flume.agent.diskfailover.TestDiskFailoverBehavior

com.cloudera.flume.conf.SinkFactory.SinkBuilder

com.cloudera.flume.conf.TestFlumeBuilderFunctional

com.cloudera.flume.core.EventSink

com.cloudera.flume.handlers.text.TestTailDirSource

com.cloudera.flume.reporter.ReportEvent

com.cloudera.flume.shell.TestFlumeShell

java.util.concurrent.atomic.AtomicLong

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.