Examples of com.cloudera.flume.reporter.ReportEvent

com.cloudera.flume.reporter.ReportEvent
Reports are just events that have some helpers for quickly outputting data in human readable or machine parseable formats. For now, we use a convention where a attribute starting with "rpt." is a a reported field. Later we may include an Avro schema or something to include type information as well.

    return rpt;
  }


  @Override
  public ReportEvent getMetrics() {
    ReportEvent rpt = new ReportEvent(getName());
    // TODO add number of nodes present.
    return rpt;
  }

View Full Code Here

      Event e = new EventImpl(("attempt " + i).getBytes());
      sink.append(e);
    }
    Assert.assertEquals(ok.get(), 100);


    ReportEvent rpt = sink.getMetrics();
    // why isn't this 25?
    Assert.assertEquals(new Long(24), rpt
        .getLongMetric(StubbornAppendSink.A_FAILS));
    Assert.assertEquals(new Long(24), rpt
        .getLongMetric(StubbornAppendSink.A_RECOVERS));


  }

View Full Code Here

      InterruptedException {
    EventSink mock = mock(EventSink.class);
    // two ok, and then two exception throwing cases
    doNothing().doNothing().doThrow(new IOException()).doThrow(
        new IOException()).when(mock).append(Mockito.<Event> anyObject());
    doReturn(new ReportEvent("stub")).when(mock).getMetrics();


    StubbornAppendSink<EventSink> sink = new StubbornAppendSink<EventSink>(mock);
    Event e = new EventImpl("foo".getBytes());
    sink.open();
    sink.append(e);
    sink.append(e);
    try {
      sink.append(e);
    } catch (Exception exn) {
      ReportEvent rpt = sink.getMetrics();
      Assert.assertEquals(new Long(2), rpt
          .getLongMetric(StubbornAppendSink.A_SUCCESSES));
      Assert.assertEquals(new Long(1), rpt
          .getLongMetric(StubbornAppendSink.A_FAILS));
      Assert.assertEquals(new Long(0), rpt
          .getLongMetric(StubbornAppendSink.A_RECOVERS));
      return;
    }
    Assert.fail("should have thrown exception");
  }

View Full Code Here

      FlumeSpecException, IOException, InterruptedException {
    ReportTestUtils.setupSinkFactory();


    EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(),
        "stubbornAppend one");
    ReportEvent rpt = ReportUtil.getFlattenedReport(snk);
    LOG.info(ReportUtil.toJSONObject(rpt).toString());
    assertNotNull(rpt.getLongMetric(StubbornAppendSink.A_FAILS));
    assertNotNull(rpt.getLongMetric(StubbornAppendSink.A_RECOVERS));
    assertNotNull(rpt.getLongMetric(StubbornAppendSink.A_SUCCESSES));
    assertEquals("One", rpt.getStringMetric("One.name"));


  }

View Full Code Here

    return "naiveWal";
  }


  @Override
  synchronized public ReportEvent getMetrics() {
    ReportEvent rpt = new ReportEvent(getName());


    // historical counts
    rpt.setLongMetric(A_IMPORTED, importedCount.get());
    rpt.setLongMetric(A_WRITING, writingCount.get());
    rpt.setLongMetric(A_LOGGED, loggedCount.get());
    rpt.setLongMetric(A_SENDING, sendingCount.get());
    rpt.setLongMetric(A_SENT, sentCount.get());
    rpt.setLongMetric(A_ACKED, ackedCount.get());
    rpt.setLongMetric(A_RETRY, retryCount.get());
    rpt.setLongMetric(A_ERROR, errCount.get());
    rpt.setLongMetric(A_RECOVERED, recoverCount.get());


    // Waiting to send
    rpt.setLongMetric(A_IN_LOGGED, loggedQ.size());


    // waiting for ack
    rpt.setLongMetric(A_IN_SENT, sentQ.size());


    rpt.setStringMetric("sentGroups", Arrays.toString(sentQ.toArray()));
    rpt.setStringMetric("loggedGroups", Arrays.toString(loggedQ.toArray()));
    return rpt;
  }

View Full Code Here

    return "AgentWALAckManager";
  }


  @Override
  synchronized public ReportEvent getMetrics() {
    ReportEvent rpt = new ReportEvent(getName());
    Attributes.setLong(rpt, A_RETRANSMIT_TIMEOUT, retransmitTime);
    StringBuilder pendingAcks = new StringBuilder();
    for (Map.Entry<String, Long> e : pending.entrySet()) {
      pendingAcks.append(e.getKey());
      pendingAcks.append(":");

View Full Code Here

      }
      snk.endBatchTimeout();
    }
    snk.close();


    ReportEvent rpt = snk.getMetrics();
    LOG.info(rpt.toString());
    assertEquals(0, (long) rpt.getLongMetric(BatchingDecorator.R_FILLED));
    assertEquals(8, (long) rpt.getLongMetric(BatchingDecorator.R_EMPTY));
    // Extra trigger happens on close
    assertEquals(11, (long) rpt.getLongMetric(BatchingDecorator.R_TRIGGERS));
  }

View Full Code Here

    EventUtil.dumpAll(src, batch);
    src.close();
    batch.close();


    // check metrics.
    ReportEvent ubRpt = unbatch.getMetrics();
    assertEquals(40, (long) ubRpt
        .getLongMetric(UnbatchingDecorator.R_BATCHED_IN));
    assertEquals(200, (long) ubRpt
        .getLongMetric(UnbatchingDecorator.R_BATCHED_OUT));
    assertEquals(0, (long) ubRpt
        .getLongMetric(UnbatchingDecorator.R_PASSTHROUGH));


    ReportEvent bRpt = batch.getMetrics();
    assertEquals(1, (long) bRpt.getLongMetric(BatchingDecorator.R_EMPTY));
    assertEquals(40, (long) bRpt.getLongMetric(BatchingDecorator.R_FILLED));
    assertEquals(0, (long) bRpt.getLongMetric(BatchingDecorator.R_TIMEOUTS));
    assertEquals(41, (long) bRpt.getLongMetric(BatchingDecorator.R_TRIGGERS));


    ReportEvent gzr = gzip.getMetrics();
    assertTrue(gzr.getLongMetric(GzipDecorator.R_GZIPSIZE) < gzr
        .getLongMetric(GzipDecorator.R_EVENTSIZE));
    assertEquals(40, (long) gzr.getLongMetric(GzipDecorator.R_EVENTCOUNT));


    ReportEvent ugzr = gunzip.getMetrics();
    assertTrue(ugzr.getLongMetric(GunzipDecorator.R_GZIPSIZE) < ugzr
        .getLongMetric(GunzipDecorator.R_GUNZIPSIZE));
    assertEquals(40, (long) ugzr.getLongMetric(GunzipDecorator.R_GZIPCOUNT));
    assertEquals(0, (long) ugzr.getLongMetric(GunzipDecorator.R_PASSTHROUGH));
  }

View Full Code Here

    return this.uniqueMasterName;
  }


  @Override
  public ReportEvent getMetrics() {
    ReportEvent rpt = new ReportEvent(getName());


    rpt.setStringMetric(REPORTKEY_HOSTNAME, NetUtils.localhost());


    rpt.setLongMetric(REPORTKEY_NODES_REPORTING_COUNT, this.getKnownNodes()
        .size());


    return rpt;
  }

View Full Code Here

    walman.stopDrains();
    try {
      LOG.debug("Waiting for subthread to complete .. ");
      int maxNoProgressTime = 10;


      ReportEvent rpt = sink.getMetrics();


      Long levts = rpt.getLongMetric(EventSink.Base.R_NUM_EVENTS);
      long evts = (levts == null) ? 0 : levts;
      int count = 0;
      while (true) {
        if (conn.join(500)) {
          // driver successfully completed
          LOG.debug(".. subthread to completed");
          break;
        }


        // driver still running, did we make progress?
        ReportEvent rpt2 = sink.getMetrics();
        Long levts2 = rpt2.getLongMetric(EventSink.Base.R_NUM_EVENTS);
        long evts2 = (levts2 == null) ? 0 : levts;
        if (evts2 > evts) {
          // progress is being made, reset counts
          count = 0;
          evts = evts2;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.cloudera.flume.reporter.ReportEvent

com.cloudera.flume.agent.AgentSink

com.cloudera.flume.agent.diskfailover.DiskFailoverDeco

com.cloudera.flume.agent.diskfailover.NaiveFileFailoverManager

com.cloudera.flume.agent.diskfailover.TestConcurrentDFOMan

com.cloudera.flume.agent.diskfailover.TestDiskFailoverDeco

com.cloudera.flume.agent.durability.NaiveFileWALDeco

com.cloudera.flume.agent.durability.NaiveFileWALManager

com.cloudera.flume.agent.FlumeNode

com.cloudera.flume.agent.LogicalNode

com.cloudera.flume.agent.LogicalNodeManager

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.