Examples of DbusEventsStatisticsCollector

com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector
Container for all monitoring mbeans

Examples of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector

        relayBuffer.start(0);
        writeEventsToBuffer(relayBuffer, eventInfos, 4);


        //prepare stream response
        Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
        final DbusEventsStatisticsCollector stats =
            new DbusEventsStatisticsCollector(1, "test1", true, false, null);


        // create ChunnelBuffer and fill it with events from relayBuffer
        ChannelBuffer streamResPrefix =
            NettyTestUtils.streamToChannelBuffer(relayBuffer, cp, 20000, stats);

View Full Code Here

Examples of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector

          //process /stream call and return a partial window
          NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*", 1000);
          objCapture.clear();


          //send back the /stream response
          final DbusEventsStatisticsCollector stats =
              new DbusEventsStatisticsCollector(1, "test1", true, false, null);
          Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
          ChannelBuffer streamRes = NettyTestUtils.streamToChannelBuffer(relayBuffer[0], cp,
                                                                          resp1EnfOfs, stats);
          HttpResponse streamResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1,
                                                             HttpResponseStatus.OK);
          streamResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
          streamResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
          NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp,
                                             new DefaultHttpChunk(streamRes));


          //make sure the client processes the response correctly
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              log.debug("LastWritten SCN:" + clientConn.getDataEventsBuffer().lastWrittenScn() );
              return clientConn.getDataEventsBuffer().lastWrittenScn() == 30;
            }
          }, "client receives /stream response", 1100, log);


          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              log.error("events num=" + consumer.getEventNum());
              return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
            }
          }, "client processes /stream response", 110000, log);
          assertEquals(-1, consumer.getRollbackScn());
          int rollbackNum = 0;
          assertEquals(stats.getTotalStats().getNumSysEvents() + 1 + rollbackNum,
                       consumer.getWinNum());


          List<DbusEventKey> expKeys = eventKeys.get(0).subList(0, (int)stats.getTotalStats().getNumDataEvents());
          List<Long> expSeqs = new ArrayList<Long>();
          for (int i = 0; i < stats.getTotalStats().getNumDataEvents(); i++)
            expSeqs.add(30L);


          long numEvents = stats.getTotalStats().getNumDataEvents();


          assertEquals("Keys", expKeys, consumer.getKeys());
          assertEquals("Sequences", expSeqs, consumer.getSequences());


          assertEquals("Keys", expKeys, consumer.getKeys());
          assertEquals("Sequences", expSeqs, consumer.getSequences());


          numEvents = stats.getTotalStats().getNumDataEvents();


          //now kill the relay and wait for a failover
          serverChannel.close();


          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              return null != relayConnInsp.getChannel() && !relayConnInsp.getChannel().isConnected();
            }
          }, "client disconnected", 200, log);


          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              return clientConn.getRelayPullThread().getLastOpenConnection() != relayConn;
            }
          }, "new netty connection", 200, log);


          /////////// FAKING CONNECTION TO NEW RELAY


          final NettyHttpDatabusRelayConnection newRelayConn =
              (NettyHttpDatabusRelayConnection)clientConn.getRelayPullThread().getLastOpenConnection();
          final NettyHttpDatabusRelayConnectionInspector newRelayConnInsp =
              new NettyHttpDatabusRelayConnectionInspector(newRelayConn);


          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              return null != newRelayConnInsp.getChannel() && newRelayConnInsp.getChannel().isConnected();
            }
          }, "client connected to new relay", 200, log);


          //figure out the connection to the relay
          clientChannel = newRelayConnInsp.getChannel();
          relayAddr = (InetSocketAddress)clientChannel.getRemoteAddress();
          clientAddr = clientChannel.getLocalAddress();
          relayPort = relayAddr.getPort();
          log.info("new relay selected: " + relayPort);


          relay = null;
          int relayIdx = 0;
          for (; relayIdx < RELAY_PORT.length; ++relayIdx)
          {
            if (relayPort == RELAY_PORT[relayIdx]) relay = _dummyServer[relayIdx];
          }
          assertTrue(null != relay);


          serverChannel = relay.getChildChannel(clientAddr);
          assertTrue(null != serverChannel);
          serverPipeline = serverChannel.getPipeline();
          objCapture = (SimpleObjectCaptureHandler)serverPipeline.get("3");


          //process the /sources request
          NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
          objCapture.clear();


          //send back the /sources response
          body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" +
                                      SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
          NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);


          //make sure the client processes the response correctly
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
              return "1".equals(idListString);
            }
          }, "client processes /sources response", 100, log);


          //process the /register request
          NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
          objCapture.clear();


          //send back the /register response
          body = new DefaultHttpChunk(
              ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
          NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);


          //make sure the client processes the response correctly
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
              return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
            }
          }, "client processes /register response", 100, log);


          //process /stream call and return a partial window
          Matcher streamMatcher =
              NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*checkPoint=([^&]*)&.*",
                                                1000);
          String cpString = streamMatcher.group(1);
          objCapture.clear();


          int respStartOfs = eventOfs.get(1).get(1);
          int respEndOfs = eventOfs.get(1).get(26);


          cp = new Checkpoint(cpString);
          assertTrue(cp.getWindowOffset() > 0); //last window read was partial
          streamRes = NettyTestUtils.streamToChannelBuffer(relayBuffer[1], cp,
                                                           respEndOfs - respStartOfs,
                                                           stats);
          NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp,
                                             new DefaultHttpChunk(streamRes));


          //make sure the client processes the response correctly
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              log.debug("lastWrittenScn=" + clientConn.getDataEventsBuffer().lastWrittenScn());
              return clientConn.getDataEventsBuffer().lastWrittenScn() == 40;
            }
          }, "client receives /stream response", 1100, log);




          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              log.debug("events num=" + consumer.getEventNum());
              return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
            }
          }, "client processes /stream response", 11000, log);
          assertEquals(20, consumer.getRollbackScn());
          //add one more onStartDataEventsSequence after the rollback()
          ++rollbackNum;
          assertEquals(stats.getTotalStats().getNumSysEvents() + 1 + rollbackNum, consumer.getWinNum());
          expKeys.addAll(eventKeys.get(1).subList(0, (int)(stats.getTotalStats().getNumDataEvents() - numEvents)));
          for (int i = 0; i < (stats.getTotalStats().getNumDataEvents() - numEvents); i++)
            expSeqs.add((i/20)*20 + 20L);


          log.info("Expected NumKeys :" + expKeys.size() + ", Got NumKeys :" + consumer.getKeys().size());
          for(int i = 0; i < expKeys.size(); i++)
          {
            if (! consumer.getKeys().contains(expKeys.get(i)))
              log.error(i + " Key :" + expKeys.get(i) + " missing !!");
            else
              log.info(i + " Key :" + expKeys.get(i) + " present !!");
          }


          assertEquals("Keys", expKeys, consumer.getKeys());
          assertEquals("Sequences", expSeqs, consumer.getSequences());


          assertEquals("Keys", expKeys, consumer.getKeys());
          assertEquals("Sequences", expSeqs, consumer.getSequences());


          numEvents = stats.getTotalStats().getNumDataEvents();


          assertEquals(clientConn.getRelayPullThread().getConnectionState().getDataEventsBuffer().isSCNRegress(), false);
          ///////////////////////////////////
          //simulate a timeout on the server; the client would have sent a /stream call and there
          //will be no response from the server, so eventually it should time out and switch servers
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              log.debug("Channel :" + newRelayConnInsp.getChannel());
              return (null != newRelayConnInsp.getChannel()) && (!newRelayConnInsp.getChannel().isConnected());
            }
          }, "waiting for a reconnect", (long)(DEFAULT_READ_TIMEOUT_MS * 1.5), log);


          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              return clientConn.getRelayPullThread().getLastOpenConnection() != relayConn;
            }
          }, "new netty connection", 200, log);


          final NettyHttpDatabusRelayConnection new2RelayConn =
              (NettyHttpDatabusRelayConnection)clientConn.getRelayPullThread().getLastOpenConnection();
          final NettyHttpDatabusRelayConnectionInspector new2RelayConnInsp =
              new NettyHttpDatabusRelayConnectionInspector(new2RelayConn);


          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              return null != new2RelayConnInsp.getChannel() && new2RelayConnInsp.getChannel().isConnected();
            }
          }, "client connected to third relay", 200, log);


          //figure out the connection to the relay
          clientChannel = new2RelayConnInsp.getChannel();
          relayAddr = (InetSocketAddress)clientChannel.getRemoteAddress();
          clientAddr = clientChannel.getLocalAddress();
          relayPort = relayAddr.getPort();
          log.info("third relay selected: " + relayPort);




          relay = null;
          relayIdx = 0;
          for (; relayIdx < RELAY_PORT.length; ++relayIdx)
          {
            if (relayPort == RELAY_PORT[relayIdx]) relay = _dummyServer[relayIdx];
          }
          assertTrue(null != relay);


          serverChannel = relay.getChildChannel(clientAddr);
          assertTrue(null != serverChannel);
          serverPipeline = serverChannel.getPipeline();
          objCapture = (SimpleObjectCaptureHandler)serverPipeline.get("3");


          //process the /sources request
          NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
          objCapture.clear();


          //send back the /sources response
          body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" +
                                      SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
          NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);


          //make sure the client processes the response correctly
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
              return "1".equals(idListString);
            }
          }, "client processes /sources response", 100, log);


          //process the /register request
          NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
          objCapture.clear();


          //send back the /register response
          body = new DefaultHttpChunk(
              ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
          NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);


          //make sure the client processes the response correctly
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
              return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
            }
          }, "client processes /sources response", 100, log);


          //process /stream call and return a partial window
          streamMatcher =
              NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*checkPoint=([^&]*)&.*",
                                                1000);
          cpString = streamMatcher.group(1);
          objCapture.clear();


          respStartOfs = eventOfs.get(2).get(20);
          respEndOfs = eventOfs.get(2).get(89);


          log.debug("Checkpoint String is :" + cpString);




          cp = new Checkpoint(cpString);
          //last window read was partial. So the client would have reset the windowOffset
          assertTrue("Is WindowOffset Cleared",cp.getWindowOffset() == -1);
          assertEquals( "WindowSCN == PrevSCN. Ckpt :" + cp, cp.getWindowScn(), cp.getPrevScn());
          streamRes = NettyTestUtils.streamToChannelBuffer(relayBuffer[2], cp,
                                                           respEndOfs - respStartOfs,
                                                           stats);
          NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp,
                                             new DefaultHttpChunk(streamRes));


          //make sure the client processes the response correctly
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              log.debug("lastWrittenScn=" + clientConn.getDataEventsBuffer().lastWrittenScn());
              return clientConn.getDataEventsBuffer().lastWrittenScn() == 90;
            }
          }, "client receives /stream response, Sequence :" + consumer.getSequences(), 10100, log);


          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              log.debug("events num=" + consumer.getEventNum());
              return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
            }
          }, "client processes /stream response", 110000, log);


          assertEquals(30, consumer.getRollbackScn());
          //add one more onStartDataEventsSequence because of the rollback
          ++rollbackNum;
          assertEquals(stats.getTotalStats().getNumSysEvents() + 1 + rollbackNum, consumer.getWinNum());


          expKeys.addAll(eventKeys.get(2).subList(20, (int)(stats.getTotalStats().getNumDataEvents() - numEvents + 20)));
          for (int i = 0; i < (stats.getTotalStats().getNumDataEvents() - numEvents); i++)
            expSeqs.add((i/10)*10 + 30L);


          numEvents = stats.getTotalStats().getNumDataEvents();


          assertEquals("Keys", expKeys, consumer.getKeys());
          assertEquals("Sequences Size ", expSeqs.size(), consumer.getSequences().size());
          for (int i = 0; i <expSeqs.size(); i++ )
          {
            System.out.println(i + " Exp : " + expSeqs.get(i) + ", Got :" + consumer.getSequences().get(i));
            if ( expSeqs.get(i) != consumer.getSequences().get(i))
               throw new RuntimeException("Failed at "+ i);
          }
          assertEquals("Sequences", expSeqs, consumer.getSequences());


          numEvents = stats.getTotalStats().getNumDataEvents();
          assertEquals(clientConn.getRelayPullThread().getConnectionState().getDataEventsBuffer().isSCNRegress(), false);


          // Expect none of the keys streamed to be missed.
          expKeys.clear();
          cp = clientConn.getRelayPullThread().getConnectionState().getCheckpoint();

View Full Code Here

Examples of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector

          log.info("process /stream call and return a partial window");
          NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*", 1000);
          objCapture.clear();


          log.info("send back the /stream response");
          final DbusEventsStatisticsCollector stats =
              new DbusEventsStatisticsCollector(1, "test1", true, false, null);
          Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
          ChannelBuffer streamRes = NettyTestUtils.streamToChannelBuffer(relayBuffer[0], cp,
                                                                          resp1EnfOfs, stats);
          HttpResponse streamResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1,
                                                             HttpResponseStatus.OK);
          streamResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
          streamResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
          NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp,
                                             new DefaultHttpChunk(streamRes));


          log.info("make sure the client processes the /stream response correctly");
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              log.debug("LastWritten SCN:" + clientConn.getDataEventsBuffer().lastWrittenScn() );
              return clientConn.getDataEventsBuffer().lastWrittenScn() == 10;
            }
          }, "client receives /stream response", 1100, log);


          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              log.debug("events num=" + consumer.getEventNum());
              return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
            }
          }, "client processes /stream response", 110000, log);
          assertEquals(-1, consumer.getRollbackScn());
          int rollbackNum = 0;
          assertEquals(stats.getTotalStats().getNumSysEvents() + 1 + rollbackNum, consumer.getWinNum());


          List<DbusEventKey> expKeys = eventKeys.get(0).subList(0, (int)stats.getTotalStats().getNumDataEvents());
          List<Long> expSeqs = new ArrayList<Long>();
          for (int i = 0; i < stats.getTotalStats().getNumDataEvents(); i++)
            expSeqs.add(10L);


          long numEvents = stats.getTotalStats().getNumDataEvents();


          assertEquals("Keys", expKeys, consumer.getKeys());
          assertEquals("Sequences", expSeqs, consumer.getSequences());


          log.info("now kill the relay and wait for a failover");
          serverChannel.close();


          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              return null != relayConnInsp.getChannel() && !relayConnInsp.getChannel().isConnected();
            }
          }, "client disconnected", 200, log);


          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              return clientConn.getRelayPullThread().getLastOpenConnection() != relayConn;
            }
          }, "new netty connection", 200, log);


          log.info("/////////// FAKING CONNECTION TO NEW RELAY //////////////");


          final NettyHttpDatabusRelayConnection newRelayConn =
              (NettyHttpDatabusRelayConnection)clientConn.getRelayPullThread().getLastOpenConnection();
          final NettyHttpDatabusRelayConnectionInspector newRelayConnInsp =
              new NettyHttpDatabusRelayConnectionInspector(newRelayConn);


          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              return null != newRelayConnInsp.getChannel() && newRelayConnInsp.getChannel().isConnected();
            }
          }, "client connected to new relay", 200, log);


          log.info("figure out the connection to the relay");
          clientChannel = newRelayConnInsp.getChannel();
          relayAddr = (InetSocketAddress)clientChannel.getRemoteAddress();
          clientAddr = clientChannel.getLocalAddress();
          relayPort = relayAddr.getPort();
          log.info("new relay selected: " + relayPort);


          relay = null;
          int relayIdx = 0;
          for (; relayIdx < RELAY_PORT.length; ++relayIdx)
          {
            if (relayPort == RELAY_PORT[relayIdx]) relay = _dummyServer[relayIdx];
          }
          assertTrue(null != relay);


          serverChannel = relay.getChildChannel(clientAddr);
          assertTrue(null != serverChannel);
          serverPipeline = serverChannel.getPipeline();
          objCapture = (SimpleObjectCaptureHandler)serverPipeline.get("3");


          log.info("process the /sources request");
          NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
          objCapture.clear();


          log.info("send back the /sources response");
          body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" +
                                      SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
          NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);


          log.info("make sure the client processes the response correctly");
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
              return "1".equals(idListString);
            }
          }, "client processes /sources response", 100, log);


          log.info("process the /register request");
          NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
          objCapture.clear();


          log.info("send back the /register response");
          body = new DefaultHttpChunk(
              ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
          NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);


          log.info("make sure the client processes the /register response correctly");
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
              return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
            }
          }, "client processes /register response", 100, log);


          log.info("process /stream call and return a partial window");
          Matcher streamMatcher =
              NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*checkPoint=([^&]*)&.*",
                                                1000);
          String cpString = streamMatcher.group(1);
          objCapture.clear();


          int respStartOfs = eventOfs.get(1).get(1);
          int respEndOfs = eventOfs.get(1).get(34);


          cp = new Checkpoint(cpString);
          assertTrue(cp.getWindowOffset() > 0); //last window read was partial
          streamRes = NettyTestUtils.streamToChannelBuffer(relayBuffer[1], cp,
                                                           respEndOfs - respStartOfs,
                                                           stats);
          NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp,
                                             new DefaultHttpChunk(streamRes));


          log.info("make sure the client processes the response correctly");
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              log.debug("lastWrittenScn=" + clientConn.getDataEventsBuffer().lastWrittenScn());
              return clientConn.getDataEventsBuffer().lastWrittenScn() == 40;
            }
          }, "client receives /stream response", 1100, log);




          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              log.debug("events num=" + consumer.getEventNum());
              return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
            }
          }, "client processes /stream response", 11000, log);
          assertEquals(20, consumer.getRollbackScn());


          log.info("one more onStartDataEventSequence because of the rolback");
          ++rollbackNum;
          assertEquals(stats.getTotalStats().getNumSysEvents() + 1 + rollbackNum, consumer.getWinNum());


          assertEquals(clientConn.getRelayPullThread().getConnectionState().getDataEventsBuffer().isSCNRegress(), false);
          expKeys.addAll(eventKeys.get(1).subList(0, (int)(stats.getTotalStats().getNumDataEvents() - numEvents)));
          for (int i = 0; i < stats.getTotalStats().getNumDataEvents() - numEvents; i++)
            expSeqs.add((i/20)*20 + 20L);


          assertEquals("Keys", expKeys, consumer.getKeys());
          assertEquals("Sequences", expSeqs, consumer.getSequences());


          numEvents = stats.getTotalStats().getNumDataEvents();


          ///////////////////////////////////
          //simulate a timeout on the server; the client would have sent a /stream call and there
          //will be no response from the server, so eventually it should time out and switch servers
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              log.debug("Channel :" + newRelayConnInsp.getChannel());
              return (null != newRelayConnInsp.getChannel()) && (!newRelayConnInsp.getChannel().isConnected());
            }
          }, "waiting for a reconnect", (long)(DEFAULT_READ_TIMEOUT_MS * 1.5), log);


          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              return clientConn.getRelayPullThread().getLastOpenConnection() != relayConn;
            }
          }, "new netty connection", 200, log);


          final NettyHttpDatabusRelayConnection new2RelayConn =
              (NettyHttpDatabusRelayConnection)clientConn.getRelayPullThread().getLastOpenConnection();
          final NettyHttpDatabusRelayConnectionInspector new2RelayConnInsp =
              new NettyHttpDatabusRelayConnectionInspector(new2RelayConn);


          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              return null != new2RelayConnInsp.getChannel() && new2RelayConnInsp.getChannel().isConnected();
            }
          }, "client connected to third relay", 200, log);


          log.info("figure out the connection to the relay");
          clientChannel = new2RelayConnInsp.getChannel();
          relayAddr = (InetSocketAddress)clientChannel.getRemoteAddress();
          clientAddr = clientChannel.getLocalAddress();
          relayPort = relayAddr.getPort();
          log.info("third relay selected: " + relayPort);




          relay = null;
          relayIdx = 0;
          for (; relayIdx < RELAY_PORT.length; ++relayIdx)
          {
            if (relayPort == RELAY_PORT[relayIdx]) relay = _dummyServer[relayIdx];
          }
          assertTrue(null != relay);


          serverChannel = relay.getChildChannel(clientAddr);
          assertTrue(null != serverChannel);
          serverPipeline = serverChannel.getPipeline();
          objCapture = (SimpleObjectCaptureHandler)serverPipeline.get("3");


          log.info("process the /sources request");
          NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
          objCapture.clear();


          log.info("send back the /sources response");
          body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" +
                                      SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
          NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);


          log.info("make sure the client processes the response correctly");
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
              return "1".equals(idListString);
            }
          }, "client processes /sources response", 100, log);


          log.info("process the /register request");
          NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
          objCapture.clear();


          log.info("SEND BACK THE /register RESPONSE");
          //clientConn.getRelayDispatcher().getLog().setLevel(Level.DEBUG);
          //RangeBasedReaderWriterLock.LOG.setLevel(Level.DEBUG);
          body = new DefaultHttpChunk(
              ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
          NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);


          log.info("make sure the client processes the response correctly");
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
              return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
            }
          }, "client processes /register response", timeoutMult * 100, log);


          log.info("PROCESS the /stream CALL AND RETURN A PARTIAL WINDOW");
          streamMatcher =
              NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*checkPoint=([^&]*)&.*",
                                                1000);
          cpString = streamMatcher.group(1);
          objCapture.clear();


          respStartOfs = eventOfs.get(2).get(1);
          respEndOfs = eventOfs.get(2).get(84);


          log.debug("Checkpoint String is :" + cpString);


          cp = new Checkpoint(cpString);
          log.info("last window read was partial. So the client would have reset the windowOffset");
          assertTrue("Is WindowOffset Cleared",cp.getWindowOffset() == -1);
          assertEquals( "WindowSCN == PrevSCN. Ckpt :" + cp, cp.getWindowScn(), cp.getPrevScn());
          streamRes = NettyTestUtils.streamToChannelBuffer(relayBuffer[2], cp,
                                                           respEndOfs - respStartOfs,
                                                           stats);
          NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp,
                                             new DefaultHttpChunk(streamRes));


          log.debug("NumEvents already seen :" + numEvents);


          log.info("make sure the client processes the response correctly");
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              log.debug("lastWrittenScn=" + clientConn.getDataEventsBuffer().lastWrittenScn() + ", NumEvents :" + stats.getTotalStats().getNumDataEvents() );
              return clientConn.getDataEventsBuffer().lastWrittenScn() == 90;
            }
          }, "client receives /stream response, Sequences :" + consumer.getSequences(),
             timeoutMult * 1100, log);


          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              log.debug("events num=" + consumer.getEventNum());
              return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
            }
          }, "client processes /stream response", timeoutMult * 1100, log);


          log.info("one more onStartDataEventSequence because of the rollback");
          assertEquals(30, consumer.getRollbackScn());
          ++rollbackNum;
          assertEquals(stats.getTotalStats().getNumSysEvents() + 1 + rollbackNum, consumer.getWinNum());


          expKeys.addAll(eventKeys.get(2).subList(0, (int)(stats.getTotalStats().getNumDataEvents() - numEvents)));
          for (int i = 0; i < stats.getTotalStats().getNumDataEvents() - numEvents; i++)
            expSeqs.add((i/30)*30 + 30L);


          assertEquals("Keys", expKeys, consumer.getKeys());
          assertEquals("Sequences", expSeqs, consumer.getSequences());


          numEvents = stats.getTotalStats().getNumDataEvents();


          assertEquals(clientConn.getRelayPullThread().getConnectionState().getDataEventsBuffer().isSCNRegress(), false);
      }
      finally
      {

View Full Code Here

Examples of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector


    StreamEventsResult result = new StreamEventsResult(0, 0);
    boolean isDebugEnabled = _log.isDebugEnabled();
    boolean oneWindowAtATime = args.getSMode() == StreamingMode.WINDOW_AT_TIME; // window at a time
    int batchFetchSize = args.getBatchFetchSize();
    DbusEventsStatisticsCollector statsCollector = args.getDbusEventsStatisticsCollector();
    int maxClientEventVersion = args.getMaxClientEventVersion();


    //int sleepTimeMs = RngUtils.randomPositiveInt()%3;


    if (isDebugEnabled)
    {
      _log.debug("Stream:begin:" + checkPoint.toString());
    }


    if (empty())
    {
      if (isDebugEnabled)
        _log.debug("Nothing to send out. Buffer is empty");
      return result;
    }


    long offset;
    // TODO (DDSDBUS-58): Put in assertions about checkPoint.getClientMode()
    long sinceScn = checkPoint.getWindowScn();
    long messagesToSkip = checkPoint.getWindowOffset();
    boolean skipWindowScn = messagesToSkip < 0;


    InternalEventIterator eventIterator = null;
    try {


      ScnIndex.ScnIndexEntry entry = null;
      if ( args.isStreamFromLatestScn() )
      {
        // If streamFromLatestScn is set, then the checkpoint is not respected in the current implementation,
        // but streaming starts from the last window available in the buffer. Hence, init() is invoked on
        // checkpoint
        sinceScn = _lastWrittenSequence;
        messagesToSkip = 0;
        skipWindowScn = false;
        checkPoint.init(); // Will no longer be flexible
      }


      if (checkPoint.getFlexible())
      {
        long minScn = 0;
        _queueLock.lock();
        try
        {
          eventIterator =
              acquireInternalIterator(_head.getPosition(),
                                      _bufferPositionParser.sanitize(_tail.getPosition(), _buffers),
                                      "streamEventsIterator");
          minScn = getMinScn();


          if (isDebugEnabled)
          {
            _log.debug("Acquired read iterator from " + _head.toString() + " to " +
                       _bufferPositionParser.toString(_bufferPositionParser.sanitize(_tail.getPosition(), _buffers), _buffers)
                       + " minScn = " + minScn);
          }
        } finally {
          _queueLock.unlock();
        }


        if (minScn < 0)
        {
          if (isDebugEnabled)
          {
            _log.debug("Nothing to send out. Buffer is empty");
          }
          return result;
        }


        sinceScn = minScn;
        messagesToSkip = 0;
        skipWindowScn = false;


      }
      else
      {
        long minScn = getMinScn();
        long prevScn = getPrevScn();
        if ((sinceScn == _lastWrittenSequence && skipWindowScn) || (minScn < 0))
        {
          //DDS-737 : guards against the situation where first window is not completely written but buffer is deemed not empty;
          if (minScn < 0) {
            if (sinceScn >= prevScn) {
              _log.error("Buffer still not fully ready; please wait for new events: sinceScn=" + sinceScn + " Anticipating events from scn=" + prevScn);
            } else {
              _log.error("Buffer still not fully ready; but request will be obsolete sinceScn=" + sinceScn + " Anticipating events from scn=" + prevScn);
              throw new ScnNotFoundException();
            }
          } else {
            _log.debug("No new events for SCN:" + sinceScn);
          }
          return result;
        }


        if (sinceScn < minScn)
        {
          //DDS-699
          prevScn = getPrevScn();
          if ( (sinceScn > prevScn) || ((sinceScn == prevScn) && skipWindowScn==true)) {
            //can serve data from next Scn - whis is minScn
            checkPoint.setWindowScn(minScn);
            checkPoint.setWindowOffset(0);
            checkPoint.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
            try
            {
              return streamEvents(checkPoint, writeChannel, args);
            }
            catch (ScnNotFoundException e1)
            {
              throw e1;
            }
          } else {
            //either ; sinceScn < prevScn or sinceScn=prevScn with offset >= 0;
            _log.error("sinceScn is less than minScn and prevScn : sinceScn=" + sinceScn + " minScn=" + minScn + " PrevScn= " + prevScn);
            throw new ScnNotFoundException();
          }
        }


        long startTimeTs1 = System.nanoTime();
        entry = _scnIndex.getClosestOffset(sinceScn);
        long endTimeTs1 = System.nanoTime();
        if (PERF_LOG.isDebugEnabled())
        {
          PERF_LOG.debug("getClosestOffset(sinceScn) took: sinceScn=" + sinceScn + " " + (endTimeTs1 - startTimeTs1) / _nanoSecsInMSec + "ms");
        }


        offset = entry.getOffset();
        eventIterator = acquireInternalIterator(offset,
                                                _bufferPositionParser.sanitize(_tail.getPosition(),
                                                                               _buffers),
                                                "streamEventsIterator");
        if (isDebugEnabled)
        {
          _log.debug("Stream:offset:" + _bufferPositionParser.toString(offset, _buffers));
          DbusEvent e = _eventFactory.createReadOnlyDbusEventFromBuffer(_buffers[_bufferPositionParser.bufferIndex(offset)],
                                                                        _bufferPositionParser.bufferOffset(offset));
          _log.debug("Stream:Event@Offset:sequence:"+e.sequence());
        }
      }


      EventScanningState state = EventScanningState.LOOKING_FOR_FIRST_VALID_EVENT; // looking for first valid event
      int skippedMessages = 0;
      // Iterate over the buffer to locate batchFetchSize events
      int batchSize = 0;
      boolean foundWindow = false;
      long currentWindowScn = 0;
      boolean isFirstEvent = true;
      long startTimeTs2 = System.nanoTime();
      boolean done = false;
      // If we encounter the situation (SCN_A < checkpoint_windowSCN < SCN_B), then we end up changing the checkpoint
      // windowScn value to SCN_B. In that case, we save SCN_A in prevWindowScn (for logging).
      long prevWindowScn = 0;
      while (!done && eventIterator.hasNext()) {
        // for the first event, we need to validate that we got a "clean" read lock
        // since we intentionally split the getOffset from the getIterator call
        DbusEventInternalWritable e;
        int eventVersion;


        try
        {
          e = eventIterator.next(isFirstEvent);
          eventVersion = e.getVersion();


          // Convert event to the supported version if possible (DDSDBUS-2063).
          // For now we also assume that clientVersion equals to the eventVersion that the client understands
          // if convertToDifferentVersion() cannot convert - it will throw a runtime exception
          if (eventVersion > maxClientEventVersion) {
            e = DbusEventInternalWritable.convertToDifferentVersion(e, (byte)maxClientEventVersion);
          }


          if ( isFirstEvent)
          {
            if ((entry != null) && (entry.getScn() != e.sequence()))
            {
              String msg = "Concurrent Overwritting of Event. Expected sequence :" + entry.getScn()
                  + ", Got event=" + e.toString();
              _log.warn(msg);
              throw new OffsetNotFoundException(msg);
            }
          }
        }
        catch (InvalidEventException e2)
        {
          _log.warn("Found invalid event on getting iterator. This is not unexpected but should be investigated.");
          _log.warn("RangeBasedLocking :" + _rwLockProvider.toString(_bufferPositionParser, true));
          if (null != statsCollector)
            statsCollector.registerEventError(DbusEventInternalReadable.EventScanStatus.ERR);
          throw new DatabusRuntimeException(e2);
        }


        isFirstEvent = false;


        if (state == EventScanningState.LOOKING_FOR_FIRST_VALID_EVENT)
        {
          if (e.sequence() > sinceScn)
          {
            _log.info("ScnIndex state = " + _scnIndex);
            _log.info("Iterator position = " + eventIterator._currentPosition.toString());
            _log.info("Iterator = " + eventIterator);
            _log.info("Found event " + e + " while looking for sinceScn = " + sinceScn);
            /* while (eventIterator.hasNext())
            {
              e = eventIterator.next();
              _log.error("DbusEventBuffer:dump:" + e);
            } */
            throw new ScnNotFoundException();
          }
          else
          {
            state = EventScanningState.IN_LESS_THAN_EQUALS_SCN_ZONE; // <= sinceScn
          }
        }


        if (state == EventScanningState.IN_LESS_THAN_EQUALS_SCN_ZONE)
        {
          if (skipWindowScn)
          {
            if (e.sequence() < sinceScn)
            {
              currentWindowScn = e.sequence();
              continue;
            }
            else
            {
              if (e.sequence() == sinceScn)
              {
                // we are in the == zone
                foundWindow = true;
                continue;
              }
              else
              {
                if (foundWindow)
                {
                  state = EventScanningState.FOUND_WINDOW_ZONE;
                }
                else
                {
                  // we never found the window but reached a greater
                  // window
                  state = EventScanningState.MISSED_WINDOW_ZONE;
                  prevWindowScn = currentWindowScn;
                  currentWindowScn = e.sequence();
                }
              }
            }
          }
          else
          {
            if (e.sequence() < sinceScn)
            {
              currentWindowScn = e.sequence();
              continue;
            }
            else
            {
              if (e.sequence() == sinceScn)
              {
                foundWindow = true;
                state = EventScanningState.FOUND_WINDOW_ZONE;
              }
              else
              {
                // we never found the window but reached a greater
                // window
                state = EventScanningState.MISSED_WINDOW_ZONE;
                prevWindowScn = currentWindowScn;
                currentWindowScn = e.sequence();
              }
            }
          }
        }


        if (state == EventScanningState.FOUND_WINDOW_ZONE)
        {
          if (skippedMessages < messagesToSkip)
          {
            ++skippedMessages;
            continue;
          }
          else
          {
            state = EventScanningState.VALID_ZONE;
          }
        }


        if (state == EventScanningState.VALID_ZONE)
        {
          boolean controlMessage = e.isControlMessage();
          boolean filterAllowed = args.getFilter().allow(e);
          if (controlMessage || filterAllowed)
          {
            if (batchSize + e.size() > batchFetchSize)
            {
              // sending this would violate our contract on upper bound of
              // bytes to send
              if (isDebugEnabled)
                _log.debug("streamEvents returning after streaming " + batchSize + " bytes because " + batchSize
                           + " + " + e.size() + " > " + batchFetchSize);
              result.setSizeOfPendingEvent(e.size());
              break;
            }


            long startTimeTs1 = System.nanoTime();
            int bytesWritten = e.writeTo(writeChannel, args.getEncoding());
            long endTimeTs1 = System.nanoTime();
            if (PERF_LOG.isDebugEnabled())
            {
              PERF_LOG.debug("writeTo(sinceScn=" + sinceScn + ", bytes=" + bytesWritten +
                             ") took: " + ((endTimeTs1 - startTimeTs1) / _nanoSecsInMSec) + "ms");
            }


            if (0 >= bytesWritten)
            {
              done = true;
            }
            else
            {
              if (null != statsCollector)
                statsCollector.registerDataEventFiltered(e);


              checkPoint.onEvent(e);


              if (isDebugEnabled)
              {
                if (e.isEndOfPeriodMarker())
                  _log.debug("Stream:sequence:"+e.sequence());
                else
                  _log.debug("Stream:sequence:"+e.sequence()+":headerCrc:"+e.headerCrc());
              }


              /** When batch writing is implemented this becomes
               * written  = EventWriter.writeTo(e, writeChannel, encoding);
               * if (written) // The write actually happened and didn't get buffered
               * {
               *  eventIterator.remove();
               * }
               */
              batchSize += e.size();
              result.incNumEventsStreamed(1);
              if (isDebugEnabled)
                _log.debug("buf.stream: GOT event scn="+e.sequence() + ";srcid=" + e.getSourceId() +
                           ";eow=" + e.isEndOfPeriodMarker() + ";oneWindatTime=" + oneWindowAtATime);
            }
          }
          else
          {
            if (isDebugEnabled)
              _log.debug("Event was valid according to checkpoint, but was filtered out :" + e);
          }


          // register both filtered and non-filtered events
          if (null != statsCollector)
          {
            statsCollector.registerDataEvent(e);
          }
          // end of the window - don't send more
          if(e.isEndOfPeriodMarker() && oneWindowAtATime)
          {
            _log.info("buf.stream: reached end of a window. scn="+e.sequence());

View Full Code Here

Examples of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector

            String errMsg = "Buffer not found for physicalPartitionKey " + pKey;
            LOG.error(errMsg);
            throw new BufferNotFoundException(errMsg);
          }
          PhysicalPartition pPartition = pKey.getPhysicalPartition();
          DbusEventsStatisticsCollector statsCollector = _statsCollectors == null ? null : _statsCollectors.getStatsCollector(pPartition.toSimpleString());


          Checkpoint cp=null;
          cp = _checkPoints.getCheckpoint(pKey.getPhysicalPartition());// get the corresponding checkpoint
          if(debugEnabled)
            LOG.debug("get Checkpoint by pPartition" + pPartition + ";cp=" + cp);

View Full Code Here

Examples of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector

          throw new RuntimeException(ice);
        }
      }


      _config = config;
      _statsCollector = new DbusEventsStatisticsCollector(1,"dummy",false, false, null);
      _generationStopped = new CountDownLatch(1);
      this._dbusEventBuffer = dbuf;
      this._duration = config.getDuration();
      this._sources = config.getIdNameList();
      this._startScn = config.getStartScn();

View Full Code Here

Examples of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector

        String statsCollectorName = generateSubsStatsName(sourcesStrList);
        int ownerId = getContainerStaticConfig().getId();


        _bootstrapEventsStats.addStatsCollector(
            statsCollectorName,
            new DbusEventsStatisticsCollector(ownerId,
                                              statsCollectorName + ".inbound.bs",
                                              true,
                                              false,
                                              getMbeanServer()));


        _inBoundStatsCollectors.addStatsCollector(
            statsCollectorName,
            new DbusEventsStatisticsCollector(ownerId,
                                              statsCollectorName + ".inbound",
                                              true,
                                              false,
                                              getMbeanServer()));


        _outBoundStatsCollectors.addStatsCollector(
            statsCollectorName,
            new DbusEventsStatisticsCollector(ownerId,
                                              statsCollectorName + ".outbound",
                                              true,
                                              false,
                                              getMbeanServer()));

View Full Code Here

Examples of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector

                             runtimeConfig.getDefaultExecutor().getKeepAliveMs(),
                             TimeUnit.MILLISECONDS,
                             new NamedThreadFactory("worker" + _containerStaticConfig.getId()));


   _containerStatsCollector = _containerStaticConfig.getOrCreateContainerStatsCollector();
    DbusEventsStatisticsCollector inboundEventStatisticsCollector = new AggregatedDbusEventsStatisticsCollector(getContainerStaticConfig().getId(),
                                                        "eventsInbound",
                                                        true,
                                                        true,
                                                        getMbeanServer());


    DbusEventsStatisticsCollector outboundEventStatisticsCollector = new AggregatedDbusEventsStatisticsCollector(getContainerStaticConfig().getId(),
                                                        "eventsOutbound",
                                                        true,
                                                        true,
                                                        getMbeanServer());

View Full Code Here

Examples of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector

   * Initialize Statistics Collectors
   */
  protected void initializeStatsCollectors(String regId, int ownerId, MBeanServer mbeanServer)
  {
    _inboundEventsStatsCollector =
        new DbusEventsStatisticsCollector(ownerId,
                                          regId + STREAM_EVENT_STATS_SUFFIX_NAME,
                                          true,
                                          false,
                                          mbeanServer);
    _bootstrapEventsStatsCollector =
        new DbusEventsStatisticsCollector(ownerId,
                                          regId + BOOTSTRAP_EVENT_STATS_SUFFIX_NAME,
                                          true,
                                          false,
                                          mbeanServer);
    _relayConsumerStats =

View Full Code Here

Examples of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector

     batchReading(srcIds, 3);
  }


  private StatsCollectors<DbusEventsStatisticsCollector> createStats(String[] pNames)
  {
    DbusEventsStatisticsCollector stats = new AggregatedDbusEventsStatisticsCollector(1, "test", true, true,
              null);
    StatsCollectors<DbusEventsStatisticsCollector> statsColl = new StatsCollectors<DbusEventsStatisticsCollector>(stats);


    int count = 1;
    for (String pname : pNames)
    {
      DbusEventsStatisticsCollector s = new DbusEventsStatisticsCollector(1, "test"+count, true, true,
                null);
       statsColl.addStatsCollector(pname, s);
       ++count;
    }
    return statsColl;

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.