Examples of com.linkedin.databus.core.DbusEventBuffer

com.linkedin.databus.core.DbusEventBuffer

    boolean enqueueMessage = true;


    try
    {
      Checkpoint cp = curState.getCheckpoint();
      DbusEventBuffer eventBuffer = curState.getDataEventsBuffer();


      if (debugEnabled) _log.debug("Sending bootstrap events to buffer");


      //eventBuffer.startEvents();
      DbusEventInternalReadable cpEvent = getEventFactory().createCheckpointEvent(cp);
      byte[] cpEventBytes = new byte[cpEvent.size()];


      if (debugEnabled)
      {
        _log.debug("checkpoint event size: " + cpEventBytes.length);
        _log.debug("checkpoint event:" + cpEvent.toString());
      }


      cpEvent.getRawBytes().get(cpEventBytes);
      ByteArrayInputStream cpIs = new ByteArrayInputStream(cpEventBytes);
      ReadableByteChannel cpRbc = Channels.newChannel(cpIs);


      UnifiedClientStats unifiedClientStats = _sourcesConn.getUnifiedClientStats();
      sendHeartbeat(unifiedClientStats);
      int ecnt = eventBuffer.readEvents(cpRbc);


      success = (ecnt > 0);


      if (!success)
      {
        _log.error("Unable to write bootstrap phase marker");
      } else {
        ChunkedBodyReadableByteChannel readChannel = curState.getReadChannel();


        String remoteErrorName = RemoteExceptionHandler.getExceptionName(readChannel);
        Throwable remoteError = _remoteExceptionHandler.getException(readChannel);
        if (null != remoteError &&
            remoteError instanceof BootstrapDatabaseTooOldException)
        {
          _log.error("Bootstrap database is too old!");
          _remoteExceptionHandler.handleException(remoteError);
          curState.switchToStreamResponseError();
        }
        else if (null != remoteErrorName)
        {
          //remote processing error
          _log.error("read events error: " + RemoteExceptionHandler.getExceptionMessage(readChannel));
          curState.switchToStreamResponseError();
        }
        else
        {
          sendHeartbeat(unifiedClientStats);
          int eventsNum = eventBuffer.readEvents(readChannel, curState.getListeners(),
                                                 _sourcesConn.getBootstrapEventsStatsCollector());


          if (eventsNum == 0 &&
              _remoteExceptionHandler.getPendingEventSize(readChannel) > eventBuffer.getMaxReadBufferCapacity())
          {
            String err = "ReadBuffer max capacity(" + eventBuffer.getMaxReadBufferCapacity() +
                         ") is less than event size(" +
                         _remoteExceptionHandler.getPendingEventSize(readChannel) +
                         "). Increase databus.client.connectionDefaults.bstEventBuffer.maxEventSize and restart.";
            _log.fatal(err);
            enqueueMessage(LifecycleMessage.createSuspendOnErroMessage(new PendingEventTooLargeException(err)));

View Full Code Here


    cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);


    cp.resetBootstrap(); // clear Bootstrap scns for future bootstraps


    DbusEventBuffer eventBuffer = curState.getDataEventsBuffer();


    try
    {
      DbusEventInternalReadable cpEvent = getEventFactory().createCheckpointEvent(cp);
      boolean success = eventBuffer.injectEvent(cpEvent);
      if (!success)
      {
        _log.error("Unable to write bootstrap phase marker");
      }
      else
      {
        //TODO need real partition for partitioned bootstrap
        DbusEventInternalReadable eopEvent = curState.createEopEvent(cp, getEventFactory());
        success = eventBuffer.injectEvent(eopEvent);
        if (! success)
        {
          _log.error("Unable to write bootstrap EOP marker");
        }
      }

View Full Code Here

  {
    // No rate control
    long rate = 0;
    PhysicalSourceStaticConfig pssc = buildPssc(rate, 0L);
    long scn = 10;
    DbusEventBuffer mb = (DbusEventBuffer)createBufMult(pssc);


    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb,
        null, null);
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates = new ArrayList<TransactionState.PerSourceTransactionalUpdate>(
        10);
    int sourceId = 505;
    HashSet<DBUpdateImage> db = new HashSet<DBUpdateImage>();


    Object key = new String("name");
    Schema.Type keyType = Schema.Type.RECORD;
    ColumnsState.KeyPair kp = new ColumnsState.KeyPair(key, keyType);
    ArrayList<ColumnsState.KeyPair> keyPairs = new ArrayList<ColumnsState.KeyPair>(
        1);
    keyPairs.add(kp);


    Schema s = Schema.parse(avroSchema);
    GenericRecord gr = new GenericData.Record(s);
    gr.put("name", "phani");


    DBUpdateImage dbi = new DBUpdateImage(keyPairs, scn, gr, s,
        DbUpdateState.DBUpdateImage.OpType.INSERT,false);
    db.add(dbi);
    TransactionState.PerSourceTransactionalUpdate dbUpdate = new TransactionState.PerSourceTransactionalUpdate(
        sourceId, db);
    dbUpdates.add(dbUpdate);


    long timestamp = System.nanoTime();
    gg.addEventToBuffer(dbUpdates, new TransactionInfo(0, 0, timestamp, scn));
    Assert.assertEquals(gg.getRateControl().getNumSleeps(), 0);
    DbusEventIterator iter  = mb.acquireIterator("test");
    int count = 0;
    long eventTs = 0;
    while(iter.hasNext()) {
      DbusEvent e = iter.next();
      if(count==1) { // first event prev control event

View Full Code Here

      r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
      log.info("Relay created");


      DbusEventBufferMult bufMult = relay1.getEventBuffer();
      PhysicalPartition pPartition = new PhysicalPartition((int)pId, pSourceName);
      DbusEventBuffer buf = (DbusEventBuffer)bufMult.getDbusEventBufferAppendable(pPartition);


      log.info("create some events");
      long windowScn = 100L;
      ByteBuffer serializationBuffer = addEvent(windowScn, srcId, relay1.getSchemaRegistryService().fetchSchemaIdForSourceNameAndVersion(srcs[0], 2).getByteArray(),
          pId, DbusEventFactory.DBUS_EVENT_V2);
      ReadableByteChannel channel = Channels.newChannel(new ByteBufferInputStream(serializationBuffer));
      int readEvents = buf.readEvents(channel);
      log.info("successfully read in " + readEvents + " events ");
      channel.close();


      windowScn = 101L;
      serializationBuffer = addEvent(windowScn, srcId, relay1.getSchemaRegistryService().fetchSchemaIdForSourceNameAndVersion(srcs[0], 2).getByteArray(),
          pId, DbusEventFactory.DBUS_EVENT_V1);
      channel = Channels.newChannel(new ByteBufferInputStream(serializationBuffer));
      readEvents = buf.readEvents(channel);
      log.info("successfully read in " + readEvents + " events ");
      channel.close();


      log.info("starting relay on port " + relayPort);
      r1.start();

View Full Code Here

     */
    public DbusEventBuffer addNewBuffer(PhysicalSourceStaticConfig pConfig, HttpRelay.StaticConfig config)
        throws DatabusException
    {
      DbusEventBufferMult eventMult = getEventBuffer();
      DbusEventBuffer buf = eventMult.addNewBuffer(pConfig, config.getEventBuffer());
      return buf;
    }

View Full Code Here

  }


  private void processInboundRequest(DatabusRequest request, PhysicalPartition pPart)
    throws IOException, DatabusException
  {        
    DbusEventBuffer evb = _eventBufferMult.getOneBuffer(pPart);
        
    if ( null == evb)
    {
      LOG.error("BufferInfoRequest : Buffer not available for physical partition :" + pPart);
        throw new BufferNotFoundException("Buffer not available for partition :" + pPart);
    }
    
    BufferInfoResponse response = new BufferInfoResponse();
    response.setMinScn(evb.getMinScn());
    response.setMaxScn(evb.lastWrittenScn());
    response.setTimestampFirstEvent(evb.getTimestampOfFirstEvent());
    response.setTimestampLatestEvent(evb.getTimestampOfLatestDataEvent());
    
      writeJsonObjectToResponse(response, request);
  }

View Full Code Here


    try
    {
      //PhysicalPartition pPartition = new PhysicalPartition(physicalPartitionId);
      // TODO this should actually use DbusEventBufferAppendable (DDSDBUS-78)
      DbusEventBuffer buf = (DbusEventBuffer)_eventBuffer.getDbusEventBufferAppendable(pPartition);
      if(buf == null)
        throw new RequestProcessingException("cannot find buffer for ph. partion " + pPartition);


      if ((buf.getMinScn() < 0) && (buf.getPrevScn() < 0)) buf.start(0);


      try
      {
        DbusEventsStatisticsCollector statsCollector = _relay.getInBoundStatsCollectors().getStatsCollector(pPartition.toSimpleString());

View Full Code Here


        Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
        Hashtable<Short, AtomicInteger> srcidCounts = new Hashtable<Short, AtomicInteger>();


    // Send dummy e
        DbusEventBuffer eventsBuf = dsc.getDataEventsBuffer();
        eventsBuf.start(0);
        eventsBuf.startEvents();
        initBufferWithEvents(eventsBuf, 1, source1EventsNum, (short)1, keyCounts, srcidCounts);
        initBufferWithEvents(eventsBuf, 1 + source1EventsNum, source2EventsNum, (short)2, keyCounts, srcidCounts);
        eventsBuf.endEvents(100L,null);


        rd.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
        rd.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));


        TestUtil.assertWithBackoff(new ConditionCheck()

View Full Code Here

        //log.setLevel(Level.DEBUG);
        final int eventsNum = 20;
        DbusEventInfo[] eventInfos = createSampleSchema1Events(eventsNum);


        //simulate relay buffers
        DbusEventBuffer relayBuffer = new DbusEventBuffer(_bufCfg);
        relayBuffer.start(0);
        writeEventsToBuffer(relayBuffer, eventInfos, 4);


        //prepare stream response
        Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
        final DbusEventsStatisticsCollector stats =

View Full Code Here

        //log.setLevel(Level.DEBUG);
        final int eventsNum = 20;
        DbusEventInfo[] eventInfos = createSampleSchema1Events(eventsNum);


        //simulate relay buffers
        DbusEventBuffer relayBuffer = new DbusEventBuffer(_bufCfg);
        relayBuffer.start(0);
        writeEventsToBuffer(relayBuffer, eventInfos, 4);


        //prepare stream response
        Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
        final DbusEventsStatisticsCollector stats =

View Full Code Here

0 1 2 3 4

TOP

Related Classes of com.linkedin.databus.core.DbusEventBuffer

com.linkedin.databus.client.BootstrapPullThread

com.linkedin.databus.client.consumer.TestMultiConsumerCallback

com.linkedin.databus.client.DatabusHttpClientImpl

com.linkedin.databus.client.DispatcherState

com.linkedin.databus.client.netty.TestNettyHttpDatabusRelayConnection

com.linkedin.databus.client.registration.DatabusV2RegistrationImpl

com.linkedin.databus.client.TestBootstrapPullThread

com.linkedin.databus.client.TestDatabusHttpClient

com.linkedin.databus.client.TestRelayPullThread

com.linkedin.databus.container.netty.HttpRelay

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.