}
long consumerTimeBudgetMs = 60*1000;
DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(tConsumer, sources, null);
List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
final UnifiedClientStats unifiedStats = new UnifiedClientStats(0, "test", "test.unified");
// single-threaded execution of consumer
MultiConsumerCallback mConsumer =
new MultiConsumerCallback(allRegistrations,
Executors.newFixedThreadPool(1),
consumerTimeBudgetMs,
new StreamConsumerCallbackFactory(null, unifiedStats),
null,
unifiedStats,
null,
null);
/* Generate events */
Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
Vector<Short> srcIdList = new Vector<Short> ();
srcIdList.add(srcId);
DbusEventGenerator evGen = new DbusEventGenerator(0,srcIdList);
Assert.assertTrue(evGen.generateEvents(numEvents, maxWindowSize, 512, payloadSize, srcTestEvents) > 0);
int totalSize=0;
int maxSize=0;
for (DbusEvent e : srcTestEvents)
{
totalSize += e.size();
maxSize = (e.size() > maxSize) ? e.size():maxSize;
}
/* Source configuration */
double thresholdChkptPct = thresholdPct;
DatabusSourcesConnection.Config conf = new DatabusSourcesConnection.Config();
conf.setCheckpointThresholdPct(thresholdChkptPct);
conf.getDispatcherRetries().setMaxRetryNum(10);
conf.setFreeBufferThreshold(maxSize);
conf.setConsumerTimeBudgetMs(consumerTimeBudgetMs);
int freeBufferThreshold = conf.getFreeBufferThreshold();
DatabusSourcesConnection.StaticConfig connConfig = conf.build();
// make buffer large enough to hold data; the control events are large that contain checkpoints
int producerBufferSize = wrapAround ? totalSize : totalSize*2 + numCheckpoints*10*maxSize*5 + freeBufferThreshold;
int individualBufferSize = producerBufferSize;
int indexSize = producerBufferSize / 10;
int stagingBufferSize = producerBufferSize;
/* Event Buffer creation */
TestGenericDispatcherEventBuffer dataEventsBuffer=
new TestGenericDispatcherEventBuffer(
getConfig(producerBufferSize, individualBufferSize, indexSize ,
stagingBufferSize, AllocationPolicy.HEAP_MEMORY,
QueuePolicy.BLOCK_ON_WRITE));
List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
/* Generic Dispatcher creation */
TestDispatcher<DatabusCombinedConsumer> dispatcher =
new TestDispatcher<DatabusCombinedConsumer>("rollBackcheck",
connConfig,
subs,
new InMemoryPersistenceProvider(),
dataEventsBuffer,
mConsumer,
bootstrapCheckpointsPerWindow == 0);
/* Launch writer */
DbusEventAppender eventProducer =
new DbusEventAppender(srcTestEvents, dataEventsBuffer,bootstrapCheckpointsPerWindow ,null);
Thread tEmitter = new Thread(eventProducer);
tEmitter.start();
/* Launch dispatcher */
Thread tDispatcher = new Thread(dispatcher);
tDispatcher.start();
/* Now initialize this state machine */
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
// be generous; use worst case for num control events
long waitTimeMs = (numEvents*timeTakenForDataEventInMs + numEvents*timeTakenForControlEventInMs) * 4;
tEmitter.join(waitTimeMs);
// wait for dispatcher to finish reading the events
tDispatcher.join(waitTimeMs);
Assert.assertFalse(tEmitter.isAlive());
System.out.println("tConsumer: " + tConsumer);
int windowBeforeDataFail=(numFailDataEvent/maxWindowSize);
int expectedDataFaults = numFailDataEvent == 0 ? 0:numFailures;
int expectedCheckPointFaults =
(numFailCheckpointEvent==0 || (expectedDataFaults!=0 && numFailCheckpointEvent==windowBeforeDataFail)) ?
0 : numFailures;
// Dispatcher/Library perspective
// check if all windows were logged by dispatcher; in online case;
if (bootstrapCheckpointsPerWindow == 0)
{
Assert.assertTrue(dispatcher.getNumCheckPoints() >= (numCheckpoints-expectedCheckPointFaults));
}
// Consumer prespective
// 1 or 0 faults injected in data callbacks; success (store) differs callback by 1
Assert.assertEquals("Mismatch between callbacks and stored data on consumer.",
expectedDataFaults, tConsumer.getDataCallbackCount()-tConsumer.getStoredDataCount());
Assert.assertTrue(tConsumer.getStoredDataCount() >= tConsumer.getNumUniqStoredEvents());
Assert.assertEquals("Consumer failed to store expected number of checkpoints.",
dispatcher.getNumCheckPoints(), tConsumer.getStoredCheckpointCount());
// Equality would be nice, but each "real" error (as seen by MultiConsumerCallback) triggers a non-
// deterministic number of cancelled callbacks, each of which is treated as another consumer error
// (as seen by StreamConsumerCallbackFactory or BootstrapConsumerCallbackFactory). So an inequality
// is the best we can do at the moment, barring a change in how numConsumerErrors is accounted.
// Special case: non-zero numFailCheckpointEvent doesn't actually trigger a callback error; instead
// it's converted to ConsumerCallbackResult.SKIP_CHECKPOINT and therefore not seen by client metrics.
if (expectedCheckPointFaults == 0 || expectedDataFaults > 0 || negativeTest)
{
Assert.assertTrue("Unexpected error count in consumer metrics (" + unifiedStats.getNumConsumerErrors() +
"); should be greater than or equal to numFailures (" + numFailures + ").",
unifiedStats.getNumConsumerErrors() >= numFailures);
}
else
{
Assert.assertEquals("Unexpected error count in consumer metrics; checkpoint errors shouldn't count. ",
// unless negativeTest ...
0, unifiedStats.getNumConsumerErrors());
}
// rollback behaviour; were all events re-sent?
if (!negativeTest)
{