double overrideDropRate = 0.0;
//simulate latency 4000 ms
//1st round we use LOAD_BALANCING strategy. Since we have a high latency we will decrease the number of points
//from 100 to 80 (transmissionRate * points per weight).
TrackerClient resultTC = simulateAndTestOneInterval(timeInterval, clock, qps, clients, adapter, clusterGenerationId,
80, true, 0.0, 4000, false, false);
assertNotNull(resultTC);
//2nd round drop rate should be increased by DegraderLoadBalancerStrategyConfig.DEFAULT_GLOBAL_STEP_UP
overrideDropRate += DegraderLoadBalancerStrategyConfig.DEFAULT_GLOBAL_STEP_UP;
resultTC = simulateAndTestOneInterval(timeInterval, clock, qps, clients, adapter, clusterGenerationId,
80, false,
overrideDropRate, 4000, false, false);
//3rd round. We alternate back to LOAD_BALANCING strategy and we drop the points even more
resultTC = simulateAndTestOneInterval(timeInterval, clock, qps, clients, adapter, clusterGenerationId,
39, true,
overrideDropRate, 4000, false, false);
//4th round. The drop rate should be increased again like 2nd round
overrideDropRate += DegraderLoadBalancerStrategyConfig.DEFAULT_GLOBAL_STEP_UP;
resultTC = simulateAndTestOneInterval(timeInterval, clock, qps, clients, adapter, clusterGenerationId,
39, false,
overrideDropRate, 4000, false, false);
//5th round. Alternate to changing hash ring again.
resultTC = simulateAndTestOneInterval(timeInterval, clock, qps, clients, adapter, clusterGenerationId,
1, true,
overrideDropRate, 4000, false, false);
//6th round. Same as 5th round, we'll increase the drop rate
overrideDropRate += DegraderLoadBalancerStrategyConfig.DEFAULT_GLOBAL_STEP_UP;
resultTC = simulateAndTestOneInterval(timeInterval, clock, qps, clients, adapter, clusterGenerationId,
1, false,
overrideDropRate, 4000, false, false);
//7th round. The # of point in hashring is at the minimum so we can't decrease it further. At this point the client
//is in recovery mode. But since we can't change the hashring anymore, we'll always in CALL_DROPPING mode
//so the next strategy is expected to be LOAD_BALANCING mode.
overrideDropRate += DegraderLoadBalancerStrategyConfig.DEFAULT_GLOBAL_STEP_UP;
resultTC = simulateAndTestOneInterval(timeInterval, clock, qps, clients, adapter, clusterGenerationId,
1, false,
overrideDropRate, 4000, false, false);
//8th round. We'll increase the drop rate to the max.
overrideDropRate += DegraderLoadBalancerStrategyConfig.DEFAULT_GLOBAL_STEP_UP;
resultTC = simulateAndTestOneInterval(timeInterval, clock, qps, clients, adapter, clusterGenerationId,
1, false,
overrideDropRate, 4000, false, false);
//9th round, now we'll simulate as if there still a call even though we drop 100% of all request to get
//tracker client. The assumption is there's some thread that still holds tracker client and we want
//to make sure we can handle the request and we can't degrade the cluster even further.
resultTC = simulateAndTestOneInterval(timeInterval, clock, qps, clients, adapter, clusterGenerationId,
1, false,
overrideDropRate, 4000, false, false);
//10th round, now we'll simulate as if there's no call because we dropped all request
//even though we are in LOAD_BALANCING mode and this tracker client is in recovery mode and there's no call
//so the hashring doesn't change so we go back to reducing the drop rate to 0.8 and that means the next
//strategy is LOAD_BALANCE
overrideDropRate -= DegraderLoadBalancerStrategyConfig.DEFAULT_GLOBAL_STEP_DOWN;
resultTC = simulateAndTestOneInterval(timeInterval, clock, 0.0, clients, adapter, clusterGenerationId,
1, false,
overrideDropRate, 4000, false, false);
//11th round, this time we'll simulate the latency is now 1000 ms (so it's within low and high watermark). Drop rate
//should stay the same and everything else should stay the same
resultTC = simulateAndTestOneInterval(timeInterval, clock, qps, clients, adapter, clusterGenerationId,
1, false,
overrideDropRate, 1000, false, false);
//we'll simulate the client dying one by one until all the clients are gone
int numberOfClients = clients.size();
HashSet<URI> uris = new HashSet<URI>();
HashSet<URI> removedUris = new HashSet<URI>();
for (TrackerClient client : clients)
{
uris.add(client.getUri());
}
LinkedList<TrackerClient> removedClients = new LinkedList<TrackerClient>();
//loadBalancing strategy will always be picked because there is no hash ring changes
boolean isLoadBalancingStrategyTurn = true;
for(int i = numberOfClients; i > 0; i--)
{
TrackerClient removed = clients.remove(0);
uris.remove(removed.getUri());
removedClients.addLast(removed);
removedUris.add(removed.getUri());
clusterGenerationId++;
resultTC = simulateAndTestOneInterval(timeInterval, clock, qps, clients, adapter, clusterGenerationId,
1, isLoadBalancingStrategyTurn, overrideDropRate, 1000, false, false);
if (i == 1)
{
assertNull(resultTC);
}
else
{
//if the tracker client is not dropped by overrideClusterDropRate (which could be true because at this point
//the override drop rate is 0.8)
if (resultTC != null)
{
assertTrue(uris.contains(resultTC.getUri()));
assertFalse(removedUris.contains(resultTC.getUri()));
}
}
}
assertTrue(uris.isEmpty());
assertTrue(clients.isEmpty());
assertEquals(removedUris.size(), numberOfClients);
assertEquals(removedClients.size(), numberOfClients);
//we'll simulate the client start reviving one by one until all clients are back up again
for (int i = numberOfClients; i > 0 ; i--)
{
TrackerClient added = removedClients.remove(0);
//we have to create a new client. The old client has a degraded DegraderImpl. And in production enviroment
//when a new client join a cluster, it should be in good state. This means there should be 100 points
//in the hash ring for this client
TrackerClient newClient = new TrackerClient(added.getUri(),
getDefaultPartitionData(1d),
new TestLoadBalancerClient(added.getUri()), clock, degraderConfig);
clients.add(newClient);
uris.add(added.getUri());
removedUris.remove(added.getUri());