cache1.getRpcManager().getMembers().size() == 3 &&
cache2.getRpcManager().getMembers().size() == 3;
}
});
CacheTopology duringJoinTopology = ltm0.getCacheTopology(CACHE_NAME);
assertEquals(duringJoinTopologyId, duringJoinTopology.getTopologyId());
assertNotNull(duringJoinTopology.getPendingCH());
final MagicKey key = getKeyForCache2(duringJoinTopology.getPendingCH());
log.tracef("Rebalance started. Found key %s with current owners %s and pending owners %s", key,
duringJoinTopology.getCurrentCH().locateOwners(key), duringJoinTopology.getPendingCH().locateOwners(key));
// Every PutKeyValueCommand will be blocked before reaching the distribution interceptor on cache1
CyclicBarrier beforeCache1Barrier = new CyclicBarrier(2);
BlockingInterceptor blockingInterceptor1 = new BlockingInterceptor(beforeCache1Barrier,
PutKeyValueCommand.class, false);
cache1.addInterceptorBefore(blockingInterceptor1, NonTxConcurrentDistributionInterceptor.class);
// Every PutKeyValueCommand will be blocked after returning to the distribution interceptor on cache2
CyclicBarrier afterCache2Barrier = new CyclicBarrier(2);
BlockingInterceptor blockingInterceptor2 = new BlockingInterceptor(afterCache2Barrier,
PutKeyValueCommand.class, true);
cache2.addInterceptorBefore(blockingInterceptor2, StateTransferInterceptor.class);
// Put from cache0 with cache0 as primary owner, cache2 will become the primary owner for the retry
Future<Object> future = fork(new Callable<Object>() {
@Override
public Object call() throws Exception {
return conditional ? cache0.putIfAbsent(key, "v") : cache0.put(key, "v");
}
});
// Wait for the command to be executed on cache2 and unblock it
afterCache2Barrier.await(10, TimeUnit.SECONDS);
afterCache2Barrier.await(10, TimeUnit.SECONDS);
// Allow the topology update to proceed on all the caches
int postJoinTopologyId = duringJoinTopologyId + 1;
checkPoint.trigger("allow_topology_" + postJoinTopologyId + "_on_" + address(0));
checkPoint.trigger("allow_topology_" + postJoinTopologyId + "_on_" + address(1));
checkPoint.trigger("allow_topology_" + postJoinTopologyId + "_on_" + address(2));
// Wait for the topology to change everywhere
TestingUtil.waitForRehashToComplete(cache0, cache1, cache2);
// Allow the put command to throw an OutdatedTopologyException on cache1
log.tracef("Unblocking the put command on node " + address(1));
beforeCache1Barrier.await(10, TimeUnit.SECONDS);
beforeCache1Barrier.await(10, TimeUnit.SECONDS);
// Allow the retry to proceed on cache1, if it's still a member.
// (In my tests, the backup was always cache0.)
CacheTopology postJoinTopology = ltm0.getCacheTopology(CACHE_NAME);
if (postJoinTopology.getCurrentCH().locateOwners(key).contains(address(1))) {
beforeCache1Barrier.await(10, TimeUnit.SECONDS);
beforeCache1Barrier.await(10, TimeUnit.SECONDS);
}
// And allow the retry to finish successfully on cache2
afterCache2Barrier.await(10, TimeUnit.SECONDS);