log.info("Killing node 1 ..");
TestingUtil.killCacheManagers(manager(1));
log.info("Node 1 killed");
DataContainer dc0 = advancedCache(0).getDataContainer();
DataContainer dc2 = advancedCache(2).getDataContainer();
// wait for state transfer on nodes A and C to progress to the point where data segments are about to be applied
if (!applyStateStartedLatch1.await(15, TimeUnit.SECONDS)) {
throw new TimeoutException();
}
if (!applyStateStartedLatch2.await(15, TimeUnit.SECONDS)) {
throw new TimeoutException();
}
if (op == Operation.CLEAR) {
log.info("Clearing cache ..");
cache(0).clear();
log.info("Finished clearing cache");
assertEquals(0, dc0.size());
assertEquals(0, dc2.size());
} else if (op == Operation.REMOVE) {
log.info("Removing all keys one by one ..");
for (int i = 0; i < numKeys; i++) {
cache(0).remove(i);
}
log.info("Finished removing keys");
assertEquals(0, dc0.size());
assertEquals(0, dc2.size());
} else if (op == Operation.PUT || op == Operation.PUT_MAP || op == Operation.REPLACE || op == Operation.PUT_IF_ABSENT) {
log.info("Updating all keys ..");
if (op == Operation.PUT) {
for (int i = 0; i < numKeys; i++) {
cache(0).put(i, "after_st_" + i);
}
} else if (op == Operation.PUT_MAP) {
Map<Integer, String> toPut = new HashMap<Integer, String>();
for (int i = 0; i < numKeys; i++) {
toPut.put(i, "after_st_" + i);
}
cache(0).putAll(toPut);
} else if (op == Operation.REPLACE) {
for (int i = 0; i < numKeys; i++) {
String expectedOldValue = "before_st_" + i;
boolean replaced = cache(0).replace(i, expectedOldValue, "after_st_" + i);
assertTrue(replaced);
}
} else { // PUT_IF_ABSENT
for (int i = 0; i < numKeys; i++) {
String expectedOldValue = "before_st_" + i;
Object prevValue = cache(0).putIfAbsent(i, "after_st_" + i);
assertEquals(expectedOldValue, prevValue);
}
}
log.info("Finished updating keys");
}
// allow state transfer to apply state
applyStateProceedLatch.countDown();
// wait for apply state to end
TestingUtil.waitForRehashToComplete(cache(0), cache(2));
// at this point state transfer is fully done
log.infof("Data container of NodeA has %d keys: %s", dc0.size(), dc0.entrySet());
log.infof("Data container of NodeC has %d keys: %s", dc2.size(), dc2.entrySet());
if (op == Operation.CLEAR || op == Operation.REMOVE) {
// caches should be empty. check that no keys were revived by an inconsistent state transfer
for (int i = 0; i < numKeys; i++) {
assertNull(dc0.get(i));
assertNull(dc2.get(i));
}
} else if (op == Operation.PUT || op == Operation.PUT_MAP || op == Operation.REPLACE) {
ConsistentHash ch = advancedCache(0).getComponentRegistry().getStateTransferManager().getCacheTopology().getReadConsistentHash();
// check that all values are the ones expected after state transfer
for (int i = 0; i < numKeys; i++) {
// check number of owners
int owners = 0;
if (dc0.get(i) != null) {
owners++;
}
if (dc2.get(i) != null) {
owners++;
}
assertEquals("Wrong number of owners", ch.locateOwners(i).size(), owners);
// check values were not overwritten with old values carried by state transfer
String expected = "after_st_" + i;
assertEquals(expected, cache(0).get(i));
assertEquals("after_st_" + i, cache(2).get(i));
}
} else { // PUT_IF_ABSENT
ConsistentHash ch = advancedCache(0).getComponentRegistry().getStateTransferManager().getCacheTopology().getReadConsistentHash();
for (int i = 0; i < numKeys; i++) {
// check number of owners
int owners = 0;
if (dc0.get(i) != null) {
owners++;
}
if (dc2.get(i) != null) {
owners++;
}
assertEquals("Wrong number of owners", ch.locateOwners(i).size(), owners);
String expected = "before_st_" + i;