}
}
List<Callable<GetAllResult>> callables = Lists.newArrayList();
for(Map.Entry<Node, List<ByteArray>> entry: nodeToKeysMap.entrySet()) {
final Node node = entry.getKey();
final Collection<ByteArray> nodeKeys = entry.getValue();
if(failureDetector.isAvailable(node))
callables.add(new GetAllCallable(node, nodeKeys, transforms));
}
// A list of thrown exceptions, indicating the number of failures
List<Throwable> failures = Lists.newArrayList();
List<NodeValue<ByteArray, byte[]>> nodeValues = Lists.newArrayList();
Map<ByteArray, MutableInt> keyToSuccessCount = Maps.newHashMap();
for(ByteArray key: keys)
keyToSuccessCount.put(key, new MutableInt(0));
List<Future<GetAllResult>> futures;
long timeoutMs = timeoutConfig.getOperationTimeout(VoldemortOpCode.GET_ALL_OP_CODE);
try {
// TODO What to do about timeouts? They should be longer as getAll
// is likely to
// take longer. At the moment, it's just timeoutMs * 3, but should
// this be based on the number of the keys?
futures = executor.invokeAll(callables, timeoutMs * 3, TimeUnit.MILLISECONDS);
} catch(InterruptedException e) {
throw new InsufficientOperationalNodesException("getAll operation interrupted.", e);
}
for(Future<GetAllResult> f: futures) {
if(f.isCancelled()) {
logger.warn("Get operation timed out after " + timeoutMs + " ms.");
continue;
}
try {
GetAllResult getResult = f.get();
if(getResult.exception != null) {
if(getResult.exception instanceof VoldemortApplicationException) {
throw (VoldemortException) getResult.exception;
}
failures.add(getResult.exception);
continue;
}
for(ByteArray key: getResult.callable.nodeKeys) {
List<Versioned<byte[]>> retrieved = getResult.retrieved.get(key);
MutableInt successCount = keyToSuccessCount.get(key);
successCount.increment();
/*
* retrieved can be null if there are no values for the key
* provided
*/
if(retrieved != null) {
List<Versioned<byte[]>> existing = result.get(key);
if(existing == null)
result.put(key, Lists.newArrayList(retrieved));
else
existing.addAll(retrieved);
}
}
nodeValues.addAll(getResult.nodeValues);
} catch(InterruptedException e) {
throw new InsufficientOperationalNodesException("getAll operation interrupted.", e);
} catch(ExecutionException e) {
// We catch all Throwables apart from Error in the callable, so
// the else part
// should never happen
if(e.getCause() instanceof Error)
throw (Error) e.getCause();
else
logger.error(e.getMessage(), e);
}
}
for(ByteArray key: keys) {
MutableInt successCountWrapper = keyToSuccessCount.get(key);
int successCount = successCountWrapper.intValue();
if(successCount < storeDef.getPreferredReads()) {
List<Node> extraNodes = keyToExtraNodesMap.get(key);
if(extraNodes != null) {
for(Node node: extraNodes) {
long startNs = System.nanoTime();
try {
List<Versioned<byte[]>> values = innerStores.get(node.getId())
.get(key,
transforms == null ? null
: transforms.get(key));
fillRepairReadsValues(nodeValues, key, node, values);
List<Versioned<byte[]>> versioneds = result.get(key);
if(versioneds == null)
result.put(key, Lists.newArrayList(values));
else
versioneds.addAll(values);
recordSuccess(node, startNs);
if(++successCount >= storeDef.getPreferredReads())
break;
} catch(UnreachableStoreException e) {
failures.add(e);
recordException(node, startNs, e);
} catch(VoldemortApplicationException e) {
throw e;
} catch(Exception e) {
logger.warn("Error in GET_ALL on node " + node.getId() + "("
+ node.getHost() + ")",
e);
failures.add(e);
}
}
}