return stats;
}
public void operateWithFailover(Operation<?> op) throws HectorException {
final StopWatch stopWatch = new Slf4JStopWatch(perf4jLogger);
int retries = Math.min(op.failoverPolicy.numRetries, hostPools.size());
HThriftClient client = null;
boolean success = false;
boolean retryable = false;
Set<CassandraHost> excludeHosts = new HashSet<CassandraHost>();
// TODO start timer for limiting retry time spent
while ( !success ) {
try {
// TODO how to 'timeout' on this op when underlying pool is exhausted
client = getClientFromLBPolicy(excludeHosts);
Cassandra.Client c = client.getCassandra(op.keyspaceName);
// Keyspace can be null for some system_* api calls
if ( !op.credentials.isEmpty() ) {
c.login(new AuthenticationRequest(op.credentials));
}
op.executeAndSetResult(c, client.cassandraHost);
success = true;
stopWatch.stop(op.stopWatchTagName + ".success_");
break;
} catch (Exception ex) {
HectorException he = exceptionsTranslator.translate(ex);
if ( he instanceof HInvalidRequestException || he instanceof HCassandraInternalException ) {
throw he;
} else if ( he instanceof HectorTransportException) {
--retries;
client.close();
markHostAsDown(client);
excludeHosts.add(client.cassandraHost);
retryable = true;
if ( retries > 0 ) {
monitor.incCounter(Counter.RECOVERABLE_TRANSPORT_EXCEPTIONS);
}
} else if (he instanceof HTimedOutException || he instanceof HUnavailableException ) {
// DO NOT drecrement retries, we will be keep retrying on timeouts until it comes back
retryable = true;
monitor.incCounter(Counter.RECOVERABLE_TIMED_OUT_EXCEPTIONS);
client.close();
// TODO timecheck on how long we've been waiting on timeouts here
// suggestion per user moores on hector-users
} else if ( he instanceof PoolExhaustedException ) {
retryable = true;
--retries;
if ( hostPools.size() == 1 ) {
throw he;
}
monitor.incCounter(Counter.POOL_EXHAUSTED);
excludeHosts.add(client.cassandraHost);
}
if ( retries <= 0 || retryable == false) throw he;
log.error("Could not fullfill request on this host {}", client);
log.error("Exception: ", he);
monitor.incCounter(Counter.SKIP_HOST_SUCCESS);
sleepBetweenHostSkips(op.failoverPolicy);
} finally {
if ( !success ) {
monitor.incCounter(op.failCounter);
stopWatch.stop(op.stopWatchTagName + ".fail_");
}
releaseClient(client);
}
}
}