System.out.println(new StringBuilder().append("MOVES: guestId=").append(updateInfo.getGuestId()).append(", waitTime=").append(waitTime).append(", RESCHEDULING").toString());
// Set the reset time info in updateInfo so that we get scheduled for when the quota becomes available
// + a random number of minutes in the range of 0 to 60 to spread the load of lots of competing
// updaters across the next hour
updateInfo.setResetTime("moves", getRandomRescheduleTime());
throw new RateLimitReachedException();
}
// We are willing to wait that long
System.out.println(new StringBuilder().append("MOVES: guestId=").append(updateInfo.getGuestId()).append(", waitTime=").append(waitTime).append(", WAITING").toString());
try { Thread.currentThread().sleep(waitTime); }
catch(Throwable e) {
e.printStackTrace();
throw new RuntimeException("Unexpected error waiting to enforce rate limits.");
}
waitTime = getQuotaWaitTime();
} while (waitTime>0);
}
// By the time we get to here, we should likely have quota available
try {
HttpGet get = new HttpGet(url);
HttpResponse response = client.execute(get);
// Get the millisecond time of the next available bit of quota. These fields will be populated for
// status 200 or status 429 (over quota). Other responses may not have them, in which case
// responseToQuotaAvailableTime will return -1. Ignore that case.
long nextQuotaAvailableTime = responseToQuotaAvailableTime(updateInfo, response);
// Update the quotaAvailableTime and check if we're the first to learn that we just blew quota
boolean firstToUpdate = tryUpdateQuotaAvailableTime(nextQuotaAvailableTime);
long now = System.currentTimeMillis();
if(firstToUpdate && nextQuotaAvailableTime>now) {
// We're the first to find out that quota is gone. We may or may not have succeeded on this call,
// depending on the status code. Regardless of the status code, fix the scheduling of moves updates
// that would otherwise happen before the next quota window opens up.
List<UpdateWorkerTask> updateWorkerTasks = connectorUpdateService.getScheduledUpdateWorkerTasksForConnectorNameBeforeTime("moves", nextQuotaAvailableTime);
for (int i=0; i<updateWorkerTasks.size(); i++) {
UpdateWorkerTask updateWorkerTask = updateWorkerTasks.get(i);
// Space the tasks 30 seconds apart so they don't all try to start at the same time
long rescheduleTime = nextQuotaAvailableTime + i*(DateTimeConstants.MILLIS_PER_SECOND*30);
// Update the scheduled execution time for any moves tasks that would otherwise happen during
// the current quota outage far enough into the future that we should have quota available by then.
// If there's more than one pending, stagger them by a few minutes so that they don't all try to
// happen at once. The reason the "incrementRetries" arg is true is that that appears to be the
// way to prevent spawning a duplicate entry in the UpdateWorkerTask table.
connectorUpdateService.reScheduleUpdateTask(updateWorkerTask.getId(),
rescheduleTime,
true,null);
logger.info("module=movesUpdater component=fetchMovesAPI action=fetchMovesAPI" +
" message=\"Rescheduling due to quota limit: " +
updateWorkerTask + "\" newUpdateTime=" + rescheduleTime);
}
}
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode == HttpStatus.SC_OK) {
ResponseHandler<String> responseHandler = new BasicResponseHandler();
content = responseHandler.handleResponse(response);
}
else {
// attempt to get a human-readable message
String message = null;
try {
message = IOUtils.toString(response.getEntity().getContent());
} catch (Throwable t) {}
if(statusCode == 401) {
// Unauthorized, so this is never going to work
// Notify the user that the tokens need to be manually renewed
notificationsService.addNamedNotification(updateInfo.getGuestId(), Notification.Type.WARNING, connector().statusNotificationName(),
"Heads Up. We failed in our attempt to update your Moves connector.<br>" +
"Please head to <a href=\"javascript:App.manageConnectors()\">Manage Connectors</a>,<br>" +
"scroll to the Moves connector, and renew your tokens (look for the <i class=\"icon-resize-small icon-large\"></i> icon)");
// Record permanent failure since this connector won't work again until
// it is reauthenticated
guestService.setApiKeyStatus(updateInfo.apiKey.getId(), ApiKey.Status.STATUS_PERMANENT_FAILURE, null, ApiKey.PermanentFailReason.NEEDS_REAUTH);
throw new UpdateFailedException("Unauthorized access", true, ApiKey.PermanentFailReason.NEEDS_REAUTH);
}
else if(statusCode == 429) {
// Over quota, so this API attempt didn't work
// Set the reset time info in updateInfo so that we get scheduled for when the quota becomes available
updateInfo.setResetTime("moves", getQuotaAvailableTime());
throw new RateLimitReachedException();
}
else if (statusCode>=400 && statusCode<500) {
String message40x = "Unexpected response code: " + statusCode;
if (message!=null) message40x += " message: " + message;
throw new UpdateFailedException(message40x, new Exception(), true, ApiKey.PermanentFailReason.clientError(statusCode, message));