method.addRequestHeader("User-Agent", userAgent);
try {
int code = http.executeMethod(method);
// TODO: Constant 200
if (code != 200) {
throw new RobotsUnavailableException(robotsUrl);
}
} catch (HttpException e) {
e.printStackTrace();
throw new RobotsUnavailableException(robotsUrl);
} catch (UnknownHostException e) {
LOGGER.info("Unknown host for URL " + robotsUrl);
throw new RobotsUnavailableException(robotsUrl);
} catch (ConnectTimeoutException e) {
LOGGER.info("Connection Timeout for URL " + robotsUrl);
throw new RobotsUnavailableException(robotsUrl);
} catch (NoRouteToHostException e) {
LOGGER.info("No route to host for URL " + robotsUrl);
throw new RobotsUnavailableException(robotsUrl);
} catch (ConnectException e) {
LOGGER.info("ConnectException URL " + robotsUrl);
throw new RobotsUnavailableException(robotsUrl);
}
RobotRules rules = new RobotRules();
rules.parse(method.getResponseBodyAsStream());
return rules;
}