CrawlerTask crawlerTask = (CrawlerTask) task;
log.debug("Processing task " + crawlerTask.getUrl());
// Getting domain statistics
DomainStatistics statistics = statisticsService.getDomainStatistics(crawlerTask.getDomain());
if (!checkMaxHttpErrors(statistics)) {
log.warn(crawlerTask.getDomain() + " has exceeded http errors limit");
return;
}
if (!checkPolitenessPeriod(statistics)) {
log.debug("Waiting for politeness period for domain " + statistics.getDomainName());
deferCrawlerTask(crawlerTask);
return;
}
try {