configuration.setMaxHttpErrors(HttpURLConnection.HTTP_BAD_GATEWAY, 10);
// Setting period between two requests to a single site (in milliseconds)
configuration.setPolitenessPeriod(500);
// Initializing crawler controller
CrawlerController crawlerController = new CrawlerController(configuration);
// Adding crawler seed
crawlerController.addSeed(new URL("http://en.wikipedia.org/"));
// Starting and joining our crawler
crawlerController.start();
System.out.println(new Date() + "Waiting for 60 seconds");
crawlerController.join(60000);
System.out.println(new Date() + "Stopping crawler");
crawlerController.stop();
}