ClassPathXmlApplicationContext context =
new ClassPathXmlApplicationContext(
ApplicationConstants.CORE_SPRING_CONFIG_FILE);
//load dispatcher from spring
Dispatcher dispatcher = (Dispatcher) context.getBean("Dispatcher");
//configure an download job filter
DownloadJobFilter filter = new DownloadJobFilter();
filter.setAllowedHostNames(new String[] {".*"});
filter.setMaxRecursionDepth(1);
filter.setSaveToDisk(new String[] {
".*jpg",
".*png",
".*gif"});
//add the filter to the dispatcher
dispatcher.addJobFilter(filter);
//create an job factory
DownloadJobFactory jobFactory = (DownloadJobFactory)
context.getBean("JobFactory");
//create an initial job
UrlDownloadJob job = jobFactory.createDownloadJob();
job.setUrl(new URL("http://itsucks.sourceforge.net/"));
job.setSavePath(new File("/tmp/crawl")); //change this for windows
job.setIgnoreFilter(true);
dispatcher.addJob(job);
mLog.info("Start demo dispatcher");
//start the dispatcher
dispatcher.processJobs();
mLog.info("Demo dispatcher finished");
//dump all found urls
Collection<Job> content =
dispatcher.getJobManager().getJobList().getContent();
for (Job finishedJob : content) {
mLog.info(finishedJob);
}