loadSystemConfiguration();
/*
* 1st : download main page and get its document objects which will be use for fetching its links and scripts later
* */
UrlFileDownloader downloadMgr = new UrlFileDownloader(homepageUrl, downloadDirectory);
Document targetUrlDocument= downloadMgr.getUrlDocument(homepageUrl);
DownloadContent mainPageDownloadContent = downloadMgr.getMainUrlDownloadContent(homepageUrl);
List<DownloadContent> mainDownloadLst = new ArrayList<DownloadContent>();
mainDownloadLst.add(mainPageDownloadContent);
// download a main page
logger.info("download main page time : " + mainPageDownloadContent.getTime());
// response status
logger.info("response status code : " + mainPageDownloadContent.getStatus());
/*
* 2nd : download main page's links and scripts.
* */
List<DownloadContent> completeDownloadLst = downloadMgr.getUrlDownloadContents(targetUrlDocument);
completeDownloadLst = UrlSorting.sort(completeDownloadLst);
logger.info("######### Result ( "+ completeDownloadLst.size()+ " )##############");