CrawlRecord record = crawlRecords.findByID(crawlId);
MDC.put("crawl_record", Integer.toString(crawlId));
File resourceDir = new File(record.getOutputFolder() + File.separatorChar + "resources");
resourceDir.mkdirs();
try {
Configuration config = configurations.findByID(record.getConfigurationId());
record.setCrawlStatus(CrawlStatusType.initializing);
LogWebSocketServlet.sendToAll("init-" + Integer.toString(crawlId));
// Build Configuration
CrawljaxConfigurationBuilder builder =
CrawljaxConfiguration.builderFor(config.getUrl());
builder.setBrowserConfig(new BrowserConfiguration(config.getBrowser(), config
.getNumBrowsers()));
if (config.getMaxDepth() > 0)
builder.setMaximumDepth(config.getMaxDepth());
else
builder.setUnlimitedCrawlDepth();
if (config.getMaxState() > 0)
builder.setMaximumStates(config.getMaxState());
else
builder.setUnlimitedStates();
if (config.getMaxDuration() > 0)
builder.setMaximumRunTime(config.getMaxDuration(), TimeUnit.MINUTES);
else
builder.setUnlimitedRuntime();
builder.crawlRules().clickOnce(config.isClickOnce());
builder.crawlRules().insertRandomDataInInputForms(config.isRandomFormInput());
builder.crawlRules().waitAfterEvent(config.getEventWaitTime(),
TimeUnit.MILLISECONDS);
builder.crawlRules().waitAfterReloadUrl(config.getReloadWaitTime(),
TimeUnit.MILLISECONDS);
// Click Rules
if (config.isClickDefault())
builder.crawlRules().clickDefaultElements();
else if (config.getClickRules().size() > 0) {
for (ClickRule r : config.getClickRules()) {
CrawlElement element;
if (r.getRule() == RuleType.click)
element = builder.crawlRules().click(r.getElementTag());
else
element = builder.crawlRules().dontClick(r.getElementTag());
if (r.getConditions().size() > 0) {
for (com.crawljax.web.model.Condition c : r.getConditions()) {
if (c.getCondition().toString().startsWith("w")) {
switch (c.getCondition()) {
case wAttribute:
String[] s =
c.getExpression().replace(" ", "").split("=");
element.withAttribute(s[0], s[1]);
break;
case wText:
element.withText(c.getExpression());
break;
case wXPath:
element.underXPath(c.getExpression());
break;
default:
break;
}
} else
element.when(getConditionFromConfig(c));
}
}
}
}
// Form Input
if (config.getFormInputValues().size() > 0) {
InputSpecification input = new InputSpecification();
for (NameValuePair p : config.getFormInputValues())
input.field(p.getName()).setValue(p.getValue());
builder.crawlRules().setInputSpec(input);
}
// Crawl Conditions
if (config.getPageConditions().size() > 0) {
for (com.crawljax.web.model.Condition c : config.getPageConditions()) {
builder.crawlRules().addCrawlCondition(
c.getCondition().toString() + c.getExpression(),
getConditionFromConfig(c));
}
}
// Invariants
if (config.getInvariants().size() > 0) {
for (com.crawljax.web.model.Condition c : config.getInvariants()) {
builder.crawlRules().addInvariant(
c.getCondition().toString() + c.getExpression(),
getConditionFromConfig(c));
}
}
// Comparators
if (config.getComparators().size() > 0)
setComparatorsFromConfig(config.getComparators(), builder.crawlRules());
//Plugins
File outputFolder = new File(record.getOutputFolder() + File.separatorChar + "plugins"
+ File.separatorChar + "0");
outputFolder.mkdirs();
builder.addPlugin(new CrawlOverview(new HostInterfaceImpl(outputFolder, new HashMap<String, String>())));
for (int i = 0, l = config.getPlugins().size(); i < l; i++) {
Plugin pluginConfig = config.getPlugins().get(i);
Plugin plugin = plugins.findByID(pluginConfig.getId());
if (plugin == null) {
LogWebSocketServlet.sendToAll("Could not find plugin: "
+ pluginConfig.getId());
continue;
}
if(!plugin.getCrawljaxVersions().contains(Main.getCrawljaxVersion())) {
LogWebSocketServlet.sendToAll("Plugin "
+ pluginConfig.getId() + " is not compatible with this version of Crawljax (" + Main.getCrawljaxVersion() + ")");
continue;
}
String pluginKey = String.valueOf(i + 1);
outputFolder = new File(record.getOutputFolder() + File.separatorChar + "plugins"
+ File.separatorChar + pluginKey);
outputFolder.mkdirs();
Map<String, String> parameters = new HashMap<>();
for (Parameter parameter : plugin.getParameters()) {
parameters.put(parameter.getId(), "");
for (Parameter configParam : pluginConfig.getParameters()) {
if (configParam.getId().equals(parameter.getId()) && configParam.getValue() != null) {
parameters.put(parameter.getId(), configParam.getValue());
}
}
}
HostInterface hostInterface = new HostInterfaceImpl(outputFolder, parameters);
com.crawljax.core.plugin.Plugin instance =
plugins.getInstanceOf(plugin, resourceDir, hostInterface);
if (instance != null) {
builder.addPlugin(instance);
record.getPlugins().put(pluginKey, plugin);
}
}
// Build Crawljax
CrawljaxRunner crawljax = new CrawljaxRunner(builder.build());
// Set Timestamps
timestamp = new Date();
record.setStartTime(timestamp);
record.setCrawlStatus(CrawlStatusType.running);
crawlRecords.update(record);
LogWebSocketServlet.sendToAll("run-" + Integer.toString(crawlId));
// run Crawljax
crawljax.call();
// set duration
long duration = (new Date()).getTime() - timestamp.getTime();
config = configurations.findByID(record.getConfigurationId()); //Reload config in case it was edited during crawl execution
config.setLastCrawl(timestamp);
config.setLastDuration(duration);
configurations.update(config);
record.setDuration(duration);
record.setCrawlStatus(CrawlStatusType.success);
crawlRecords.update(record);