public IndexerModule (String moduleName, Config globalConfig) {
super(moduleName, globalConfig);
textLengthLimit = globalConfig.getInt("page.text.max.length");
titleLengthLimit = globalConfig.getInt("page.title.max.length");
Config mdlConfig = getModuleConfig();
indexerBusyRetryTime = mdlConfig.getInt("indexer.busy.retry.time");
crawlName = globalConfig.getString("crawler.name");
categoryBoostDamp = weightToDamp(mdlConfig.getFloat("category.boost.weight"));
pagerankBoostDamp = weightToDamp(mdlConfig.getFloat("pagerank.boost.weight"));
spamrankBoostDamp = weightToDamp(mdlConfig.getFloat("spamrank.boost.weight"));
logBoostDamp = weightToDamp(mdlConfig.getFloat("log.boost.weight"));
freshnessBoostDamp = weightToDamp(mdlConfig.getFloat("freshness.boost.weight"));
int[] freshnessParams = mdlConfig.getIntArray("freshness.params");
freshnessWeight = freshnessParams[0];
freshnessDamp = freshnessParams[1];
hostStopWords = new HashSet<String>(Arrays.asList(mdlConfig.getStringArray("host.stopwords")));
sendContent = mdlConfig.getBoolean("send.content.to.indexer");
// instantiate the indexer.
if (mdlConfig.getBoolean("use.mock.indexer")) {
logger.warn("Using a mock indexer. This should be used only for testing.");
this.indexers = new IRemoteIndexer[1];
this.indexers[0] = new MockIndexer();
pageMapper = new UrlHashMapper(mdlConfig, 1);
} else {
String[] specs = null;
try {
specs = mdlConfig.getStringArray("indexer.node.list");
} catch (Exception e) {
// for backward compatibility:
specs = mdlConfig.getStringArray("remoteRmiIndexer.host");
}
this.indexers = new IRemoteIndexer[specs.length];
for (int i=0; i<specs.length; i++) {
Pair<String, Integer> host = PortUtil.parseHost(specs[i], "indexer.rmi");
this.indexers[i] = new RmiIndexerStub(host.last(), host.first());