Date startDate = new Date();
_logger.info("Starting harvest process at: " + startDate + ", host=" + hostname);
// Perform processing
PropertiesManager threadConfig = new PropertiesManager();
String sThreadConfig = threadConfig.getHarvestThreadConfig();
// Max time for harvester (defaults to 25 mins)
long maxTime_secs = threadConfig.getMaximumHarvestTime();
if (maxTime_secs > 0) {
new Timer().schedule(new InternalShutdown(), maxTime_secs*1000); // (arg in ms)
}//TOTEST
try {
// All source types in a single thread
int nThreads = Integer.parseInt(sThreadConfig);
SourceTypeHarvesterRunnable allTypes = new SourceTypeHarvesterRunnable(sources, nThreads);
_logger.info("(Launching " + nThreads + " threads for all source types)");
allTypes.run();
}
catch (NumberFormatException e) {
// The thread config must be comma-separated list of type:threads
// (step over each type and launch that number of threads for that type)
String[] sConfigBlocks = sThreadConfig.split("\\s*,\\s*");
ExecutorService exec = Executors.newFixedThreadPool(sConfigBlocks.length);
for (String sConfigBlock: sConfigBlocks) {
String[] sTypeOrNumThreads = sConfigBlock.split("\\s*:\\s*");
if (2 == sTypeOrNumThreads.length) {
try {
int nThreads = Integer.parseInt(sTypeOrNumThreads[1]);
SourceTypeHarvesterRunnable typeRunner = new SourceTypeHarvesterRunnable(sources, nThreads, sTypeOrNumThreads[0]);
_logger.info("(Launching " + nThreads + " threads for " + sTypeOrNumThreads[0] + " source types)");
exec.submit(typeRunner);
}
catch (NumberFormatException e2) {
_logger.error("Error in harvester thread configuration: " + sThreadConfig);
}
}
else {
_logger.error("Error in harvester thread configuration: " + sThreadConfig);
}
}//(end loop over different file types)
exec.shutdown();
while (!exec.isTerminated()) {
try {
Thread.sleep(1000);
}
catch (InterruptedException e3) { }
}
}
com.ikanow.infinit.e.processing.generic.utils.PropertiesManager aggProps = new com.ikanow.infinit.e.processing.generic.utils.PropertiesManager();
boolean bAggDisabled = aggProps.getAggregationDisabled();
StoreAndIndexManager dataStore = new StoreAndIndexManager();
boolean bResizedDB = dataStore.resizeDB();
if (!bAggDisabled) {
AggregationManager.updateEntitiesFromDeletedDocuments(dataStore.getUUID());
}
dataStore.removeSoftDeletedDocuments();
if (!bAggDisabled) {
AggregationManager.updateDocEntitiesFromDeletedDocuments(dataStore.getUUID());
}
if (bResizedDB) {
_logger.info("(resized DB, now " + dataStore.getDatabaseSize() + " documents)");
}
HarvestController.logHarvesterStats();
_logger.info("Completed harvest process at: " + new Date().toString());
Date endDate = new Date();
// Not allowed to cycle harvester runs too quickly
// Sleep for some period:
long nDiff = endDate.getTime() - startDate.getTime();
long nToSleep = threadConfig.getMinimumHarvestTimeMs() - nDiff;
if ((nToSleep > 0) && !_bCurrentlySleepingBeforeExit) {
try {
_bCurrentlySleepingBeforeExit = true; // (don't really care there's a minor race condition here)
Thread.sleep(nToSleep);
} catch (InterruptedException e) {