testWithTrainingSet(instances);
}
private void testWithTrainingSet(final List<Instance<AttributesMap>> instances) {
final PredictiveModelWithDataBuilderFactory predictiveModelBuilderFactory = new PredictiveModelWithDataBuilderFactory(new RandomForestBuilderFactory());
final ClassifierStationaryCrossValidator crossVal = new ClassifierStationaryCrossValidator(4, 4, new ClassifierLogCVLossFunction());
PredictiveModelOptimizer predictiveModelOptimizer = new PredictiveModelOptimizer(predictiveModelBuilderFactory, instances, crossVal);
final Map<String, Object> optimalParameters = predictiveModelOptimizer.determineOptimalConfiguration();
logger.info("Optimal parameters: " + optimalParameters);
RandomForestBuilder defaultRFBuilder = new RandomForestBuilder();
final PredictiveModelWithDataBuilder optimalRFBuilder = predictiveModelBuilderFactory.buildBuilder(optimalParameters);
double defaultLoss = crossVal.getCrossValidatedLoss(defaultRFBuilder, instances);
double optimizedLoss = crossVal.getCrossValidatedLoss(optimalRFBuilder, instances);
logger.info("Default PM loss: "+defaultLoss+", optimized PM loss: "+optimizedLoss);
Assert.assertTrue(optimizedLoss <= defaultLoss, "Default PM loss (" + defaultLoss + ") should be higher or equal to optimized PM loss (" + optimizedLoss + ")");
}