if (Double.isNaN(relevanceThreshold)) {
throw new IllegalArgumentException("Invalid relevanceThreshold: " + evaluationPercentage);
}
int numItems = dataModel.getNumItems();
RunningAverage precision = new FullRunningAverage();
RunningAverage recall = new FullRunningAverage();
RunningAverage fallOut = new FullRunningAverage();
for (User user : dataModel.getUsers()) {
if (random.nextDouble() < evaluationPercentage) {
Object id = user.getID();
Collection<Item> relevantItems = new FastSet<Item>(at);
Preference[] prefs = user.getPreferencesAsArray();
double theRelevanceThreshold = Double.isNaN(relevanceThreshold) ? computeThreshold(prefs) : relevanceThreshold;
for (Preference pref : prefs) {
if (pref.getValue() >= theRelevanceThreshold) {
relevantItems.add(pref.getItem());
}
}
int numRelevantItems = relevantItems.size();
if (numRelevantItems > 0) {
List<User> trainingUsers = new ArrayList<User>(dataModel.getNumUsers());
for (User user2 : dataModel.getUsers()) {
processOtherUser(id, relevantItems, trainingUsers, user2);
}
DataModel trainingModel = new GenericDataModel(trainingUsers);
Recommender recommender = recommenderBuilder.buildRecommender(trainingModel);
try {
trainingModel.getUser(id);
} catch (NoSuchUserException nsee) {
continue; // Oops we excluded all prefs for the user -- just move on
}
int intersectionSize = 0;
List<RecommendedItem> recommendedItems = recommender.recommend(id, at, rescorer);
for (RecommendedItem recommendedItem : recommendedItems) {
if (relevantItems.contains(recommendedItem.getItem())) {
intersectionSize++;
}
}
int numRecommendedItems = recommendedItems.size();
if (numRecommendedItems > 0) {
precision.addDatum((double) intersectionSize / (double) numRecommendedItems);
}
recall.addDatum((double) intersectionSize / (double) numRelevantItems);
if (numRelevantItems < prefs.length) {
fallOut.addDatum((double) (numRecommendedItems - intersectionSize) /
(double) (numItems - numRelevantItems));
}
log.info("Precision/recall/fall-out: {} / {} / {}", new Object[] {
precision.getAverage(), recall.getAverage(), fallOut.getAverage()
});
}
}
}
return new IRStatisticsImpl(precision.getAverage(), recall.getAverage(), fallOut.getAverage());
}