if (Double.isNaN(evaluationPercentage) || (evaluationPercentage <= 0.0) || (evaluationPercentage > 1.0)) {
throw new IllegalArgumentException("Invalid evaluationPercentage: " + evaluationPercentage);
}
int numItems = dataModel.getNumItems();
RunningAverage precision = new FullRunningAverage();
RunningAverage recall = new FullRunningAverage();
RunningAverage fallOut = new FullRunningAverage();
LongPrimitiveIterator it = dataModel.getUserIDs();
while (it.hasNext()) {
long userID = it.nextLong();
if (random.nextDouble() < evaluationPercentage) {
long start = System.currentTimeMillis();
FastIDSet relevantItemIDs = new FastIDSet(at);
PreferenceArray prefs = dataModel.getPreferencesFromUser(userID);
int size = prefs.length();
if (size < 2 * at) {
// Really not enough prefs to meaningfully evaluate this user
continue;
}
// List some most-preferred items that would count as (most) "relevant" results
double theRelevanceThreshold = Double.isNaN(relevanceThreshold) ?
computeThreshold(prefs) : relevanceThreshold;
prefs.sortByValueReversed();
for (int i = 0; (i < size) && (relevantItemIDs.size() < at); i++) {
if (prefs.getValue(i) >= theRelevanceThreshold) {
relevantItemIDs.add(prefs.getItemID(i));
}
}
int numRelevantItems = relevantItemIDs.size();
if (numRelevantItems > 0) {
FastByIDMap<PreferenceArray> trainingUsers = new FastByIDMap<PreferenceArray>(dataModel
.getNumUsers());
LongPrimitiveIterator it2 = dataModel.getUserIDs();
while (it2.hasNext()) {
processOtherUser(userID, relevantItemIDs, trainingUsers, it2
.nextLong(), dataModel);
}
DataModel trainingModel = dataModelBuilder == null ? new GenericDataModel(trainingUsers)
: dataModelBuilder.buildDataModel(trainingUsers);
Recommender recommender = recommenderBuilder.buildRecommender(trainingModel);
try {
trainingModel.getPreferencesFromUser(userID);
} catch (NoSuchUserException nsee) {
continue; // Oops we excluded all prefs for the user -- just move on
}
int intersectionSize = 0;
List<RecommendedItem> recommendedItems = recommender.recommend(userID, at, rescorer);
for (RecommendedItem recommendedItem : recommendedItems) {
if (relevantItemIDs.contains(recommendedItem.getItemID())) {
intersectionSize++;
}
}
int numRecommendedItems = recommendedItems.size();
if (numRecommendedItems > 0) {
precision.addDatum((double) intersectionSize / (double) numRecommendedItems);
}
recall.addDatum((double) intersectionSize / (double) numRelevantItems);
if (numRelevantItems < size) {
fallOut.addDatum((double) (numRecommendedItems - intersectionSize)
/ (double) (numItems - numRelevantItems));
}
long end = System.currentTimeMillis();
GenericRecommenderIRStatsEvaluator.log
.info("Evaluated with user {} in {}ms", userID, (end - start));
log.info("Precision/recall/fall-out: {} / {} / {}",
new Object[] {precision.getAverage(), recall.getAverage(), fallOut.getAverage()});
}
}
}
return new IRStatisticsImpl(precision.getAverage(), recall.getAverage(), fallOut.getAverage());
}