if (Double.isNaN(evaluationPercentage) || evaluationPercentage <= 0.0 || evaluationPercentage > 1.0) {
throw new IllegalArgumentException("Invalid evaluationPercentage: " + evaluationPercentage);
}
int numItems = dataModel.getNumItems();
RunningAverage precision = new FullRunningAverage();
RunningAverage recall = new FullRunningAverage();
RunningAverage fallOut = new FullRunningAverage();
LongPrimitiveIterator it = dataModel.getUserIDs();
while (it.hasNext()) {
long userID = it.nextLong();
if (random.nextDouble() < evaluationPercentage) {
long start = System.currentTimeMillis();
FastIDSet relevantItemIDs = new FastIDSet(at);
PreferenceArray prefs = dataModel.getPreferencesFromUser(userID);
int size = prefs.length();
if (size < 2 * at) {
// Really not enough prefs to meaningfully evaluate this user
continue;
}
// List some most-preferred items that would count as (most) "relevant" results
double theRelevanceThreshold = Double.isNaN(relevanceThreshold) ? computeThreshold(prefs) : relevanceThreshold;
prefs.sortByValueReversed();
for (int i = 0; i < size && relevantItemIDs.size() < at; i++) {
if (prefs.getValue(i) >= theRelevanceThreshold) {
relevantItemIDs.add(prefs.getItemID(i));
}
}
int numRelevantItems = relevantItemIDs.size();
if (numRelevantItems > 0) {
FastByIDMap<PreferenceArray> trainingUsers =
new FastByIDMap<PreferenceArray>(dataModel.getNumUsers());
LongPrimitiveIterator it2 = dataModel.getUserIDs();
while (it2.hasNext()) {
processOtherUser(userID, relevantItemIDs, trainingUsers, it2.nextLong(), dataModel);
}
DataModel trainingModel = dataModelBuilder == null ?
new GenericDataModel(trainingUsers) :
dataModelBuilder.buildDataModel(trainingUsers);
Recommender recommender = recommenderBuilder.buildRecommender(trainingModel);
try {
trainingModel.getPreferencesFromUser(userID);
} catch (NoSuchUserException nsee) {
continue; // Oops we excluded all prefs for the user -- just move on
}
int intersectionSize = 0;
List<RecommendedItem> recommendedItems = recommender.recommend(userID, at, rescorer);
for (RecommendedItem recommendedItem : recommendedItems) {
if (relevantItemIDs.contains(recommendedItem.getItemID())) {
intersectionSize++;
}
}
int numRecommendedItems = recommendedItems.size();
if (numRecommendedItems > 0) {
precision.addDatum((double) intersectionSize / (double) numRecommendedItems);
}
recall.addDatum((double) intersectionSize / (double) numRelevantItems);
if (numRelevantItems < size) {
fallOut.addDatum((double) (numRecommendedItems - intersectionSize) /
(double) (numItems - numRelevantItems));
}
long end = System.currentTimeMillis();
log.info("Evaluated with user " + userID + " in " + (end - start) + "ms");
log.info("Precision/recall/fall-out: {} / {} / {}", new Object[]{
precision.getAverage(), recall.getAverage(), fallOut.getAverage()
});
}
}
}
return new IRStatisticsImpl(precision.getAverage(), recall.getAverage(), fallOut.getAverage());
}