Abstract

Recommender systems are commonly evaluated by trying to predict known, withheld, ratings for a set of users. Measures such as the Root-Mean-Square Error are used to estimate the quality of the recommender algorithms. This process does however not acknowledge the inherent rating inconsistencies of users. In this paper we present the first results from a noise measurement user study for estimating the magic barrier of recommender systems conducted on a commercial movie recommendation community. The magic barrier is the expected squared error of the optimal recommendation algorithm, or, the lowest error we can expect from any recommendation algorithm. Our results show that the barrier can be estimated by collecting the opinions of users on already rated items.

@inproceedings{Said:2012:EMB,
 author = {Said, Alan and Jain, Brijnesh J. and Narr, Sascha and Plumbaum, Till and Albayrak, Sahin and Scheel, Christian},
 title = {Estimating the magic barrier of recommender systems: a user study},
 booktitle = {Proceedings of the 35th international ACM SIGIR conference on Research and development in information retrieval},
 series = {SIGIR '12},
 year = {2012},
 isbn = {978-1-4503-1472-5},
 location = {Portland, Oregon, USA},
 pages = {1061--1062},
 numpages = {2},
 url = {http://doi.acm.org/10.1145/2348283.2348469},
 doi = {10.1145/2348283.2348469},
 acmid = {2348469},
 publisher = {ACM},
 address = {New York, NY, USA},
 keywords = {evaluation, noise, recommender systems},
}