Abstract

Providing high-quality news recommendations is a challenging task because the set of potentially relevant news items changes continuously, the relevance of news highly depends on the context, and there are tight time constraints for computing recommendations. The CLEF NewsREEL challenge is a campaign-style evaluation lab allowing participants to evaluate and optimize news recommender algorithms online and offline. In this paper, we discuss the objectives and challenges of the NewsREEL lab. We motivate the metrics used for benchmarking the recommender algorithms and explain the challenge dataset. In addition, we introduce the evaluation framework that we have developed. The framework makes possible the reproducible evaluation of recommender algorithms for stream data, taking into account recommender precision as well as the technical complexity of the recommender algorithms.

@inproceedings{KLTSLBS:Stream-BasedRecommendations-EvaluationAsAService,
author = {Benjamin Kille and Andreas Lommatzsch and Roberto Turrin and Andras Sereny and Martha Larson and Torben Brodt and Jonas Seiler and Frank Hopfgartner},
title = {Stream-Based Recommendations: Online and Offline Evaluation as a Service},
booktitle = {{CLEF}'15: Proceedings of the 6th International Conference of the {CLEF} Initiative},
publisher = {Springer Verlag},
year = {2015},
series = {LNCS, vol. 9283},
location = {Toulouse, France},
numpages = {20},
isbn = {978-3-319-24026-8},
pages = {497-517},
url = {http://dx.doi.org/10.1007/978-3-319-24027-5},
}