Abstract

In real-world scenarios, recommenders face non-functional requirements of technical nature and must handle dynamic data in the form of sequential streams. Evaluation of recommender systems must take these issues into account in order to be maximally informative. In this paper, we present Idomaar - a framework which enables the efficient multi-dimensional benchmarking of recommender algorithms. Idomaar exceeds current academic research practices by creating a realistic evaluation environment and computing both effectiveness and technical metrics for stream-based as well as set based evaluation. A scenario focussing on "research to prototyping to productization" cycle at a company illustrates Idomaar's potential. We show that Idomaar simplifies testing with varying configurations and supports flexible integration of different data.

@inproceedings{ScriminaciEtAl:Idomaar-AFrameworkForMultiDimensionalBenchmarkingOfRecommenderAlgorithms,
author = {Mario Scriminaci and Andreas Lommatzsch and Benjamin Kille and Frank Hopfgartner and Martha Larson and Davide Malagoli and Andas Sereny},
title = {Idomaar: A Framework for Multi-dimensional Benchmarking of Recommender Algorithms},
booktitle = {Poster Proc. of the 10th  ACM Conference on Recommender Systems, Boston, USA},
year = {2016},
series = {RecSys '16},
numpages = {2},
isbn = {urn:nbn:de:0074-1688-5},
location = {Boston, USA},
publisher = {ceur-ws.org},
keywords = {recommender systems, multi-dimensional benchmarking, stream-based evaluation, realistic evaluation}
}