@article{DBLP:journals/tmi/ZimmererFIJAPKR22,
  author    = {David Zimmerer and
               Peter M. Full and
               Fabian Isensee and
               Paul J{\"{a}}ger and
               Tim Adler and
               Jens Petersen and
               Gregor K{\"{o}}hler and
               Tobias Ro{\ss} and
               Annika Reinke and
               Antanas Kascenas and
               Bj{\o}rn Sand Jensen and
               Alison Q. O'Neil and
               Jeremy Tan and
               Benjamin Hou and
               James Batten and
               Huaqi Qiu and
               Bernhard Kainz and
               Nina Shvetsova and
               Irina Fedulova and
               Dmitry V. Dylov and
               Baolun Yu and
               Jianyang Zhai and
               Jingtao Hu and
               Runxuan Si and
               Sihang Zhou and
               Siqi Wang and
               Xinyang Li and
               Xuerun Chen and
               Yang Zhao and
               Sergio Naval Marimont and
               Giacomo Tarroni and
               Victor Saase and
               Lena Maier{-}Hein and
               Klaus H. Maier{-}Hein},
  title     = {{MOOD} 2020: {A} Public Benchmark for Out-of-Distribution Detection
               and Localization on Medical Images},
  journal   = {{IEEE} Trans. Medical Imaging},
  volume    = {41},
  number    = {10},
  pages     = {2728--2738},
  year      = {2022},
  doi = {10.1109/tmi.2022.3170077},
  url = {https://doi.org/10.1109/tmi.2022.3170077},
  abstract = {Detecting Out-of-Distribution (OoD) data is one of the greatest challenges in safe and robust deployment of machine learning algorithms in medicine. When the algorithms encounter cases that deviate from the distribution of the training data, they often produce incorrect and over-confident predictions. OoD detection algorithms aim to catch erroneous predictions in advance by analysing the data distribution and detecting potential instances of failure. Moreover, flagging OoD cases may support human readers in identifying incidental findings. Due to the increased interest in OoD algorithms, benchmarks for different domains have recently been established. In the medical imaging domain, for which reliable predictions are often essential, an open benchmark has been missing. We introduce the Medical-Out-Of-Distribution-Analysis-Challenge (MOOD) as an open, fair, and unbiased benchmark for OoD methods in the medical imaging domain. The analysis of the submitted algorithms shows that performance has a strong positive correlation with the perceived difficulty, and that all algorithms show a high variance for different anomalies, making it yet hard to recommend them for clinical practice. We also see a strong correlation between challenge ranking and performance on a simple toy test set, indicating that this might be a valuable addition as a proxy dataset during anomaly detection algorithm development.},
  code = {https://github.com/MIC-DKFZ/mood},
}