\section{Data Sets}\label{sec:experiments:data_sets}
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{מערך נתונים}}

\paragraph{KEEL benchmark.}
We evaluate on 12 binary imbalanced datasets from the KEEL repository~\cite{alcalafdez2011keel}. All labels are clean. \Cref{tab:datasets:keel} summarises the datasets; imbalance ratios range from 1.8 to 11.6. No known noise is reported in the original data.

\begin{table}[htbp]
\centering
\caption{KEEL benchmark datasets used in experiments. IR = negative/positive count ratio.}
\label{tab:datasets:keel}
\small
\begin{tabular}{lrrrrl}
\toprule
Dataset & $n$ & Pos & Neg & IR & Source \\
\midrule
Glass0       &  214 &  70 & 144 &  2.1 & Glass identification (UCI) \\
Glass1       &  214 &  76 & 138 &  1.8 & Glass identification (UCI) \\
Glass2       &  214 &  17 & 197 & 11.6 & Glass identification (UCI) \\
Glass6       &  214 &  29 & 185 &  6.4 & Glass identification (UCI) \\
Haberman     &  306 &  81 & 225 &  2.8 & Breast cancer survival, 1958--1970 \\
Iris0        &  150 &  50 & 100 &  2.0 & \emph{Iris setosa} vs.\ others \\
New-Thyroid1 &  215 &  35 & 180 &  5.1 & Thyroid function classification \\
Pima         &  768 & 268 & 500 &  1.9 & Pima Indian diabetes (UCI) \\
Vehicle2     &  846 & 218 & 628 &  2.9 & Vehicle silhouette (UCI) \\
Vowel0       &  988 &  90 & 898 & 10.0 & Vowel recognition, class 0 vs.\ rest \\
Wisconsin    &  683 & 239 & 444 &  1.9 & Wisconsin breast cancer (UCI) \\
Yeast3       & 1484 & 163 &1321 &  8.1 & Yeast protein localisation (UCI) \\
\bottomrule
\end{tabular}
\end{table}

\paragraph{CIFAR-10N.}
CIFAR-10 is a widely used image dataset of 60{,}000 32x32 color images belonging to 10 mutuall exclusive classes. CIFAR-10N~\cite{wei2022cifar10n} augments the 50{,}000 training set labels with real human-annotated noisy labels collected from Amazon Mechanical Turk~\cite{wei2022cifar10n}. We form a binary task (class~1, automobile, vs.\ rest; IR~$\approx 9$:1) and train under three noise conditions:
\emph{clean} (original CIFAR-10 labels, 0\% noise),
\emph{aggre} (majority-vote aggregate of crowdworker annotations, $\approx$9\% noise), and
\emph{worse} (single worst-annotator labels, $\approx$40\% noise).
This allows us to measure the degredation of the model's performance under increasing label noise conditions.
The architecture tested is a three-block convolutional network trained from scratch.

\paragraph{Large tabular datasets.}
We include three large-scale tabular datasets spanning diverse imbalanced real-world settings.

\begin{itemize}
  \item \textbf{Bank Marketing}~\cite{moro2014bank}, UCI repository; Portuguese bank telephone marketing campaigns (Moro et al., 2014). $n = 41{,}188$ samples, 48 features after one-hot encoding. Positive class: client subscribed to term deposit. IR~$\approx 8$:1 ($\approx$11.3\% positive). No label noise.

  \item \textbf{Credit Card Fraud}~\cite{dalpozzolo2015calibrating}, Kaggle / ULB (Worldline + MLG, 2013). $n = 284{,}807$ transactions, 30 features (V1--V28 are PCA-transformed; Time and Amount rescaled). Extreme imbalance: 492 confirmed frauds, IR~$\approx 577$:1. Labels reflect confirmed-fraud reports; undetected fraudulent transactions constitute a structural source of class-conditional false negatives~\cite{tittelfitz2023fpr}.

  \item \textbf{Home Credit}~\cite{homecredit2018kaggle}, Kaggle Home Credit Default Risk competition (2018). $n = 307{,}511$ loan applications, $\approx$120 numerical features from the primary application table. Positive class: loan default. IR~$\approx 11.4$:1 ($\approx$8\% default rate). Labels are proxy labels (repayment outcome); no injected noise.
\end{itemize}

\paragraph{NIH ChestX-ray14.}
NIH ChestX-ray14~\cite{wang2017chestxray14} is a large-scale medical chest radiograph dataset of $112{,}120$ X-rays from $30{,}805$ patients, annotated with 14 unique labels. Labels were extracted from radiological reports via natural language processing; the dataset authors estimate labelling accuracy at $>$90\%~\cite{wang2017chestxray14}, though subsequent audits have found the true error rate substantially higher, with positive predictive values 10--30 percentage points below the reported figures~\cite{oakdenrayner2020exploring}, making this a naturally noisy benchmark.  We form a binary task targeting \emph{pleural effusion} detection (Effusion vs.\ rest; IR~$\approx 7.5$:1).

We extract $1{,}024$-dimensional latent representations using a \mbox{DenseNet-121} pretrained on the dataset via the TorchXRayVision library~\cite{cohen2022torchxrayvision}, with global average pooling applied to the penultimate feature map.  These frozen embeddings are passed to a lightweight $1024{\to}512{\to}128{\to}1$ MLP with dropout, treating image representations as tabular features.  This setup evaluates ROLL in a transfer-learning scenario and avoids GPU-intensive end-to-end training.

\section{Performance Measures}\label{sec:experiments:performance_measures}
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{מדדי ביצועים}}

We report the \emph{area under the ROC curve} (AUC) averaged over independent training episodes (see \Cref{sec:experiments:configurations}). AUC measures overall ranking quality and is a standard metric for imbalanced classification. All results are presented as mean $\pm$ standard deviation over episodes.

\section{Configurations and Parameter Settings}
\label{sec:experiments:configurations}
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{קונפיגורציה ופרמטרים}}

We compare \textsc{Roll}-AoC (the primary proposed loss, optimising the area under the ROC curve via KDE) against the following baselines:

\begin{itemize}
  \item \textbf{BCE-W}: binary cross-entropy with class-frequency inverse weighting.
  \item \textbf{Focal}: focal loss~\cite{lin2017focal} with focusing parameter $\gamma = 2$.
  \item \textbf{Asym.}: asymmetric loss~\cite{zhou2021asymmetric}.
  \item \textbf{GCE}: generalised cross-entropy~\cite{zhang2018generalized} with $q = 0.7$.
  \item \textbf{LibAUC}: PESG-based AUC maximisation from the LibAUC library~\cite{yuan2021auc}.
  \item \textbf{MAE}: mean absolute error (symmetric noise-robust loss).
\end{itemize}

All methods use the same network architecture, optimiser (Adam for non-LibAUC methods), and early stopping. The best model per-episode is chosen from the epoch with lowest loss. Each configuration is run for $n = 5$ independent episodes on KEEL and large datasets, and $n = 3$ on CIFAR-10N. Reported AUC is evaluated on the test split, averaged between episodes.

\section{Results}\label{sec:experiments:results}
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{תוצאות}}

% TODO: Review which KEEL noise tables to keep in the main body.
% Low noise levels (5%, 10%) show little differentiation and may not add much to the narrative.
% Consider: keep only 20% noise table here (clearest signal, Roll-AoC leads on 6/12 datasets)
% and move 5%/10% tables to an appendix, referencing them from here for completeness.
\subsection{KEEL Benchmark}\label{sec:results:keel}

\Cref{tab:results:keel} reports mean test AUC across the 12 KEEL datasets under clean labels.
No single method dominates: on easy datasets (Iris0, Wisconsin, Vowel0, Vehicle2, Glass6) nearly all methods converge to similar AUCs, leaving little room for differentiation.
On the harder, more imbalanced datasets the picture is more varied.
\textsc{Roll}-AoC competes well but does not claim the top position on most individual datasets, establishing it as a competitive approach.

To assess label-noise robustness, we inject symmetric label noise. A fraction of all class labels are randomly flipped.
We evaluate at noise levels of $5\%, 10\%, 20\%$, running 10 independent episodes per dataset per noise level.
\Cref{tab:results:keel-noise-05,tab:results:keel-noise-10,tab:results:keel-noise-20} show per-dataset results at each noise level.
The randomly flipped labels change each episode, yet are consistent between configurations, ensuring every configuration, per episode, sees the same labels, changed in the same way.

At $0\%$ noise, results are quite similar across configurations. \textsc{Roll}-AOC trails right behind the leader in most configurations. Since datasets are small, the \textbf{std} given for each point is relatively high, and quite a bit of variability is expected. Still, we see that, with few exceptions, results between configurations seem within throwing distance of eachother.
As noise increases the ranking shifts markedly: at $20\%$ noise \textsc{Roll}-AoC reaches the top on average (0.847), while traditionally noise-robust losses --- \textsc{GCE} (0.831) and \textsc{MAE} (0.804) --- degrade substantially.
The per-dataset tables reveal that this trend is consistent across most datasets rather than driven by a small subset: at $20\%$ noise \textsc{Roll}-AoC leads on 6 of 12 datasets (Glass0, Glass1, Glass2, Pima, Vehicle2, Yeast3), while no other method leads on more than 3.
From $0\%$ to $20\%$ noise, \textsc{Roll}-AoC drops 0.033 in mean AUC (3.7\%), compared to 0.056 for \textsc{GCE} (6.3\%), 0.071 for \textsc{MAE} (8.1\%), and 0.073 for \textsc{LibAUC} (8.3\%).
This supports the hypothesis that ROLL's distributional objective provides implicit noise robustness: mislabelled samples shift the fitted score distribution gradually rather than producing isolated large-gradient outliers.

% TODO: audit bold/emphasis across all result tables — check that bold marks the correct per-row best and that no ties or near-ties are missed or incorrectly marked.
\begin{table}[htbp]
\centering
\caption{Mean test AUC ($\pm$\,std, $n=5$ episodes) on KEEL benchmark datasets under clean labels. Best per row in \textbf{bold}.}
\label{tab:results:keel}
\resizebox{\textwidth}{!}{%
\begin{tabular}{lcccccccc}
\toprule
Dataset & \textsc{Roll}-AoC & \textsc{Roll}-TPR & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\
\midrule
Glass0 & 0.796{\tiny{$\pm$0.046}} & 0.826{\tiny{$\pm$0.032}} & 0.839{\tiny{$\pm$0.033}} & 0.802{\tiny{$\pm$0.020}} & 0.814{\tiny{$\pm$0.034}} & 0.818{\tiny{$\pm$0.032}} & \textbf{0.843{\tiny{$\pm$0.035}}} & 0.826{\tiny{$\pm$0.037}} \\
Glass1 & 0.722{\tiny{$\pm$0.033}} & 0.698{\tiny{$\pm$0.042}} & 0.710{\tiny{$\pm$0.018}} & 0.712{\tiny{$\pm$0.036}} & 0.744{\tiny{$\pm$0.044}} & 0.742{\tiny{$\pm$0.039}} & 0.723{\tiny{$\pm$0.054}} & \textbf{0.769{\tiny{$\pm$0.031}}} \\
Glass2 & 0.723{\tiny{$\pm$0.062}} & 0.701{\tiny{$\pm$0.055}} & 0.661{\tiny{$\pm$0.069}} & 0.677{\tiny{$\pm$0.072}} & \textbf{0.750{\tiny{$\pm$0.075}}} & 0.689{\tiny{$\pm$0.023}} & 0.719{\tiny{$\pm$0.084}} & 0.679{\tiny{$\pm$0.018}} \\
Glass6 & 0.975{\tiny{$\pm$0.014}} & 0.975{\tiny{$\pm$0.027}} & 0.972{\tiny{$\pm$0.025}} & 0.974{\tiny{$\pm$0.020}} & 0.982{\tiny{$\pm$0.013}} & 0.971{\tiny{$\pm$0.020}} & 0.969{\tiny{$\pm$0.012}} & \textbf{0.985{\tiny{$\pm$0.006}}} \\
Haberman & 0.696{\tiny{$\pm$0.045}} & 0.675{\tiny{$\pm$0.063}} & 0.736{\tiny{$\pm$0.012}} & \textbf{0.742{\tiny{$\pm$0.025}}} & 0.741{\tiny{$\pm$0.018}} & 0.737{\tiny{$\pm$0.016}} & 0.616{\tiny{$\pm$0.116}} & 0.671{\tiny{$\pm$0.102}} \\
Iris0 & \textbf{1.000{\tiny{$\pm$0.000}}} & 0.981{\tiny{$\pm$0.027}} & 0.900{\tiny{$\pm$0.224}} & \textbf{1.000{\tiny{$\pm$0.000}}} & \textbf{1.000{\tiny{$\pm$0.000}}} & \textbf{1.000{\tiny{$\pm$0.000}}} & \textbf{1.000{\tiny{$\pm$0.000}}} & 0.900{\tiny{$\pm$0.224}} \\
New-Thyroid1 & 0.989{\tiny{$\pm$0.010}} & 0.845{\tiny{$\pm$0.241}} & 0.992{\tiny{$\pm$0.005}} & 0.977{\tiny{$\pm$0.033}} & 0.988{\tiny{$\pm$0.007}} & 0.987{\tiny{$\pm$0.011}} & 0.985{\tiny{$\pm$0.012}} & \textbf{0.994{\tiny{$\pm$0.003}}} \\
Pima & 0.778{\tiny{$\pm$0.024}} & 0.755{\tiny{$\pm$0.014}} & \textbf{0.787{\tiny{$\pm$0.014}}} & 0.775{\tiny{$\pm$0.020}} & 0.771{\tiny{$\pm$0.019}} & 0.785{\tiny{$\pm$0.005}} & 0.761{\tiny{$\pm$0.015}} & 0.778{\tiny{$\pm$0.005}} \\
Vehicle2 & 0.982{\tiny{$\pm$0.003}} & 0.966{\tiny{$\pm$0.020}} & \textbf{0.984{\tiny{$\pm$0.004}}} & 0.982{\tiny{$\pm$0.003}} & 0.980{\tiny{$\pm$0.004}} & 0.982{\tiny{$\pm$0.004}} & 0.982{\tiny{$\pm$0.003}} & 0.983{\tiny{$\pm$0.004}} \\
Vowel0 & 0.993{\tiny{$\pm$0.005}} & 0.987{\tiny{$\pm$0.002}} & 0.995{\tiny{$\pm$0.004}} & 0.996{\tiny{$\pm$0.003}} & 0.995{\tiny{$\pm$0.003}} & 0.997{\tiny{$\pm$0.003}} & 0.991{\tiny{$\pm$0.007}} & \textbf{0.998{\tiny{$\pm$0.001}}} \\
Wisconsin & 0.997{\tiny{$\pm$0.002}} & 0.800{\tiny{$\pm$0.441}} & \textbf{0.999{\tiny{$\pm$0.000}}} & \textbf{0.999{\tiny{$\pm$0.001}}} & 0.999{\tiny{$\pm$0.001}} & 0.999{\tiny{$\pm$0.001}} & 0.999{\tiny{$\pm$0.000}} & 0.998{\tiny{$\pm$0.001}} \\
Yeast3 & 0.906{\tiny{$\pm$0.024}} & 0.908{\tiny{$\pm$0.021}} & 0.894{\tiny{$\pm$0.008}} & 0.908{\tiny{$\pm$0.019}} & \textbf{0.935{\tiny{$\pm$0.014}}} & 0.927{\tiny{$\pm$0.006}} & \textbf{0.935{\tiny{$\pm$0.016}}} & 0.915{\tiny{$\pm$0.010}} \\
\bottomrule
\end{tabular}%
}
\end{table}

\begin{table}[htbp]
\centering
\caption{Mean test AUC ($\pm$\,std, $n=10$ episodes) on KEEL datasets at $\epsilon=5\%$ asymmetric label noise. Best per row in \textbf{bold}.}
\label{tab:results:keel-noise-05}
\resizebox{\textwidth}{!}{%
\begin{tabular}{lcccccccc}
\toprule
Dataset & \textsc{Roll}-AoC & \textsc{Roll}-TPR & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\
\midrule
Glass0       & 0.797{\tiny{$\pm$0.046}} & 0.759{\tiny{$\pm$0.060}} & 0.786{\tiny{$\pm$0.019}} & \textbf{0.806{\tiny{$\pm$0.030}}} & 0.792{\tiny{$\pm$0.041}} & 0.794{\tiny{$\pm$0.029}} & 0.791{\tiny{$\pm$0.056}} & 0.792{\tiny{$\pm$0.058}} \\
Glass1       & 0.728{\tiny{$\pm$0.037}} & 0.702{\tiny{$\pm$0.041}} & 0.736{\tiny{$\pm$0.032}} & 0.730{\tiny{$\pm$0.074}} & 0.704{\tiny{$\pm$0.028}} & 0.717{\tiny{$\pm$0.059}} & 0.694{\tiny{$\pm$0.060}} & \textbf{0.738{\tiny{$\pm$0.047}}} \\
Glass2       & 0.677{\tiny{$\pm$0.095}} & 0.607{\tiny{$\pm$0.114}} & 0.665{\tiny{$\pm$0.128}} & 0.581{\tiny{$\pm$0.091}} & 0.662{\tiny{$\pm$0.050}} & 0.651{\tiny{$\pm$0.078}} & 0.614{\tiny{$\pm$0.085}} & \textbf{0.686{\tiny{$\pm$0.091}}} \\
Glass6       & 0.972{\tiny{$\pm$0.021}} & 0.909{\tiny{$\pm$0.068}} & 0.972{\tiny{$\pm$0.035}} & 0.978{\tiny{$\pm$0.016}} & \textbf{0.981{\tiny{$\pm$0.016}}} & 0.947{\tiny{$\pm$0.049}} & 0.967{\tiny{$\pm$0.036}} & 0.971{\tiny{$\pm$0.031}} \\
Haberman     & 0.666{\tiny{$\pm$0.094}} & 0.562{\tiny{$\pm$0.148}} & 0.627{\tiny{$\pm$0.124}} & 0.694{\tiny{$\pm$0.111}} & 0.693{\tiny{$\pm$0.065}} & 0.687{\tiny{$\pm$0.099}} & 0.651{\tiny{$\pm$0.125}} & \textbf{0.716{\tiny{$\pm$0.055}}} \\
Iris0        & 0.990{\tiny{$\pm$0.014}} & 0.770{\tiny{$\pm$0.284}} & 0.998{\tiny{$\pm$0.006}} & 0.999{\tiny{$\pm$0.002}} & 0.999{\tiny{$\pm$0.002}} & \textbf{1.000{\tiny{$\pm$0.000}}} & \textbf{1.000{\tiny{$\pm$0.000}}} & 0.999{\tiny{$\pm$0.003}} \\
New-Thyroid1 & 0.984{\tiny{$\pm$0.010}} & 0.609{\tiny{$\pm$0.408}} & 0.986{\tiny{$\pm$0.007}} & 0.971{\tiny{$\pm$0.046}} & 0.989{\tiny{$\pm$0.006}} & 0.987{\tiny{$\pm$0.019}} & 0.983{\tiny{$\pm$0.023}} & \textbf{0.991{\tiny{$\pm$0.012}}} \\
Pima         & 0.780{\tiny{$\pm$0.021}} & 0.761{\tiny{$\pm$0.030}} & 0.784{\tiny{$\pm$0.016}} & 0.780{\tiny{$\pm$0.019}} & 0.767{\tiny{$\pm$0.023}} & \textbf{0.787{\tiny{$\pm$0.011}}} & 0.756{\tiny{$\pm$0.023}} & 0.775{\tiny{$\pm$0.023}} \\
Vehicle2     & 0.975{\tiny{$\pm$0.007}} & 0.954{\tiny{$\pm$0.043}} & \textbf{0.977{\tiny{$\pm$0.006}}} & 0.974{\tiny{$\pm$0.010}} & 0.969{\tiny{$\pm$0.016}} & 0.976{\tiny{$\pm$0.007}} & 0.975{\tiny{$\pm$0.008}} & 0.976{\tiny{$\pm$0.009}} \\
Vowel0       & 0.988{\tiny{$\pm$0.012}} & 0.959{\tiny{$\pm$0.028}} & 0.981{\tiny{$\pm$0.019}} & 0.989{\tiny{$\pm$0.017}} & \textbf{0.991{\tiny{$\pm$0.006}}} & 0.985{\tiny{$\pm$0.014}} & 0.973{\tiny{$\pm$0.025}} & 0.978{\tiny{$\pm$0.014}} \\
Wisconsin    & 0.998{\tiny{$\pm$0.002}} & 0.997{\tiny{$\pm$0.004}} & 0.998{\tiny{$\pm$0.001}} & \textbf{0.999{\tiny{$\pm$0.001}}} & \textbf{0.999{\tiny{$\pm$0.001}}} & 0.998{\tiny{$\pm$0.001}} & 0.998{\tiny{$\pm$0.001}} & 0.998{\tiny{$\pm$0.001}} \\
Yeast3       & 0.900{\tiny{$\pm$0.020}} & 0.843{\tiny{$\pm$0.070}} & 0.914{\tiny{$\pm$0.028}} & 0.915{\tiny{$\pm$0.034}} & \textbf{0.916{\tiny{$\pm$0.026}}} & \textbf{0.916{\tiny{$\pm$0.032}}} & 0.908{\tiny{$\pm$0.029}} & 0.909{\tiny{$\pm$0.024}} \\
\bottomrule
\end{tabular}%
}
\end{table}

\begin{table}[htbp]
\centering
\caption{Mean test AUC ($\pm$\,std, $n=10$ episodes) on KEEL datasets at $\epsilon=10\%$ asymmetric label noise. Best per row in \textbf{bold}.}
\label{tab:results:keel-noise-10}
\resizebox{\textwidth}{!}{%
\begin{tabular}{lcccccccc}
\toprule
Dataset & \textsc{Roll}-AoC & \textsc{Roll}-TPR & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\
\midrule
Glass0       & 0.773{\tiny{$\pm$0.070}} & 0.767{\tiny{$\pm$0.046}} & 0.789{\tiny{$\pm$0.037}} & 0.771{\tiny{$\pm$0.037}} & \textbf{0.821{\tiny{$\pm$0.026}}} & 0.786{\tiny{$\pm$0.057}} & 0.800{\tiny{$\pm$0.032}} & 0.807{\tiny{$\pm$0.050}} \\
Glass1       & 0.719{\tiny{$\pm$0.040}} & 0.725{\tiny{$\pm$0.029}} & 0.716{\tiny{$\pm$0.079}} & 0.718{\tiny{$\pm$0.056}} & 0.710{\tiny{$\pm$0.049}} & 0.729{\tiny{$\pm$0.048}} & 0.675{\tiny{$\pm$0.070}} & \textbf{0.738{\tiny{$\pm$0.060}}} \\
Glass2       & 0.695{\tiny{$\pm$0.075}} & 0.649{\tiny{$\pm$0.119}} & 0.644{\tiny{$\pm$0.072}} & 0.687{\tiny{$\pm$0.127}} & 0.655{\tiny{$\pm$0.048}} & \textbf{0.713{\tiny{$\pm$0.107}}} & 0.637{\tiny{$\pm$0.093}} & 0.673{\tiny{$\pm$0.111}} \\
Glass6       & 0.947{\tiny{$\pm$0.050}} & 0.910{\tiny{$\pm$0.067}} & \textbf{0.973{\tiny{$\pm$0.026}}} & 0.950{\tiny{$\pm$0.041}} & 0.971{\tiny{$\pm$0.024}} & 0.938{\tiny{$\pm$0.047}} & 0.937{\tiny{$\pm$0.048}} & 0.935{\tiny{$\pm$0.054}} \\
Haberman     & 0.638{\tiny{$\pm$0.098}} & 0.564{\tiny{$\pm$0.144}} & 0.700{\tiny{$\pm$0.074}} & 0.652{\tiny{$\pm$0.106}} & 0.658{\tiny{$\pm$0.087}} & 0.636{\tiny{$\pm$0.112}} & 0.694{\tiny{$\pm$0.129}} & \textbf{0.732{\tiny{$\pm$0.022}}} \\
Iris0        & 0.991{\tiny{$\pm$0.020}} & 0.879{\tiny{$\pm$0.209}} & 0.950{\tiny{$\pm$0.158}} & 0.950{\tiny{$\pm$0.158}} & \textbf{1.000{\tiny{$\pm$0.000}}} & \textbf{1.000{\tiny{$\pm$0.001}}} & \textbf{1.000{\tiny{$\pm$0.000}}} & \textbf{1.000{\tiny{$\pm$0.000}}} \\
New-Thyroid1 & 0.986{\tiny{$\pm$0.011}} & 0.588{\tiny{$\pm$0.358}} & 0.980{\tiny{$\pm$0.024}} & 0.987{\tiny{$\pm$0.012}} & \textbf{0.991{\tiny{$\pm$0.013}}} & 0.986{\tiny{$\pm$0.019}} & 0.973{\tiny{$\pm$0.049}} & 0.986{\tiny{$\pm$0.013}} \\
Pima         & 0.756{\tiny{$\pm$0.026}} & 0.752{\tiny{$\pm$0.027}} & 0.760{\tiny{$\pm$0.024}} & 0.767{\tiny{$\pm$0.023}} & \textbf{0.780{\tiny{$\pm$0.013}}} & 0.761{\tiny{$\pm$0.030}} & 0.757{\tiny{$\pm$0.032}} & 0.754{\tiny{$\pm$0.032}} \\
Vehicle2     & 0.967{\tiny{$\pm$0.015}} & 0.939{\tiny{$\pm$0.020}} & 0.964{\tiny{$\pm$0.016}} & 0.962{\tiny{$\pm$0.020}} & 0.962{\tiny{$\pm$0.017}} & 0.965{\tiny{$\pm$0.009}} & 0.953{\tiny{$\pm$0.022}} & \textbf{0.970{\tiny{$\pm$0.010}}} \\
Vowel0       & \textbf{0.985{\tiny{$\pm$0.011}}} & 0.946{\tiny{$\pm$0.036}} & \textbf{0.984{\tiny{$\pm$0.016}}} & 0.979{\tiny{$\pm$0.025}} & 0.975{\tiny{$\pm$0.022}} & 0.977{\tiny{$\pm$0.024}} & 0.959{\tiny{$\pm$0.035}} & 0.946{\tiny{$\pm$0.039}} \\
Wisconsin    & 0.996{\tiny{$\pm$0.001}} & 0.717{\tiny{$\pm$0.453}} & 0.996{\tiny{$\pm$0.004}} & 0.997{\tiny{$\pm$0.003}} & 0.997{\tiny{$\pm$0.003}} & \textbf{0.998{\tiny{$\pm$0.001}}} & 0.996{\tiny{$\pm$0.004}} & \textbf{0.998{\tiny{$\pm$0.001}}} \\
Yeast3       & 0.895{\tiny{$\pm$0.034}} & 0.810{\tiny{$\pm$0.097}} & 0.891{\tiny{$\pm$0.054}} & 0.868{\tiny{$\pm$0.069}} & \textbf{0.915{\tiny{$\pm$0.030}}} & 0.891{\tiny{$\pm$0.029}} & 0.872{\tiny{$\pm$0.040}} & 0.857{\tiny{$\pm$0.059}} \\
\bottomrule
\end{tabular}%
}
\end{table}

\begin{table}[htbp]
\centering
\caption{Mean test AUC ($\pm$\,std, $n=10$ episodes) on KEEL datasets at $\epsilon=20\%$ asymmetric label noise. Best per row in \textbf{bold}.}
\label{tab:results:keel-noise-20}
\resizebox{\textwidth}{!}{%
\begin{tabular}{lcccccccc}
\toprule
Dataset & \textsc{Roll}-AoC & \textsc{Roll}-TPR & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\
\midrule
Glass0       & \textbf{0.801{\tiny{$\pm$0.042}}} & 0.758{\tiny{$\pm$0.049}} & 0.767{\tiny{$\pm$0.042}} & 0.796{\tiny{$\pm$0.040}} & 0.793{\tiny{$\pm$0.051}} & 0.765{\tiny{$\pm$0.052}} & 0.745{\tiny{$\pm$0.089}} & 0.791{\tiny{$\pm$0.044}} \\
Glass1       & \textbf{0.725{\tiny{$\pm$0.036}}} & 0.682{\tiny{$\pm$0.068}} & 0.718{\tiny{$\pm$0.054}} & 0.714{\tiny{$\pm$0.036}} & 0.700{\tiny{$\pm$0.039}} & 0.706{\tiny{$\pm$0.041}} & 0.635{\tiny{$\pm$0.052}} & 0.689{\tiny{$\pm$0.060}} \\
Glass2       & \textbf{0.622{\tiny{$\pm$0.110}}} & 0.585{\tiny{$\pm$0.131}} & 0.620{\tiny{$\pm$0.166}} & 0.620{\tiny{$\pm$0.082}} & 0.586{\tiny{$\pm$0.121}} & 0.614{\tiny{$\pm$0.196}} & 0.601{\tiny{$\pm$0.138}} & 0.573{\tiny{$\pm$0.163}} \\
Glass6       & 0.915{\tiny{$\pm$0.077}} & 0.884{\tiny{$\pm$0.085}} & 0.925{\tiny{$\pm$0.063}} & 0.939{\tiny{$\pm$0.057}} & \textbf{0.962{\tiny{$\pm$0.025}}} & 0.875{\tiny{$\pm$0.102}} & 0.939{\tiny{$\pm$0.046}} & 0.827{\tiny{$\pm$0.150}} \\
Haberman     & 0.632{\tiny{$\pm$0.099}} & 0.548{\tiny{$\pm$0.116}} & 0.601{\tiny{$\pm$0.129}} & 0.662{\tiny{$\pm$0.091}} & 0.633{\tiny{$\pm$0.110}} & 0.586{\tiny{$\pm$0.131}} & \textbf{0.666{\tiny{$\pm$0.130}}} & 0.629{\tiny{$\pm$0.106}} \\
Iris0        & 0.993{\tiny{$\pm$0.011}} & 0.894{\tiny{$\pm$0.286}} & 0.997{\tiny{$\pm$0.009}} & 0.897{\tiny{$\pm$0.217}} & 0.950{\tiny{$\pm$0.158}} & 0.957{\tiny{$\pm$0.135}} & 0.998{\tiny{$\pm$0.004}} & \textbf{1.000{\tiny{$\pm$0.001}}} \\
New-Thyroid1 & 0.930{\tiny{$\pm$0.047}} & 0.802{\tiny{$\pm$0.223}} & 0.951{\tiny{$\pm$0.044}} & 0.962{\tiny{$\pm$0.035}} & 0.895{\tiny{$\pm$0.160}} & \textbf{0.973{\tiny{$\pm$0.065}}} & 0.888{\tiny{$\pm$0.300}} & 0.910{\tiny{$\pm$0.152}} \\
Pima         & \textbf{0.758{\tiny{$\pm$0.015}}} & 0.722{\tiny{$\pm$0.062}} & 0.756{\tiny{$\pm$0.024}} & 0.753{\tiny{$\pm$0.032}} & 0.742{\tiny{$\pm$0.048}} & 0.757{\tiny{$\pm$0.026}} & 0.741{\tiny{$\pm$0.027}} & 0.745{\tiny{$\pm$0.027}} \\
Vehicle2     & \textbf{0.949{\tiny{$\pm$0.024}}} & 0.901{\tiny{$\pm$0.052}} & 0.948{\tiny{$\pm$0.021}} & 0.936{\tiny{$\pm$0.026}} & 0.928{\tiny{$\pm$0.029}} & 0.932{\tiny{$\pm$0.038}} & 0.864{\tiny{$\pm$0.049}} & 0.892{\tiny{$\pm$0.057}} \\
Vowel0       & 0.966{\tiny{$\pm$0.033}} & 0.929{\tiny{$\pm$0.054}} & \textbf{0.978{\tiny{$\pm$0.019}}} & 0.975{\tiny{$\pm$0.018}} & 0.966{\tiny{$\pm$0.015}} & 0.956{\tiny{$\pm$0.020}} & 0.847{\tiny{$\pm$0.157}} & 0.838{\tiny{$\pm$0.079}} \\
Wisconsin    & 0.995{\tiny{$\pm$0.003}} & 0.912{\tiny{$\pm$0.214}} & \textbf{0.998{\tiny{$\pm$0.002}}} & 0.997{\tiny{$\pm$0.002}} & 0.995{\tiny{$\pm$0.006}} & 0.996{\tiny{$\pm$0.004}} & 0.993{\tiny{$\pm$0.014}} & 0.997{\tiny{$\pm$0.002}} \\
Yeast3       & \textbf{0.879{\tiny{$\pm$0.032}}} & 0.657{\tiny{$\pm$0.140}} & 0.861{\tiny{$\pm$0.070}} & 0.853{\tiny{$\pm$0.063}} & 0.846{\tiny{$\pm$0.091}} & 0.850{\tiny{$\pm$0.038}} & 0.793{\tiny{$\pm$0.104}} & 0.752{\tiny{$\pm$0.133}} \\
\bottomrule
\end{tabular}%
}
\end{table}

\subsection{CIFAR-10N: Label-Noise Robustness}\label{sec:results:cifar10n}

\Cref{tab:results:cifar10n} shows AUC under three noise levels on CIFAR-10N. Under \emph{clean} labels all methods reach AUC $\geq 0.972$, with \textsc{Roll}-AoC at the top (0.980). As noise increases the advantage of \textsc{Roll}-AoC grows: under \emph{aggre} noise it leads by 0.002 over the next-best method (GCE, 0.932) while other methods degrade more significantly, and under \emph{worse} noise, the margin widens to 0.012 over MAE (0.809) and 0.047 over BCE-W (0.738). Observing degredation w.r.t the clean scenario, we can see that \testxc{Roll}-AoC degraded the least (-0.159 relative to clean results) relative to other configurations, with the other least-degrading result being MAE (-0.167)

\begin{table}[htbp]
\centering
\caption{Mean test AUC ($\pm$\,std, $n=3$ episodes) on CIFAR-10N under three label-noise levels. Best per row in \textbf{bold}.}
\label{tab:results:cifar10n}
\resizebox{\textwidth}{!}{%
\begin{tabular}{lccccccc}
\toprule
Noise & \textsc{Roll}-AoC & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\
\midrule
Clean & \textbf{0.980{\tiny{$\pm$0.001}}} & 0.977{\tiny{$\pm$0.002}} & 0.977{\tiny{$\pm$0.000}} & 0.972{\tiny{$\pm$0.003}} & 0.976{\tiny{$\pm$0.002}} & 0.979{\tiny{$\pm$0.000}} & 0.976{\tiny{$\pm$0.002}} \\
Aggre. & \textbf{0.934{\tiny{$\pm$0.002}}} & 0.924{\tiny{$\pm$0.003}} & 0.912{\tiny{$\pm$0.014}} & 0.910{\tiny{$\pm$0.004}} & 0.932{\tiny{$\pm$0.005}} & 0.921{\tiny{$\pm$0.004}} & 0.930{\tiny{$\pm$0.002}} \\
Worse & \textbf{0.821{\tiny{$\pm$0.001}}} & 0.738{\tiny{$\pm$0.009}} & 0.734{\tiny{$\pm$0.009}} & 0.720{\tiny{$\pm$0.004}} & 0.774{\tiny{$\pm$0.005}} & 0.752{\tiny{$\pm$0.006}} & 0.809{\tiny{$\pm$0.004}} \\
\bottomrule
\end{tabular}%
}
\end{table}

% TODO: Remove Roll-TPR column from tab:results:large and update the discussion accordingly.
% Roll-TPR is not the focus for these datasets and its results (e.g. 0.438 on Bank Marketing) distract from the main story.
\subsection{Large Real-World Datasets}\label{sec:results:large}

% TODO: Reword Bank Marketing discussion after rerun — current results (Roll-AoC 0.511, Roll-TPR 0.438) are likely a bug. Update numbers and narrative once confirmed.
\Cref{tab:results:large} covers three large-scale datasets. On Credit Card Fraud \textsc{Roll}-TPR is the top performer (0.974) with \textsc{Roll}-AoC close behind (0.968), and both outperform all non-ROLL baselines. On Home Credit \textsc{Roll}-AoC edges LibAUC (0.712 vs.\ 0.711). On Bank Marketing, however, GCE achieves the highest AUC (0.746) while \textsc{Roll}-AoC and \textsc{Roll}-TPR underperform (0.511 and 0.438 respectively); the cause of this degradation on this dataset is an open question and a direction for future investigation.

\begin{table}[htbp]
\centering
\caption{Mean test AUC ($\pm$\,std, $n=5$ episodes) on large real-world datasets. Best per row in \textbf{bold}.}
\label{tab:results:large}
\resizebox{\textwidth}{!}{%
\begin{tabular}{lcccccccc}
\toprule
Dataset & \textsc{Roll}-AoC & \textsc{Roll}-TPR & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\
\midrule
Bank Marketing & 0.511{\tiny{$\pm$0.047}} & 0.438{\tiny{$\pm$0.044}} & 0.580{\tiny{$\pm$0.040}} & 0.573{\tiny{$\pm$0.044}} & 0.684{\tiny{$\pm$0.033}} & \textbf{0.746{\tiny{$\pm$0.022}}} & 0.652{\tiny{$\pm$0.031}} & 0.707{\tiny{$\pm$0.035}} \\
Credit Card Fraud & 0.968{\tiny{$\pm$0.001}} & \textbf{0.974{\tiny{$\pm$0.003}}} & 0.949{\tiny{$\pm$0.015}} & 0.942{\tiny{$\pm$0.020}} & 0.940{\tiny{$\pm$0.013}} & 0.948{\tiny{$\pm$0.014}} & 0.966{\tiny{$\pm$0.006}} & 0.954{\tiny{$\pm$0.009}} \\
Home Credit & \textbf{0.712{\tiny{$\pm$0.001}}} & 0.687{\tiny{$\pm$0.004}} & 0.693{\tiny{$\pm$0.002}} & 0.694{\tiny{$\pm$0.002}} & 0.688{\tiny{$\pm$0.003}} & 0.698{\tiny{$\pm$0.002}} & 0.711{\tiny{$\pm$0.001}} & 0.699{\tiny{$\pm$0.004}} \\
\bottomrule
\end{tabular}%
}
\end{table}


\subsection{NIH ChestX-ray14}\label{sec:results:chestxray14}

\Cref{tab:results:chestxray14} reports test AUC on the ChestX-ray14 Effusion task.
\textsc{Roll}-AoC achieves the highest AUC ($0.839\pm0.001$), a clear margin above the
next-best methods GCE ($0.829\pm0.004$) and \textsc{Roll}-TPR ($0.828\pm0.002$).
Standard cross-entropy variants trail substantially: BCE-W reaches $0.788\pm0.008$,
Focal $0.798\pm0.009$, and Asym.\ $0.729\pm0.007$. MAE ($0.810\pm0.001$) benefits from
its noise-robust design but still falls behind the distributional losses. LibAUC results
are excluded due to an optimizer instability on this dataset (to be resolved in a
follow-up run).

The ${\approx}5$~pp gap between \textsc{Roll}-AoC and BCE-W, on a dataset whose labels
carry substantial organic NLP noise~\cite{oakdenrayner2020exploring}, is consistent with the findings from the controlled
KEEL noise experiment (\Cref{sec:results:keel}) and CIFAR-10N
(\Cref{sec:results:cifar10n}). The result also demonstrates that ROLL-AOC generalises
beyond from-scratch training: the DenseNet-121 embeddings are frozen, so ROLL operates
entirely within the MLP head, fitting score distributions over a fixed representation.

\begin{table}[htbp]
\centering
\caption{Mean test AUC ($\pm$\,std, $n=5$ episodes) on NIH ChestX-ray14 (Effusion
  vs.\ rest, IR~${\approx}7.5$:1). Labels carry substantial organic NLP noise~\cite{oakdenrayner2020exploring}.
  LibAUC excluded due to optimizer instability. Best in \textbf{bold}.}
\label{tab:results:chestxray14}
\begin{tabular}{lc}
\toprule
Method & AUC \\
\midrule
\textsc{Roll}-AoC  & \textbf{0.839{\tiny{$\pm$0.001}}} \\
\textsc{Roll}-TPR  & 0.828{\tiny{$\pm$0.002}} \\
GCE                & 0.829{\tiny{$\pm$0.004}} \\
MAE                & 0.810{\tiny{$\pm$0.001}} \\
Focal              & 0.798{\tiny{$\pm$0.009}} \\
BCE-W              & 0.788{\tiny{$\pm$0.008}} \\
Asym.              & 0.729{\tiny{$\pm$0.007}} \\
LibAUC             & --- \\
\bottomrule
\end{tabular}
\end{table}



\section{Ablation Study}\label{sec:experiments:ablation}
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{ניתוח אבלציה}}

% Isolate the contribution of ROLL's individual design choices to determine which
% components drive its performance and noise robustness.

\subsection{Effect of KDE Smoothing}\label{sec:experiments:ablation:kde}

% Compare ROLL-AoC against a hard-threshold variant (no KDE; threshold estimated
% by sorting scores and selecting the rank corresponding to FPR $= \alpha$).
% Expected finding: without smooth KDE the gradient signal collapses and training
% is unstable, confirming that differentiability is not just a theoretical convenience.

\subsection{Effect of Kernel Choice}\label{sec:experiments:ablation:kernel}

% Compare Gaussian vs.\ sigmoid kernel in ROLL-AoC on representative datasets.
% The sigmoid kernel admits closed-form gradients (see \Cref{appendix:kde-sigmoid});
% this subsection assesses whether kernel choice materially affects accuracy or
% whether performance is robust to kernel selection.

\subsection{ROLL-AoC vs.\ ROLL-TPR}\label{sec:experiments:ablation:variants}

% Systematically compare the two ROLL variants across all benchmarks.  Discuss when
% targeting a single operating point (ROLL-TPR) helps (e.g.\ Credit Card Fraud, where
% a strict FPR budget is natural) vs.\ when whole-curve optimization (ROLL-AoC) is
% preferable (heterogeneous or unconstrained evaluation).


\section{Sensitivity Analysis}\label{sec:experiments:sensitivity}
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{ניתוח רגישות}}

% Characterize ROLL's robustness to its key hyperparameters so practitioners can
% apply it without exhaustive tuning.

\subsection{Bandwidth Sensitivity}\label{sec:experiments:sensitivity:bandwidth}

% Plot AUC vs.\ log-bandwidth $h$ for ROLL-AoC across a representative subset of
% datasets (at least one from KEEL, CIFAR-10N, and large-scale).  Report the range
% of $h$ over which performance stays within 1\% of its peak --- this defines a
% practical safe tuning zone.  Show that Silverman's rule falls within this zone,
% justifying it as a default.

\subsection{Operating-Point Sensitivity (ROLL-TPR)}\label{sec:experiments:sensitivity:alpha}

% For ROLL-TPR, sweep the target FPR budget $\alpha$ and measure TPR achieved at
% each setting.  Validate that the trained model's operating point tracks the
% specified $\alpha$, demonstrating that ROLL-TPR reliably targets the operating
% point it was trained for and generalises to nearby values.


\section{Summary}\label{sec:experiments:summary}
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{סיכום}}

% TODO: write 2--3 paragraphs synthesising findings.
% (1) Noise robustness is the primary result: ROLL-AoC is best at 20\% KEEL noise
%     and best at all three CIFAR-10N noise levels. The story is consistent across
%     very different settings (small tabular vs. image).
% (2) Large-dataset picture: strong on Credit Card Fraud and Home Credit; clear
%     failure on Bank Marketing (0.511 vs 0.746 for GCE) -- flag as open question.
% (3) ROLL-TPR is inconsistent: best on Credit Card Fraud (0.974), poor elsewhere.
%     Its value is as a targeted tool when a specific FPR budget is fixed.
% Bridge: Discussion chapter \Cref{chapter:discussion_and_conclusions} addresses
%     the Bank Marketing failure and the mechanism behind noise robustness.

TODO write this section.