\section{Data Sets}\label{sec:experiments:data_sets}
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{מערך נתונים}}

\paragraph{KEEL benchmark.}
We evaluate on 12 binary imbalanced datasets from the KEEL repository~\cite{TODO} (Knowledge Extraction based on Evolutionary Learning). All labels are clean. \Cref{tab:datasets:keel} summarises the datasets; imbalance ratios range from 1.8 to 11.6, covering mild to moderate imbalance regimes commonly used in the literature.

\begin{table}[htbp]
\centering
\caption{KEEL benchmark datasets used in experiments. IR = negative/positive count ratio.}
\label{tab:datasets:keel}
\small
\begin{tabular}{lrrrrl}
\toprule
Dataset & $n$ & Pos & Neg & IR & Source \\
\midrule
Glass0       &  214 &  70 & 144 &  2.1 & Glass identification (UCI) \\
Glass1       &  214 &  76 & 138 &  1.8 & Glass identification (UCI) \\
Glass2       &  214 &  17 & 197 & 11.6 & Glass identification (UCI) \\
Glass6       &  214 &  29 & 185 &  6.4 & Glass identification (UCI) \\
Haberman     &  306 &  81 & 225 &  2.8 & Breast cancer survival, 1958--1970 \\
Iris0        &  150 &  50 & 100 &  2.0 & \emph{Iris setosa} vs.\ others \\
New-Thyroid1 &  215 &  35 & 180 &  5.1 & Thyroid function classification \\
Pima         &  768 & 268 & 500 &  1.9 & Pima Indian diabetes (UCI) \\
Vehicle2     &  846 & 218 & 628 &  2.9 & Vehicle silhouette (UCI) \\
Vowel0       &  988 &  90 & 898 & 10.0 & Vowel recognition, class 0 vs.\ rest \\
Wisconsin    &  683 & 239 & 444 &  1.9 & Wisconsin breast cancer (UCI) \\
Yeast3       & 1484 & 163 &1321 &  8.1 & Yeast protein localisation (UCI) \\
\bottomrule
\end{tabular}
\end{table}

\paragraph{CIFAR-10N.}
CIFAR-10N~\cite{TODO} augments the 50{,}000-image CIFAR-10 training set ($32{\times}32$ RGB) with real human-annotated noisy labels collected from Amazon Mechanical Turk~\cite{TODO}. We form a binary task (class~1, automobile, vs.\ rest; IR~$\approx 9$:1) and train under three noise conditions:
\emph{clean} (original CIFAR-10 labels, 0\% noise),
\emph{aggre} (majority-vote aggregate of crowdworker annotations, $\approx$9\% noise), and
\emph{worse} (single worst-annotator labels, $\approx$40\% noise).
Architecture: a three-block convolutional network trained from scratch.

\paragraph{Large tabular datasets.}
We include three large-scale tabular datasets spanning diverse imbalanced real-world settings. All labels are considered clean (no injected or crowd-sourced noise).

\begin{itemize}
  \item \textbf{Bank Marketing}~\cite{TODO} --- UCI repository; Portuguese bank telephone marketing campaigns (Moro et al., 2014). $n = 41{,}188$ samples, 48 features after one-hot encoding. Positive class: client subscribed to term deposit. IR~$\approx 8$:1 ($\approx$11.3\% positive). No label noise.

  \item \textbf{Credit Card Fraud}~\cite{TODO} --- Kaggle / ULB (Worldline + MLG, 2013). $n = 284{,}807$ transactions, 30 features (V1--V28 are PCA-transformed; Time and Amount rescaled). Extreme imbalance: 492 confirmed frauds, IR~$\approx 577$:1. No label noise.

  \item \textbf{Home Credit}~\cite{TODO} --- Kaggle Home Credit Default Risk competition (2018). $n = 307{,}511$ loan applications, $\approx$120 numerical features from the primary application table. Positive class: loan default. IR~$\approx 11.4$:1 ($\approx$8\% default rate). Labels are proxy labels (repayment outcome); no injected noise.
\end{itemize}

\section{Performance Measures}\label{sec:experiments:performance_measures}
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{מדדי ביצועים}}

We report the \emph{area under the ROC curve} (AUC) averaged over independent training episodes (see \Cref{sec:experiments:configurations}). AUC measures overall ranking quality and is a standard metric for imbalanced classification. All results are presented as mean $\pm$ standard deviation over episodes.

\section{Configurations and Parameter Settings}
\label{sec:experiments:configurations}
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{קונפיגורציה ופרמטרים}}

We compare \textsc{Roll}-AoC (the primary proposed loss, optimising the area under the ROC curve via KDE) and \textsc{Roll}-TPR (a variant targeting TPR at a fixed operating point) against the following baselines:

\begin{itemize}
  \item \textbf{BCE-W} --- binary cross-entropy with class-frequency inverse weighting.
  \item \textbf{Focal} --- focal loss~\cite{TODO} with focusing parameter $\gamma = 2$.
  \item \textbf{Asym.} --- asymmetric loss~\cite{TODO}.
  \item \textbf{GCE} --- generalised cross-entropy~\cite{TODO} with $q = 0.7$.
  \item \textbf{LibAUC} --- PESG-based AUC maximisation from the LibAUC library~\cite{TODO}.
  \item \textbf{MAE} --- mean absolute error (symmetric noise-robust loss).
\end{itemize}

All methods use the same network architecture, optimiser (Adam for non-LibAUC methods), and early stopping. Each configuration is run for $n = 5$ independent episodes on KEEL and large datasets, and $n = 3$ on CIFAR-10N. Reported AUC is evaluated on the held-out test split.

\section{Results}\label{sec:experiments:results}
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{תוצאות}}

\subsection{KEEL Benchmark}\label{sec:results:keel}

\Cref{tab:results:keel} reports mean test AUC across the 12 KEEL datasets. No single method dominates the benchmark: on easy datasets (Iris0, Wisconsin, Vowel0, Vehicle2, Glass6) nearly all methods converge to near-perfect AUC, leaving little room for differentiation. On the harder, more imbalanced datasets the picture is more varied. \textsc{Roll}-AoC competes well but does not claim the top position on most individual datasets; this is consistent with no-free-lunch intuitions for heterogeneous benchmarks.

\begin{table}[htbp]
\centering
\caption{Mean test AUC ($\pm$\,std, $n=5$ episodes) on KEEL benchmark datasets. Best per row in \textbf{bold}.}
\label{tab:results:keel}
\resizebox{\textwidth}{!}{%
\begin{tabular}{lcccccccc}
\toprule
Dataset & \textsc{Roll}-AoC & \textsc{Roll}-TPR & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\
\midrule
Glass0 & 0.796{\tiny{$\pm$0.046}} & 0.826{\tiny{$\pm$0.032}} & 0.839{\tiny{$\pm$0.033}} & 0.802{\tiny{$\pm$0.020}} & 0.814{\tiny{$\pm$0.034}} & 0.818{\tiny{$\pm$0.032}} & \textbf{0.843{\tiny{$\pm$0.035}}} & 0.826{\tiny{$\pm$0.037}} \\
Glass1 & 0.722{\tiny{$\pm$0.033}} & 0.698{\tiny{$\pm$0.042}} & 0.710{\tiny{$\pm$0.018}} & 0.712{\tiny{$\pm$0.036}} & 0.744{\tiny{$\pm$0.044}} & 0.742{\tiny{$\pm$0.039}} & 0.723{\tiny{$\pm$0.054}} & \textbf{0.769{\tiny{$\pm$0.031}}} \\
Glass2 & 0.723{\tiny{$\pm$0.062}} & 0.701{\tiny{$\pm$0.055}} & 0.661{\tiny{$\pm$0.069}} & 0.677{\tiny{$\pm$0.072}} & \textbf{0.750{\tiny{$\pm$0.075}}} & 0.689{\tiny{$\pm$0.023}} & 0.719{\tiny{$\pm$0.084}} & 0.679{\tiny{$\pm$0.018}} \\
Glass6 & 0.975{\tiny{$\pm$0.014}} & 0.975{\tiny{$\pm$0.027}} & 0.972{\tiny{$\pm$0.025}} & 0.974{\tiny{$\pm$0.020}} & 0.982{\tiny{$\pm$0.013}} & 0.971{\tiny{$\pm$0.020}} & 0.969{\tiny{$\pm$0.012}} & \textbf{0.985{\tiny{$\pm$0.006}}} \\
Haberman & 0.696{\tiny{$\pm$0.045}} & 0.675{\tiny{$\pm$0.063}} & 0.736{\tiny{$\pm$0.012}} & \textbf{0.742{\tiny{$\pm$0.025}}} & 0.741{\tiny{$\pm$0.018}} & 0.737{\tiny{$\pm$0.016}} & 0.616{\tiny{$\pm$0.116}} & 0.671{\tiny{$\pm$0.102}} \\
Iris0 & \textbf{1.000{\tiny{$\pm$0.000}}} & 0.981{\tiny{$\pm$0.027}} & 0.900{\tiny{$\pm$0.224}} & \textbf{1.000{\tiny{$\pm$0.000}}} & \textbf{1.000{\tiny{$\pm$0.000}}} & \textbf{1.000{\tiny{$\pm$0.000}}} & \textbf{1.000{\tiny{$\pm$0.000}}} & 0.900{\tiny{$\pm$0.224}} \\
New-Thyroid1 & 0.989{\tiny{$\pm$0.010}} & 0.845{\tiny{$\pm$0.241}} & 0.992{\tiny{$\pm$0.005}} & 0.977{\tiny{$\pm$0.033}} & 0.988{\tiny{$\pm$0.007}} & 0.987{\tiny{$\pm$0.011}} & 0.985{\tiny{$\pm$0.012}} & \textbf{0.994{\tiny{$\pm$0.003}}} \\
Pima & 0.778{\tiny{$\pm$0.024}} & 0.755{\tiny{$\pm$0.014}} & \textbf{0.787{\tiny{$\pm$0.014}}} & 0.775{\tiny{$\pm$0.020}} & 0.771{\tiny{$\pm$0.019}} & 0.785{\tiny{$\pm$0.005}} & 0.761{\tiny{$\pm$0.015}} & 0.778{\tiny{$\pm$0.005}} \\
Vehicle2 & 0.982{\tiny{$\pm$0.003}} & 0.966{\tiny{$\pm$0.020}} & \textbf{0.984{\tiny{$\pm$0.004}}} & 0.982{\tiny{$\pm$0.003}} & 0.980{\tiny{$\pm$0.004}} & 0.982{\tiny{$\pm$0.004}} & 0.982{\tiny{$\pm$0.003}} & 0.983{\tiny{$\pm$0.004}} \\
Vowel0 & 0.993{\tiny{$\pm$0.005}} & 0.987{\tiny{$\pm$0.002}} & 0.995{\tiny{$\pm$0.004}} & 0.996{\tiny{$\pm$0.003}} & 0.995{\tiny{$\pm$0.003}} & 0.997{\tiny{$\pm$0.003}} & 0.991{\tiny{$\pm$0.007}} & \textbf{0.998{\tiny{$\pm$0.001}}} \\
Wisconsin & 0.997{\tiny{$\pm$0.002}} & 0.800{\tiny{$\pm$0.441}} & \textbf{0.999{\tiny{$\pm$0.000}}} & \textbf{0.999{\tiny{$\pm$0.001}}} & 0.999{\tiny{$\pm$0.001}} & 0.999{\tiny{$\pm$0.001}} & 0.999{\tiny{$\pm$0.000}} & 0.998{\tiny{$\pm$0.001}} \\
Yeast3 & 0.906{\tiny{$\pm$0.024}} & 0.908{\tiny{$\pm$0.021}} & 0.894{\tiny{$\pm$0.008}} & 0.908{\tiny{$\pm$0.019}} & \textbf{0.935{\tiny{$\pm$0.014}}} & 0.927{\tiny{$\pm$0.006}} & \textbf{0.935{\tiny{$\pm$0.016}}} & 0.915{\tiny{$\pm$0.010}} \\
\bottomrule
\end{tabular}%
}
\end{table}

\subsection{CIFAR-10N: Label-Noise Robustness}\label{sec:results:cifar10n}

\Cref{tab:results:cifar10n} shows AUC under three noise levels on CIFAR-10N. Under \emph{clean} labels all methods reach AUC $\geq 0.972$, with \textsc{Roll}-AoC at the top (0.980). As noise increases the advantage of \textsc{Roll}-AoC grows: under \emph{aggre} noise it leads by 0.2~pp over the next-best method (GCE, 0.932), and under \emph{worse} noise the margin widens to 1.2~pp over MAE (0.809) and 4.7~pp over BCE-W (0.738). This suggests that the KDE-smoothed loss surface provides implicit robustness to label noise.

\begin{table}[htbp]
\centering
\caption{Mean test AUC ($\pm$\,std, $n=3$ episodes) on CIFAR-10N under three label-noise levels. Best per row in \textbf{bold}.}
\label{tab:results:cifar10n}
\resizebox{\textwidth}{!}{%
\begin{tabular}{lccccccc}
\toprule
Noise & \textsc{Roll}-AoC & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\
\midrule
Clean & \textbf{0.980{\tiny{$\pm$0.001}}} & 0.977{\tiny{$\pm$0.002}} & 0.977{\tiny{$\pm$0.000}} & 0.972{\tiny{$\pm$0.003}} & 0.976{\tiny{$\pm$0.002}} & 0.979{\tiny{$\pm$0.000}} & 0.976{\tiny{$\pm$0.002}} \\
Aggre. & \textbf{0.934{\tiny{$\pm$0.002}}} & 0.924{\tiny{$\pm$0.003}} & 0.912{\tiny{$\pm$0.014}} & 0.910{\tiny{$\pm$0.004}} & 0.932{\tiny{$\pm$0.005}} & 0.921{\tiny{$\pm$0.004}} & 0.930{\tiny{$\pm$0.002}} \\
Worse & \textbf{0.821{\tiny{$\pm$0.001}}} & 0.738{\tiny{$\pm$0.009}} & 0.734{\tiny{$\pm$0.009}} & 0.720{\tiny{$\pm$0.004}} & 0.774{\tiny{$\pm$0.005}} & 0.752{\tiny{$\pm$0.006}} & 0.809{\tiny{$\pm$0.004}} \\
\bottomrule
\end{tabular}%
}
\end{table}

\subsection{Large Real-World Datasets}\label{sec:results:large}

\Cref{tab:results:large} covers three large-scale datasets. On Credit Card Fraud \textsc{Roll}-TPR is the top performer (0.974) with \textsc{Roll}-AoC close behind (0.968), and both outperform all non-ROLL baselines. On Home Credit \textsc{Roll}-AoC edges LibAUC (0.712 vs.\ 0.711). On Bank Marketing, however, GCE achieves the highest AUC (0.746) while \textsc{Roll}-AoC and \textsc{Roll}-TPR underperform (0.511 and 0.438 respectively); the cause of this degradation on this dataset is an open question and a direction for future investigation.

\begin{table}[htbp]
\centering
\caption{Mean test AUC ($\pm$\,std, $n=5$ episodes) on large real-world datasets. Best per row in \textbf{bold}.}
\label{tab:results:large}
\resizebox{\textwidth}{!}{%
\begin{tabular}{lcccccccc}
\toprule
Dataset & \textsc{Roll}-AoC & \textsc{Roll}-TPR & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\
\midrule
Bank Marketing & 0.511{\tiny{$\pm$0.047}} & 0.438{\tiny{$\pm$0.044}} & 0.580{\tiny{$\pm$0.040}} & 0.573{\tiny{$\pm$0.044}} & 0.684{\tiny{$\pm$0.033}} & \textbf{0.746{\tiny{$\pm$0.022}}} & 0.652{\tiny{$\pm$0.031}} & 0.707{\tiny{$\pm$0.035}} \\
Credit Card Fraud & 0.968{\tiny{$\pm$0.001}} & \textbf{0.974{\tiny{$\pm$0.003}}} & 0.949{\tiny{$\pm$0.015}} & 0.942{\tiny{$\pm$0.020}} & 0.940{\tiny{$\pm$0.013}} & 0.948{\tiny{$\pm$0.014}} & 0.966{\tiny{$\pm$0.006}} & 0.954{\tiny{$\pm$0.009}} \\
Home Credit & \textbf{0.712{\tiny{$\pm$0.001}}} & 0.687{\tiny{$\pm$0.004}} & 0.693{\tiny{$\pm$0.002}} & 0.694{\tiny{$\pm$0.002}} & 0.688{\tiny{$\pm$0.003}} & 0.698{\tiny{$\pm$0.002}} & 0.711{\tiny{$\pm$0.001}} & 0.699{\tiny{$\pm$0.004}} \\
\bottomrule
\end{tabular}%
}
\end{table}


\section{Label-Noise Poisoning Experiment}\label{sec:experiments:poisoning}
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{ניסוי הרעלת תוויות}}

% A controlled experiment that isolates ROLL's label-noise robustness from confounders
% (dataset size, imbalance ratio).  Protocol: start from a clean dataset, inject
% \emph{asymmetric} noise --- flip a fraction of positive labels to negative, leave
% negative labels intact --- and track AUC degradation as the flip rate increases.
% This mirrors a worst-case practical scenario: rare true positives become false negatives,
% compounding the imbalance.  Empirical complement to the structural argument in
% \Cref{sec:noisy-labels}.

\subsection{Protocol}\label{sec:experiments:poisoning:protocol}

% Specify: which dataset(s) were poisoned (one or two from KEEL or large-scale, chosen
% for mid-range difficulty where differences are visible), the range of flip rates tested
% (e.g.\ 0\% to 40\% in steps of 10\%), and number of independent episodes per rate.
% Justify asymmetric noise as the harder, more realistic case compared to symmetric flipping.

\subsection{Results}\label{sec:experiments:poisoning:results}

% Present AUC vs.\ noise-rate curves for all methods.  Key finding: ROLL-AoC degrades
% more gracefully than pointwise losses (BCE-W, Focal, Asym.) because a distributional
% objective spreads the influence of each mislabeled sample over the estimated density
% rather than producing a single large gradient.  Cite structural explanation from
% \Cref{sec:noisy-labels}; note consistency with CIFAR-10N results in
% \Cref{sec:results:cifar10n}.


\section{Ablation Study}\label{sec:experiments:ablation}
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{ניתוח אבלציה}}

% Isolate the contribution of ROLL's individual design choices to determine which
% components drive its performance and noise robustness.

\subsection{Effect of KDE Smoothing}\label{sec:experiments:ablation:kde}

% Compare ROLL-AoC against a hard-threshold variant (no KDE; threshold estimated
% by sorting scores and selecting the rank corresponding to FPR $= \alpha$).
% Expected finding: without smooth KDE the gradient signal collapses and training
% is unstable, confirming that differentiability is not just a theoretical convenience.

\subsection{Effect of Kernel Choice}\label{sec:experiments:ablation:kernel}

% Compare Gaussian vs.\ sigmoid kernel in ROLL-AoC on representative datasets.
% The sigmoid kernel admits closed-form gradients (see \Cref{appendix:kde-sigmoid});
% this subsection assesses whether kernel choice materially affects accuracy or
% whether performance is robust to kernel selection.

\subsection{ROLL-AoC vs.\ ROLL-TPR}\label{sec:experiments:ablation:variants}

% Systematically compare the two ROLL variants across all benchmarks.  Discuss when
% targeting a single operating point (ROLL-TPR) helps (e.g.\ Credit Card Fraud, where
% a strict FPR budget is natural) vs.\ when whole-curve optimization (ROLL-AoC) is
% preferable (heterogeneous or unconstrained evaluation).


\section{Sensitivity Analysis}\label{sec:experiments:sensitivity}
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{ניתוח רגישות}}

% Characterize ROLL's robustness to its key hyperparameters so practitioners can
% apply it without exhaustive tuning.

\subsection{Bandwidth Sensitivity}\label{sec:experiments:sensitivity:bandwidth}

% Plot AUC vs.\ log-bandwidth $h$ for ROLL-AoC across a representative subset of
% datasets (at least one from KEEL, CIFAR-10N, and large-scale).  Report the range
% of $h$ over which performance stays within 1\% of its peak --- this defines a
% practical safe tuning zone.  Show that Silverman's rule falls within this zone,
% justifying it as a default.

\subsection{Operating-Point Sensitivity (ROLL-TPR)}\label{sec:experiments:sensitivity:alpha}

% For ROLL-TPR, sweep the target FPR budget $\alpha$ and measure TPR achieved at
% each setting.  Validate that the trained model's operating point tracks the
% specified $\alpha$, demonstrating that ROLL-TPR reliably targets the operating
% point it was trained for and generalises to nearby values.


\section{Summary}\label{sec:experiments:summary}
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{סיכום}}

% Synthesise the empirical findings in 2--3 paragraphs.
% (1) Where ROLL-AoC wins clearly: CIFAR-10N noise robustness, Credit Card Fraud,
%     Home Credit --- relate to the distributional objective and KDE smoothing.
% (2) Where it underperforms: Bank Marketing --- flag as an open question; discuss
%     possible causes (feature scale, label proxy quality, optimizer interaction).
% (3) What the ablation reveals: which components are load-bearing vs.\ incidental.
% Close with a sentence bridging to \Cref{chapter:discussion_and_conclusions}.

TODO write this section.