aner/thesis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Results overhaul

Browse Source
This commit is contained in:
Aner Zakobar
2026-08-29 06:54:06 +03:00
parent 4bf2ec9b97
commit 475e992bff
5 changed files with 730 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bibliography/thesis.bib
+98
View File
@@ -187,3 +187,101 @@
year={1933},
publisher={The Royal Society}
}
@inproceedings{wang2017chestxray14,
title={{ChestX-ray8}: Hospital-Scale Chest {X-Ray} Database and Benchmarks on Weakly-Supervised Classification and Localization of Common Thorax Diseases},
author={Wang, Xiaosong and Peng, Yifan and Lu, Le and Lu, Zhiyong and Bagheri, Mohammadhadi and Summers, Ronald M.},
booktitle={Proceedings of the {IEEE} Conference on Computer Vision and Pattern Recognition ({CVPR})},
pages={2097--2106},
year={2017},
doi={10.1109/CVPR.2017.369}
}
@article{oakdenrayner2020exploring,
title={Exploring Large-scale Public Medical Image Datasets},
author={Oakden-Rayner, Lauren},
journal={Academic Radiology},
volume={27},
number={1},
pages={106--112},
year={2020},
doi={10.1016/j.acra.2019.10.006},
publisher={Elsevier}
}
@article{alcalafdez2011keel,
title={{KEEL} Data-Mining Software Tool: Data Set Repository, Integration of Algorithms and Experimental Analysis Framework},
author={Alcal{\'a}-Fdez, J. and Fern{\'a}ndez, A. and Luengo, J. and Derrac, J. and Garc{\'i}a, S. and S{\'a}nchez, L. and Herrera, F.},
journal={Journal of Multiple-Valued Logic and Soft Computing},
volume={17},
number={2--3},
pages={255--287},
year={2011},
publisher={Old City Publishing}
}
@inproceedings{wei2022cifar10n,
title={Learning with Noisy Labels Revisited: A Study Using Real-World Human Annotations},
author={Wei, Jiaheng and Zhu, Zhaowei and Cheng, Hao and Liu, Tongliang and Niu, Gang and Liu, Yang},
booktitle={International Conference on Learning Representations ({ICLR})},
year={2022},
url={https://openreview.net/forum?id=TBWA6PLJZQm}
}
@article{moro2014bank,
title={A Data-Driven Approach to Predict the Success of Bank Telemarketing},
author={Moro, S{\'e}rgio and Cortez, Paulo and Rita, Paulo},
journal={Decision Support Systems},
volume={62},
pages={22--31},
year={2014},
doi={10.1016/j.dss.2014.03.001},
publisher={Elsevier}
}
@inproceedings{dalpozzolo2015calibrating,
title={Calibrating Probability with Undersampling for Unbalanced Classification},
author={Dal Pozzolo, Andrea and Caelen, Olivier and Johnson, Reid A. and Bontempi, Gianluca},
booktitle={2015 {IEEE} Symposium Series on Computational Intelligence ({SSCI})},
pages={159--166},
year={2015},
doi={10.1109/SSCI.2015.33},
organization={IEEE}
}
@misc{tittelfitz2023fpr,
title={{FPR} Estimation for Fraud Detection in the Presence of Class-Conditional Label Noise},
author={Tittelfitz, Justin},
year={2023},
eprint={2308.02695},
archivePrefix={arXiv},
primaryClass={stat.ML}
}
@misc{homecredit2018kaggle,
title={Home Credit Default Risk},
author={{Home Credit Group}},
year={2018},
howpublished={Kaggle Competition},
url={https://www.kaggle.com/c/home-credit-default-risk}
}
@inproceedings{cohen2022torchxrayvision,
title={{TorchXRayVision}: A Library of Chest {X-Ray} Datasets and Models},
author={Cohen, Joseph Paul and Viviano, Joseph D. and Bertin, Paul and Morrison, Paul and Torabian, Parsa and Guarrera, Matteo and Lungren, Matthew P. and Chaudhari, Akshay and Brooks, Rupert and Hashir, Mohammad and Bertrand, Hadrien},
booktitle={Proceedings of The 5th International Conference on Medical Imaging with Deep Learning ({MIDL})},
series={Proceedings of Machine Learning Research},
volume={172},
pages={231--249},
year={2022},
publisher={{PMLR}}
}
@inproceedings{lin2017focal,
title={Focal Loss for Dense Object Detection},
author={Lin, Tsung-Yi and Goyal, Priya and Girshick, Ross and He, Kaiming and Doll{\'a}r, Piotr},
booktitle={Proceedings of the {IEEE} International Conference on Computer Vision ({ICCV})},
pages={2999--3007},
year={2017},
doi={10.1109/ICCV.2017.324}
}
content/appendixs/appendix_roll_code.tex
+343
View File
@@ -0,0 +1,343 @@
This appendix contains the full PyTorch implementation of the ROLL loss
functions used in the experiments. The code is reproduced verbatim from
\texttt{src/roll.py} in the accompanying implementation repository, with
unrelated losses (baselines, Beta-ROLL) omitted.
\section{Gaussian ROLL}
The Gaussian variant fits a normal distribution to each class's scores and
evaluates TPR at the requested FPR analytically via the Gaussian CDF/ICDF.
\begin{lstlisting}[language=Python]
import torch
from functools import partial
import numpy as np
# -- helpers --
def _torch_mean_std(yh):
yhmean = torch.mean(yh)
yhstd = torch.std(yh)
return yhmean, yhstd
def _torch_normal_fit(yh):
yhmean, yhstd = _torch_mean_std(yh)
return torch.distributions.normal.Normal(yhmean, yhstd)
# -- batch splitting --
def split_true_false_indeces(yh, y):
true_indeces = torch.argwhere(y)[:, 0]
false_indeces = torch.argwhere(torch.logical_not(y))[:, 0]
return true_indeces, false_indeces
def split_true_false(yh, y):
true_indeces, false_indeces = split_true_false_indeces(yh, y)
return yh[true_indeces], yh[false_indeces]
# -- Gaussian ROLL --
def _roll_loss_from_fpr_internal(yh, y, fpr):
true_yh, false_yh = split_true_false(yh, y)
true_normal = _torch_normal_fit(true_yh)
false_normal = _torch_normal_fit(false_yh)
return true_normal.cdf(false_normal.icdf(torch.tensor(1 - fpr)))
def roll_loss_from_fpr(fpr):
return partial(_roll_loss_from_fpr_internal, fpr=fpr)
\end{lstlisting}
\section{Kernelized (KDE) ROLL}
The KDE variant replaces the parametric Gaussian with a sigmoid-kernel KDE,
enabling a custom autograd backward pass that directly implements the
analytic gradients derived in \Cref{appendix:kde-grad}.
\begin{lstlisting}[language=Python]
import torch
import logging
from functools import partial
from torch.autograd import Function
import numpy as np
from KDEpy.bw_selection import silvermans_rule, improved_sheather_jones
# -- bandwidth selection --
def _bandwidth(scores_1d, max_v=10000.0):
"""Return 1/h (kernel precision v), clipped to max_v.
Tries ISJ; falls back to Silverman on convergence failure
(common on small or near-constant batches early in training).
"""
data = scores_1d[:, np.newaxis]
try:
h = float(improved_sheather_jones(data))
except Exception:
h = float(silvermans_rule(data))
return min(1.0 / h, max_v)
# -- FPR grid for AOC objective --
# 0.01, 0.02, ..., 0.99 (99 values); sum / 100 approximates
# integral_0^1 (1 - TPR(alpha)) d alpha = AOC = 1 - AUC.
_AOC_FPRS = np.arange(0.01, 1.0, 0.01).tolist()
_AOC_ALPHAS = [1.0 - fpr for fpr in _AOC_FPRS]
_AOC_DIVISOR = 100
class KernelizedROLLoss(Function):
"""KDE-ROLL loss with custom backward pass.
Accepts a list of alpha values (CDF targets, alpha_k = 1 - FPR_k) and a
divisor, so one class covers both objectives:
point-FPR : alphas=[1-fpr], divisor=1
AOC : alphas=_AOC_ALPHAS, divisor=_AOC_DIVISOR
Notation:
v0, v1 -- KDE bandwidths for B_0 (negative) and B_1 (positive)
taus -- per-alpha thresholds from inverting F_0_hat
scores_pos -- positive-class model scores in the current batch
scores_neg -- negative-class model scores in the current batch
"""
# -- sigmoid kernel primitives --
@staticmethod
def _sigma(v, x):
"""sigma(x; v) = 1 / (1 + exp(-v*x)) [kernel CDF]."""
return 1 / (1 + torch.exp(-v * x))
@staticmethod
def _sigma_prime(v, x):
"""sigma'(x; v) = v*exp(-v|x|) / (1+exp(-v|x|))^2 [kernel PDF].
Uses |x| to avoid overflow: exp(-v|x|)->0 for large |x|,
whereas exp(-v*x)->inf for large negative x.
sigma' is even so the two forms are equivalent.
"""
x = torch.abs(x)
return (v * torch.exp(-v * x)) / ((1 + torch.exp(-v * x)) ** 2)
# -- KDE CDF and ICDF --
@staticmethod
def _cdf(v, tau, scores):
"""F_hat(tau; scores) = (1/n) sum_j sigma(tau - s_j; v)."""
return torch.mean(KernelizedROLLoss._sigma(v, tau - scores))
@staticmethod
def _icdf(v, scores, alpha, tol=1e-4, max_iters=60):
"""Find tau s.t. F_hat(tau; scores) = alpha via bisection.
Bracket: [min(scores) - 10/v, max(scores) + 10/v] always
satisfies F_hat(lo) < alpha < F_hat(hi). Converges in <=16
steps for normalised scores; 60 is a generous cap.
"""
if v > 10000.:
logging.warning(
f'Bandwidth v={v:.1f} exceeds maximum 10000, capping')
v = 10000.
margin = 10.0 / v
lo = scores.min() - margin
hi = scores.max() + margin
for _ in range(max_iters):
mid = (lo + hi) / 2
F_mid = KernelizedROLLoss._cdf(v, mid, scores)
if torch.abs(F_mid - alpha) < tol:
break
if F_mid < alpha:
lo = mid
else:
hi = mid
return mid
@staticmethod
def _icdf_batch(v, scores, alphas, tol=1e-4, max_iters=60):
"""Vectorized bisection: find tau_k s.t. F_hat(tau_k) = alpha_k.
Each iteration evaluates F_hat at all K midpoints via a single
(K, n) broadcast, replacing K sequential _icdf calls.
"""
if float(v) > 10000.:
logging.warning(
f'Bandwidth v={float(v):.1f} exceeds maximum 10000, capping')
v = (v.clamp(max=10000.)
if isinstance(v, torch.Tensor) else 10000.)
K = len(alphas)
alphas_t = torch.tensor(alphas,
dtype=scores.dtype, device=scores.device)
margin = 10.0 / float(v)
lo = torch.full((K,), float(scores.min()) - margin,
device=scores.device)
hi = torch.full((K,), float(scores.max()) + margin,
device=scores.device)
mid = (lo + hi) / 2
for _ in range(max_iters):
diff = mid.unsqueeze(1) - scores.unsqueeze(0) # (K, n)
F_mid = torch.mean(
KernelizedROLLoss._sigma(v, diff), dim=1) # (K,)
if torch.all(torch.abs(F_mid - alphas_t) < tol):
break
too_low = F_mid < alphas_t
lo = torch.where(too_low, mid, lo)
hi = torch.where(too_low, hi, mid)
mid = (lo + hi) / 2
return mid # (K,)
# -- forward pass --
@staticmethod
def forward(ctx, alphas, divisor, yh, y, gamma=1.0, normalize=True):
# Normalise scores by std for numerical stability.
# Chain-rule correction is applied in backward().
# clamp avoids inf when all batch scores are identical (zero variance).
if normalize:
score_scale = 1 / torch.sqrt(
torch.var(yh, correction=1).clamp(min=1e-8))
else:
score_scale = torch.tensor(1.0, device=yh.device)
yh_norm = yh * score_scale
pos_idx, neg_idx = split_true_false_indeces(yh, y)
scores_pos = yh_norm[pos_idx] # s^(1)
scores_neg = yh_norm[neg_idx] # s^(0)
# Bandwidths via ISJ; gamma > 1 widens kernels early in training.
v0 = (torch.tensor(
_bandwidth(scores_neg.cpu().numpy().astype(np.float64)))
/ gamma).to(yh.device)
v1 = (torch.tensor(
_bandwidth(scores_pos.cpu().numpy().astype(np.float64)))
/ gamma).to(yh.device)
# tau_k = F_0_hat^{-1}(alpha_k):
# threshold where the negative-class CDF equals alpha_k.
if len(alphas) == 1:
taus = KernelizedROLLoss._icdf(
v0, scores_neg, alphas[0]).unsqueeze(0)
else:
taus = KernelizedROLLoss._icdf_batch(
v0, scores_neg, alphas) # (K,)
# loss_k = F_1_hat(tau_k) = 1 - TPR@FPR_k
diff_pos = taus.unsqueeze(1) - scores_pos.unsqueeze(0) # (K, n1)
losses = torch.mean(
KernelizedROLLoss._sigma(v1, diff_pos), dim=1) # (K,)
total_loss = losses.sum() / divisor
logging.debug(
f'KDE-ROLL K={len(alphas)} gamma={gamma:.4f}'
f' v0={v0:.4f} v1={v1:.4f} loss={total_loss:.4f}')
ctx.save_for_backward(yh, scores_pos, scores_neg,
pos_idx, neg_idx, taus, v1, v0,
score_scale)
ctx.divisor = divisor
return total_loss
# -- backward pass --
@staticmethod
def backward(ctx, grad_output):
yh, scores_pos, scores_neg, pos_idx, neg_idx, \
taus, v1, v0, score_scale = ctx.saved_tensors
divisor = ctx.divisor
n1 = scores_pos.shape[0]
diff_pos = taus.unsqueeze(1) - scores_pos.unsqueeze(0) # (K, n1)
sp_pos = KernelizedROLLoss._sigma_prime(v1, diff_pos) # (K, n1)
# d(sum_k loss_k)/d(s_i^(1)) = -(1/n1) sum_k sigma'(tau_k - s_i)
grad_pos = -(1.0 / n1) * sp_pos.sum(dim=0) / divisor # (n1,)
# dF_1_hat(tau_k)/d(tau_k) = (1/n1) sum_j sigma'(tau_k - s_j^(1))
dF1_dtau = sp_pos.mean(dim=1) # (K,)
# d(tau_k)/d(s_i^(0))
diff_neg = taus.unsqueeze(1) - scores_neg.unsqueeze(0) # (K, n0)
s_neg = KernelizedROLLoss._sigma(v0, diff_neg) # (K, n0)
kernel = s_neg * (1 - s_neg) # v0 cancels
grad_tau_neg = kernel / kernel.sum( # (K, n0)
dim=1, keepdim=True).clamp(min=1e-10)
# d(sum_k loss_k)/d(s_i^(0))
grad_neg = (dF1_dtau.unsqueeze(1)
* grad_tau_neg).sum(dim=0) / divisor # (n0,)
grad_input = torch.zeros(yh.shape[0], device=yh.device)
grad_input[pos_idx] = grad_pos * grad_output
grad_input[neg_idx] = grad_neg * grad_output
if torch.any(torch.isnan(grad_input)):
logging.error('NaN gradient detected in KDE-ROLL backward')
# Chain-rule for score normalisation: yh_norm = yh * score_scale
return None, None, grad_input * score_scale, None, None, None
# -- public constructors --
def _kernelized_roll_fpr_internal(yh, y, fpr, gamma=1.0, normalize=True,
bce_weight=0.0, bce_pos_weight=1.0):
roll = KernelizedROLLoss.apply([fpr], 1, yh, y, gamma, normalize)
if bce_weight == 0.0:
return roll
pos_weight = torch.tensor([bce_pos_weight],
dtype=yh.dtype, device=yh.device)
bce = torch.nn.functional.binary_cross_entropy_with_logits(
yh, y.float(), pos_weight=pos_weight)
return roll + bce_weight * bce
def kernelized_roll_fpr(fpr, normalize=True,
bce_weight=0.0, bce_pos_weight=1.0):
# Pass 1-fpr as CDF target: F_0_hat(tau) = 1-fpr iff P(neg > tau) = fpr
return partial(_kernelized_roll_fpr_internal, fpr=1 - fpr,
normalize=normalize, bce_weight=bce_weight,
bce_pos_weight=bce_pos_weight)
def _kernelized_roll_tpr_internal(yh, y, tpr, gamma=1.0,
bce_weight=0.0, bce_pos_weight=1.0):
# Reduce TPR@FPR to FPR@TPR by negating scores and swapping labels
roll = KernelizedROLLoss.apply([tpr], 1, 1 - yh, 1 - y, gamma)
if bce_weight == 0.0:
return roll
pos_weight = torch.tensor([bce_pos_weight],
dtype=yh.dtype, device=yh.device)
bce = torch.nn.functional.binary_cross_entropy_with_logits(
yh, y.float(), pos_weight=pos_weight)
return roll + bce_weight * bce
def kernelized_roll_tpr(tpr, bce_weight=0.0, bce_pos_weight=1.0):
return partial(_kernelized_roll_tpr_internal, tpr=tpr,
bce_weight=bce_weight, bce_pos_weight=bce_pos_weight)
def _kernelized_roll_aoc_internal(yh, y, gamma=1.0, normalize=True,
bce_weight=0.0, bce_pos_weight=1.0):
roll = KernelizedROLLoss.apply(
_AOC_ALPHAS, _AOC_DIVISOR, yh, y, gamma, normalize)
if bce_weight == 0.0:
return roll
pos_weight = torch.tensor([bce_pos_weight],
dtype=yh.dtype, device=yh.device)
bce = torch.nn.functional.binary_cross_entropy_with_logits(
yh, y.float(), pos_weight=pos_weight)
return roll + bce_weight * bce
def kernelized_roll_aoc(normalize=True,
bce_weight=0.0, bce_pos_weight=1.0):
return partial(_kernelized_roll_aoc_internal, normalize=normalize,
bce_weight=bce_weight, bce_pos_weight=bce_pos_weight)
\end{lstlisting}
content/experiments/experiments.tex
+170 -47
View File
@@ -3,7 +3,7 @@
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{מערך נתונים}}
\paragraph{KEEL benchmark.}
We evaluate on 12 binary imbalanced datasets from the KEEL repository~\cite{TODO} (Knowledge Extraction based on Evolutionary Learning). All labels are clean. \Cref{tab:datasets:keel} summarises the datasets; imbalance ratios range from 1.8 to 11.6, covering mild to moderate imbalance regimes commonly used in the literature.
We evaluate on 12 binary imbalanced datasets from the KEEL repository~\cite{alcalafdez2011keel}. All labels are clean. \Cref{tab:datasets:keel} summarises the datasets; imbalance ratios range from 1.8 to 11.6. No known noise is reported in the original data.
\begin{table}[htbp]
\centering
@@ -31,23 +31,29 @@ Yeast3 & 1484 & 163 &1321 & 8.1 & Yeast protein localisation (UCI) \\
\end{table}
\paragraph{CIFAR-10N.}
CIFAR-10N~\cite{TODO} augments the 50{,}000-image CIFAR-10 training set ($32{\times}32$ RGB) with real human-annotated noisy labels collected from Amazon Mechanical Turk~\cite{TODO}. We form a binary task (class~1, automobile, vs.\ rest; IR~$\approx 9$:1) and train under three noise conditions:
CIFAR-10 is a widely used image dataset of 60{,}000 32x32 color images belonging to 10 mutuall exclusive classes. CIFAR-10N~\cite{wei2022cifar10n} augments the 50{,}000 training set labels with real human-annotated noisy labels collected from Amazon Mechanical Turk~\cite{wei2022cifar10n}. We form a binary task (class~1, automobile, vs.\ rest; IR~$\approx 9$:1) and train under three noise conditions:
\emph{clean} (original CIFAR-10 labels, 0\% noise),
\emph{aggre} (majority-vote aggregate of crowdworker annotations, $\approx$9\% noise), and
\emph{worse} (single worst-annotator labels, $\approx$40\% noise).
Architecture: a three-block convolutional network trained from scratch.
This allows us to measure the degredation of the model's performance under increasing label noise conditions.
The architecture tested is a three-block convolutional network trained from scratch.
\paragraph{Large tabular datasets.}
We include three large-scale tabular datasets spanning diverse imbalanced real-world settings. All labels are considered clean (no injected or crowd-sourced noise).
We include three large-scale tabular datasets spanning diverse imbalanced real-world settings.
\begin{itemize}
\item \textbf{Bank Marketing}~\cite{TODO} --- UCI repository; Portuguese bank telephone marketing campaigns (Moro et al., 2014). $n = 41{,}188$ samples, 48 features after one-hot encoding. Positive class: client subscribed to term deposit. IR~$\approx 8$:1 ($\approx$11.3\% positive). No label noise.
\item \textbf{Bank Marketing}~\cite{moro2014bank}, UCI repository; Portuguese bank telephone marketing campaigns (Moro et al., 2014). $n = 41{,}188$ samples, 48 features after one-hot encoding. Positive class: client subscribed to term deposit. IR~$\approx 8$:1 ($\approx$11.3\% positive). No label noise.
\item \textbf{Credit Card Fraud}~\cite{TODO} --- Kaggle / ULB (Worldline + MLG, 2013). $n = 284{,}807$ transactions, 30 features (V1--V28 are PCA-transformed; Time and Amount rescaled). Extreme imbalance: 492 confirmed frauds, IR~$\approx 577$:1. No label noise.
\item \textbf{Credit Card Fraud}~\cite{dalpozzolo2015calibrating}, Kaggle / ULB (Worldline + MLG, 2013). $n = 284{,}807$ transactions, 30 features (V1--V28 are PCA-transformed; Time and Amount rescaled). Extreme imbalance: 492 confirmed frauds, IR~$\approx 577$:1. Labels reflect confirmed-fraud reports; undetected fraudulent transactions constitute a structural source of class-conditional false negatives~\cite{tittelfitz2023fpr}.
\item \textbf{Home Credit}~\cite{TODO} --- Kaggle Home Credit Default Risk competition (2018). $n = 307{,}511$ loan applications, $\approx$120 numerical features from the primary application table. Positive class: loan default. IR~$\approx 11.4$:1 ($\approx$8\% default rate). Labels are proxy labels (repayment outcome); no injected noise.
\item \textbf{Home Credit}~\cite{homecredit2018kaggle}, Kaggle Home Credit Default Risk competition (2018). $n = 307{,}511$ loan applications, $\approx$120 numerical features from the primary application table. Positive class: loan default. IR~$\approx 11.4$:1 ($\approx$8\% default rate). Labels are proxy labels (repayment outcome); no injected noise.
\end{itemize}
\paragraph{NIH ChestX-ray14.}
NIH ChestX-ray14~\cite{wang2017chestxray14} is a large-scale medical chest radiograph dataset of $112{,}120$ X-rays from $30{,}805$ patients, annotated with 14 unique labels. Labels were extracted from radiological reports via natural language processing; the dataset authors estimate labelling accuracy at $>$90\%~\cite{wang2017chestxray14}, though subsequent audits have found the true error rate substantially higher, with positive predictive values 10--30 percentage points below the reported figures~\cite{oakdenrayner2020exploring}, making this a naturally noisy benchmark. We form a binary task targeting \emph{pleural effusion} detection (Effusion vs.\ rest; IR~$\approx 7.5$:1).
We extract $1{,}024$-dimensional latent representations using a \mbox{DenseNet-121} pretrained on the dataset via the TorchXRayVision library~\cite{cohen2022torchxrayvision}, with global average pooling applied to the penultimate feature map. These frozen embeddings are passed to a lightweight $1024{\to}512{\to}128{\to}1$ MLP with dropout, treating image representations as tabular features. This setup evaluates ROLL in a transfer-learning scenario and avoids GPU-intensive end-to-end training.
\section{Performance Measures}\label{sec:experiments:performance_measures}
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{מדדי ביצועים}}
@@ -57,29 +63,48 @@ We report the \emph{area under the ROC curve} (AUC) averaged over independent tr
\label{sec:experiments:configurations}
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{קונפיגורציה ופרמטרים}}
We compare \textsc{Roll}-AoC (the primary proposed loss, optimising the area under the ROC curve via KDE) and \textsc{Roll}-TPR (a variant targeting TPR at a fixed operating point) against the following baselines:
We compare \textsc{Roll}-AoC (the primary proposed loss, optimising the area under the ROC curve via KDE) against the following baselines:
\begin{itemize}
\item \textbf{BCE-W} --- binary cross-entropy with class-frequency inverse weighting.
\item \textbf{Focal} --- focal loss~\cite{TODO} with focusing parameter $\gamma = 2$.
\item \textbf{Asym.} --- asymmetric loss~\cite{TODO}.
\item \textbf{GCE} --- generalised cross-entropy~\cite{TODO} with $q = 0.7$.
\item \textbf{LibAUC} --- PESG-based AUC maximisation from the LibAUC library~\cite{TODO}.
\item \textbf{MAE} --- mean absolute error (symmetric noise-robust loss).
\item \textbf{BCE-W}: binary cross-entropy with class-frequency inverse weighting.
\item \textbf{Focal}: focal loss~\cite{lin2017focal} with focusing parameter $\gamma = 2$.
\item \textbf{Asym.}: asymmetric loss~\cite{zhou2021asymmetric}.
\item \textbf{GCE}: generalised cross-entropy~\cite{zhang2018generalized} with $q = 0.7$.
\item \textbf{LibAUC}: PESG-based AUC maximisation from the LibAUC library~\cite{yuan2021auc}.
\item \textbf{MAE}: mean absolute error (symmetric noise-robust loss).
\end{itemize}
All methods use the same network architecture, optimiser (Adam for non-LibAUC methods), and early stopping. Each configuration is run for $n = 5$ independent episodes on KEEL and large datasets, and $n = 3$ on CIFAR-10N. Reported AUC is evaluated on the held-out test split.
All methods use the same network architecture, optimiser (Adam for non-LibAUC methods), and early stopping. The best model per-episode is chosen from the epoch with lowest loss. Each configuration is run for $n = 5$ independent episodes on KEEL and large datasets, and $n = 3$ on CIFAR-10N. Reported AUC is evaluated on the test split, averaged between episodes.
\section{Results}\label{sec:experiments:results}
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{תוצאות}}
% TODO: Review which KEEL noise tables to keep in the main body.
% Low noise levels (5%, 10%) show little differentiation and may not add much to the narrative.
% Consider: keep only 20% noise table here (clearest signal, Roll-AoC leads on 6/12 datasets)
% and move 5%/10% tables to an appendix, referencing them from here for completeness.
\subsection{KEEL Benchmark}\label{sec:results:keel}
\Cref{tab:results:keel} reports mean test AUC across the 12 KEEL datasets. No single method dominates the benchmark: on easy datasets (Iris0, Wisconsin, Vowel0, Vehicle2, Glass6) nearly all methods converge to near-perfect AUC, leaving little room for differentiation. On the harder, more imbalanced datasets the picture is more varied. \textsc{Roll}-AoC competes well but does not claim the top position on most individual datasets; this is consistent with no-free-lunch intuitions for heterogeneous benchmarks.
\Cref{tab:results:keel} reports mean test AUC across the 12 KEEL datasets under clean labels.
No single method dominates: on easy datasets (Iris0, Wisconsin, Vowel0, Vehicle2, Glass6) nearly all methods converge to similar AUCs, leaving little room for differentiation.
On the harder, more imbalanced datasets the picture is more varied.
\textsc{Roll}-AoC competes well but does not claim the top position on most individual datasets, establishing it as a competitive approach.
To assess label-noise robustness, we inject symmetric label noise. A fraction of all class labels are randomly flipped.
We evaluate at noise levels of $5\%, 10\%, 20\%$, running 10 independent episodes per dataset per noise level.
\Cref{tab:results:keel-noise-05,tab:results:keel-noise-10,tab:results:keel-noise-20} show per-dataset results at each noise level.
The randomly flipped labels change each episode, yet are consistent between configurations, ensuring every configuration, per episode, sees the same labels, changed in the same way.
At $0\%$ noise, results are quite similar across configurations. \textsc{Roll}-AOC trails right behind the leader in most configurations. Since datasets are small, the \textbf{std} given for each point is relatively high, and quite a bit of variability is expected. Still, we see that, with few exceptions, results between configurations seem within throwing distance of eachother.
As noise increases the ranking shifts markedly: at $20\%$ noise \textsc{Roll}-AoC reaches the top on average (0.847), while traditionally noise-robust losses --- \textsc{GCE} (0.831) and \textsc{MAE} (0.804) --- degrade substantially.
The per-dataset tables reveal that this trend is consistent across most datasets rather than driven by a small subset: at $20\%$ noise \textsc{Roll}-AoC leads on 6 of 12 datasets (Glass0, Glass1, Glass2, Pima, Vehicle2, Yeast3), while no other method leads on more than 3.
From $0\%$ to $20\%$ noise, \textsc{Roll}-AoC drops 0.033 in mean AUC (3.7\%), compared to 0.056 for \textsc{GCE} (6.3\%), 0.071 for \textsc{MAE} (8.1\%), and 0.073 for \textsc{LibAUC} (8.3\%).
This supports the hypothesis that ROLL's distributional objective provides implicit noise robustness: mislabelled samples shift the fitted score distribution gradually rather than producing isolated large-gradient outliers.
% TODO: audit bold/emphasis across all result tables — check that bold marks the correct per-row best and that no ties or near-ties are missed or incorrectly marked.
\begin{table}[htbp]
\centering
\caption{Mean test AUC ($\pm$\,std, $n=5$ episodes) on KEEL benchmark datasets. Best per row in \textbf{bold}.}
\caption{Mean test AUC ($\pm$\,std, $n=5$ episodes) on KEEL benchmark datasets under clean labels. Best per row in \textbf{bold}.}
\label{tab:results:keel}
\resizebox{\textwidth}{!}{%
\begin{tabular}{lcccccccc}
@@ -103,9 +128,87 @@ Yeast3 & 0.906{\tiny{$\pm$0.024}} & 0.908{\tiny{$\pm$0.021}} & 0.894{\tiny{$\pm$
}
\end{table}
\begin{table}[htbp]
\centering
\caption{Mean test AUC ($\pm$\,std, $n=10$ episodes) on KEEL datasets at $\epsilon=5\%$ asymmetric label noise. Best per row in \textbf{bold}.}
\label{tab:results:keel-noise-05}
\resizebox{\textwidth}{!}{%
\begin{tabular}{lcccccccc}
\toprule
Dataset & \textsc{Roll}-AoC & \textsc{Roll}-TPR & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\
\midrule
Glass0 & 0.797{\tiny{$\pm$0.046}} & 0.759{\tiny{$\pm$0.060}} & 0.786{\tiny{$\pm$0.019}} & \textbf{0.806{\tiny{$\pm$0.030}}} & 0.792{\tiny{$\pm$0.041}} & 0.794{\tiny{$\pm$0.029}} & 0.791{\tiny{$\pm$0.056}} & 0.792{\tiny{$\pm$0.058}} \\
Glass1 & 0.728{\tiny{$\pm$0.037}} & 0.702{\tiny{$\pm$0.041}} & 0.736{\tiny{$\pm$0.032}} & 0.730{\tiny{$\pm$0.074}} & 0.704{\tiny{$\pm$0.028}} & 0.717{\tiny{$\pm$0.059}} & 0.694{\tiny{$\pm$0.060}} & \textbf{0.738{\tiny{$\pm$0.047}}} \\
Glass2 & 0.677{\tiny{$\pm$0.095}} & 0.607{\tiny{$\pm$0.114}} & 0.665{\tiny{$\pm$0.128}} & 0.581{\tiny{$\pm$0.091}} & 0.662{\tiny{$\pm$0.050}} & 0.651{\tiny{$\pm$0.078}} & 0.614{\tiny{$\pm$0.085}} & \textbf{0.686{\tiny{$\pm$0.091}}} \\
Glass6 & 0.972{\tiny{$\pm$0.021}} & 0.909{\tiny{$\pm$0.068}} & 0.972{\tiny{$\pm$0.035}} & 0.978{\tiny{$\pm$0.016}} & \textbf{0.981{\tiny{$\pm$0.016}}} & 0.947{\tiny{$\pm$0.049}} & 0.967{\tiny{$\pm$0.036}} & 0.971{\tiny{$\pm$0.031}} \\
Haberman & 0.666{\tiny{$\pm$0.094}} & 0.562{\tiny{$\pm$0.148}} & 0.627{\tiny{$\pm$0.124}} & 0.694{\tiny{$\pm$0.111}} & 0.693{\tiny{$\pm$0.065}} & 0.687{\tiny{$\pm$0.099}} & 0.651{\tiny{$\pm$0.125}} & \textbf{0.716{\tiny{$\pm$0.055}}} \\
Iris0 & 0.990{\tiny{$\pm$0.014}} & 0.770{\tiny{$\pm$0.284}} & 0.998{\tiny{$\pm$0.006}} & 0.999{\tiny{$\pm$0.002}} & 0.999{\tiny{$\pm$0.002}} & \textbf{1.000{\tiny{$\pm$0.000}}} & \textbf{1.000{\tiny{$\pm$0.000}}} & 0.999{\tiny{$\pm$0.003}} \\
New-Thyroid1 & 0.984{\tiny{$\pm$0.010}} & 0.609{\tiny{$\pm$0.408}} & 0.986{\tiny{$\pm$0.007}} & 0.971{\tiny{$\pm$0.046}} & 0.989{\tiny{$\pm$0.006}} & 0.987{\tiny{$\pm$0.019}} & 0.983{\tiny{$\pm$0.023}} & \textbf{0.991{\tiny{$\pm$0.012}}} \\
Pima & 0.780{\tiny{$\pm$0.021}} & 0.761{\tiny{$\pm$0.030}} & 0.784{\tiny{$\pm$0.016}} & 0.780{\tiny{$\pm$0.019}} & 0.767{\tiny{$\pm$0.023}} & \textbf{0.787{\tiny{$\pm$0.011}}} & 0.756{\tiny{$\pm$0.023}} & 0.775{\tiny{$\pm$0.023}} \\
Vehicle2 & 0.975{\tiny{$\pm$0.007}} & 0.954{\tiny{$\pm$0.043}} & \textbf{0.977{\tiny{$\pm$0.006}}} & 0.974{\tiny{$\pm$0.010}} & 0.969{\tiny{$\pm$0.016}} & 0.976{\tiny{$\pm$0.007}} & 0.975{\tiny{$\pm$0.008}} & 0.976{\tiny{$\pm$0.009}} \\
Vowel0 & 0.988{\tiny{$\pm$0.012}} & 0.959{\tiny{$\pm$0.028}} & 0.981{\tiny{$\pm$0.019}} & 0.989{\tiny{$\pm$0.017}} & \textbf{0.991{\tiny{$\pm$0.006}}} & 0.985{\tiny{$\pm$0.014}} & 0.973{\tiny{$\pm$0.025}} & 0.978{\tiny{$\pm$0.014}} \\
Wisconsin & 0.998{\tiny{$\pm$0.002}} & 0.997{\tiny{$\pm$0.004}} & 0.998{\tiny{$\pm$0.001}} & \textbf{0.999{\tiny{$\pm$0.001}}} & \textbf{0.999{\tiny{$\pm$0.001}}} & 0.998{\tiny{$\pm$0.001}} & 0.998{\tiny{$\pm$0.001}} & 0.998{\tiny{$\pm$0.001}} \\
Yeast3 & 0.900{\tiny{$\pm$0.020}} & 0.843{\tiny{$\pm$0.070}} & 0.914{\tiny{$\pm$0.028}} & 0.915{\tiny{$\pm$0.034}} & \textbf{0.916{\tiny{$\pm$0.026}}} & \textbf{0.916{\tiny{$\pm$0.032}}} & 0.908{\tiny{$\pm$0.029}} & 0.909{\tiny{$\pm$0.024}} \\
\bottomrule
\end{tabular}%
}
\end{table}
\begin{table}[htbp]
\centering
\caption{Mean test AUC ($\pm$\,std, $n=10$ episodes) on KEEL datasets at $\epsilon=10\%$ asymmetric label noise. Best per row in \textbf{bold}.}
\label{tab:results:keel-noise-10}
\resizebox{\textwidth}{!}{%
\begin{tabular}{lcccccccc}
\toprule
Dataset & \textsc{Roll}-AoC & \textsc{Roll}-TPR & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\
\midrule
Glass0 & 0.773{\tiny{$\pm$0.070}} & 0.767{\tiny{$\pm$0.046}} & 0.789{\tiny{$\pm$0.037}} & 0.771{\tiny{$\pm$0.037}} & \textbf{0.821{\tiny{$\pm$0.026}}} & 0.786{\tiny{$\pm$0.057}} & 0.800{\tiny{$\pm$0.032}} & 0.807{\tiny{$\pm$0.050}} \\
Glass1 & 0.719{\tiny{$\pm$0.040}} & 0.725{\tiny{$\pm$0.029}} & 0.716{\tiny{$\pm$0.079}} & 0.718{\tiny{$\pm$0.056}} & 0.710{\tiny{$\pm$0.049}} & 0.729{\tiny{$\pm$0.048}} & 0.675{\tiny{$\pm$0.070}} & \textbf{0.738{\tiny{$\pm$0.060}}} \\
Glass2 & 0.695{\tiny{$\pm$0.075}} & 0.649{\tiny{$\pm$0.119}} & 0.644{\tiny{$\pm$0.072}} & 0.687{\tiny{$\pm$0.127}} & 0.655{\tiny{$\pm$0.048}} & \textbf{0.713{\tiny{$\pm$0.107}}} & 0.637{\tiny{$\pm$0.093}} & 0.673{\tiny{$\pm$0.111}} \\
Glass6 & 0.947{\tiny{$\pm$0.050}} & 0.910{\tiny{$\pm$0.067}} & \textbf{0.973{\tiny{$\pm$0.026}}} & 0.950{\tiny{$\pm$0.041}} & 0.971{\tiny{$\pm$0.024}} & 0.938{\tiny{$\pm$0.047}} & 0.937{\tiny{$\pm$0.048}} & 0.935{\tiny{$\pm$0.054}} \\
Haberman & 0.638{\tiny{$\pm$0.098}} & 0.564{\tiny{$\pm$0.144}} & 0.700{\tiny{$\pm$0.074}} & 0.652{\tiny{$\pm$0.106}} & 0.658{\tiny{$\pm$0.087}} & 0.636{\tiny{$\pm$0.112}} & 0.694{\tiny{$\pm$0.129}} & \textbf{0.732{\tiny{$\pm$0.022}}} \\
Iris0 & 0.991{\tiny{$\pm$0.020}} & 0.879{\tiny{$\pm$0.209}} & 0.950{\tiny{$\pm$0.158}} & 0.950{\tiny{$\pm$0.158}} & \textbf{1.000{\tiny{$\pm$0.000}}} & \textbf{1.000{\tiny{$\pm$0.001}}} & \textbf{1.000{\tiny{$\pm$0.000}}} & \textbf{1.000{\tiny{$\pm$0.000}}} \\
New-Thyroid1 & 0.986{\tiny{$\pm$0.011}} & 0.588{\tiny{$\pm$0.358}} & 0.980{\tiny{$\pm$0.024}} & 0.987{\tiny{$\pm$0.012}} & \textbf{0.991{\tiny{$\pm$0.013}}} & 0.986{\tiny{$\pm$0.019}} & 0.973{\tiny{$\pm$0.049}} & 0.986{\tiny{$\pm$0.013}} \\
Pima & 0.756{\tiny{$\pm$0.026}} & 0.752{\tiny{$\pm$0.027}} & 0.760{\tiny{$\pm$0.024}} & 0.767{\tiny{$\pm$0.023}} & \textbf{0.780{\tiny{$\pm$0.013}}} & 0.761{\tiny{$\pm$0.030}} & 0.757{\tiny{$\pm$0.032}} & 0.754{\tiny{$\pm$0.032}} \\
Vehicle2 & 0.967{\tiny{$\pm$0.015}} & 0.939{\tiny{$\pm$0.020}} & 0.964{\tiny{$\pm$0.016}} & 0.962{\tiny{$\pm$0.020}} & 0.962{\tiny{$\pm$0.017}} & 0.965{\tiny{$\pm$0.009}} & 0.953{\tiny{$\pm$0.022}} & \textbf{0.970{\tiny{$\pm$0.010}}} \\
Vowel0 & \textbf{0.985{\tiny{$\pm$0.011}}} & 0.946{\tiny{$\pm$0.036}} & \textbf{0.984{\tiny{$\pm$0.016}}} & 0.979{\tiny{$\pm$0.025}} & 0.975{\tiny{$\pm$0.022}} & 0.977{\tiny{$\pm$0.024}} & 0.959{\tiny{$\pm$0.035}} & 0.946{\tiny{$\pm$0.039}} \\
Wisconsin & 0.996{\tiny{$\pm$0.001}} & 0.717{\tiny{$\pm$0.453}} & 0.996{\tiny{$\pm$0.004}} & 0.997{\tiny{$\pm$0.003}} & 0.997{\tiny{$\pm$0.003}} & \textbf{0.998{\tiny{$\pm$0.001}}} & 0.996{\tiny{$\pm$0.004}} & \textbf{0.998{\tiny{$\pm$0.001}}} \\
Yeast3 & 0.895{\tiny{$\pm$0.034}} & 0.810{\tiny{$\pm$0.097}} & 0.891{\tiny{$\pm$0.054}} & 0.868{\tiny{$\pm$0.069}} & \textbf{0.915{\tiny{$\pm$0.030}}} & 0.891{\tiny{$\pm$0.029}} & 0.872{\tiny{$\pm$0.040}} & 0.857{\tiny{$\pm$0.059}} \\
\bottomrule
\end{tabular}%
}
\end{table}
\begin{table}[htbp]
\centering
\caption{Mean test AUC ($\pm$\,std, $n=10$ episodes) on KEEL datasets at $\epsilon=20\%$ asymmetric label noise. Best per row in \textbf{bold}.}
\label{tab:results:keel-noise-20}
\resizebox{\textwidth}{!}{%
\begin{tabular}{lcccccccc}
\toprule
Dataset & \textsc{Roll}-AoC & \textsc{Roll}-TPR & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\
\midrule
Glass0 & \textbf{0.801{\tiny{$\pm$0.042}}} & 0.758{\tiny{$\pm$0.049}} & 0.767{\tiny{$\pm$0.042}} & 0.796{\tiny{$\pm$0.040}} & 0.793{\tiny{$\pm$0.051}} & 0.765{\tiny{$\pm$0.052}} & 0.745{\tiny{$\pm$0.089}} & 0.791{\tiny{$\pm$0.044}} \\
Glass1 & \textbf{0.725{\tiny{$\pm$0.036}}} & 0.682{\tiny{$\pm$0.068}} & 0.718{\tiny{$\pm$0.054}} & 0.714{\tiny{$\pm$0.036}} & 0.700{\tiny{$\pm$0.039}} & 0.706{\tiny{$\pm$0.041}} & 0.635{\tiny{$\pm$0.052}} & 0.689{\tiny{$\pm$0.060}} \\
Glass2 & \textbf{0.622{\tiny{$\pm$0.110}}} & 0.585{\tiny{$\pm$0.131}} & 0.620{\tiny{$\pm$0.166}} & 0.620{\tiny{$\pm$0.082}} & 0.586{\tiny{$\pm$0.121}} & 0.614{\tiny{$\pm$0.196}} & 0.601{\tiny{$\pm$0.138}} & 0.573{\tiny{$\pm$0.163}} \\
Glass6 & 0.915{\tiny{$\pm$0.077}} & 0.884{\tiny{$\pm$0.085}} & 0.925{\tiny{$\pm$0.063}} & 0.939{\tiny{$\pm$0.057}} & \textbf{0.962{\tiny{$\pm$0.025}}} & 0.875{\tiny{$\pm$0.102}} & 0.939{\tiny{$\pm$0.046}} & 0.827{\tiny{$\pm$0.150}} \\
Haberman & 0.632{\tiny{$\pm$0.099}} & 0.548{\tiny{$\pm$0.116}} & 0.601{\tiny{$\pm$0.129}} & 0.662{\tiny{$\pm$0.091}} & 0.633{\tiny{$\pm$0.110}} & 0.586{\tiny{$\pm$0.131}} & \textbf{0.666{\tiny{$\pm$0.130}}} & 0.629{\tiny{$\pm$0.106}} \\
Iris0 & 0.993{\tiny{$\pm$0.011}} & 0.894{\tiny{$\pm$0.286}} & 0.997{\tiny{$\pm$0.009}} & 0.897{\tiny{$\pm$0.217}} & 0.950{\tiny{$\pm$0.158}} & 0.957{\tiny{$\pm$0.135}} & 0.998{\tiny{$\pm$0.004}} & \textbf{1.000{\tiny{$\pm$0.001}}} \\
New-Thyroid1 & 0.930{\tiny{$\pm$0.047}} & 0.802{\tiny{$\pm$0.223}} & 0.951{\tiny{$\pm$0.044}} & 0.962{\tiny{$\pm$0.035}} & 0.895{\tiny{$\pm$0.160}} & \textbf{0.973{\tiny{$\pm$0.065}}} & 0.888{\tiny{$\pm$0.300}} & 0.910{\tiny{$\pm$0.152}} \\
Pima & \textbf{0.758{\tiny{$\pm$0.015}}} & 0.722{\tiny{$\pm$0.062}} & 0.756{\tiny{$\pm$0.024}} & 0.753{\tiny{$\pm$0.032}} & 0.742{\tiny{$\pm$0.048}} & 0.757{\tiny{$\pm$0.026}} & 0.741{\tiny{$\pm$0.027}} & 0.745{\tiny{$\pm$0.027}} \\
Vehicle2 & \textbf{0.949{\tiny{$\pm$0.024}}} & 0.901{\tiny{$\pm$0.052}} & 0.948{\tiny{$\pm$0.021}} & 0.936{\tiny{$\pm$0.026}} & 0.928{\tiny{$\pm$0.029}} & 0.932{\tiny{$\pm$0.038}} & 0.864{\tiny{$\pm$0.049}} & 0.892{\tiny{$\pm$0.057}} \\
Vowel0 & 0.966{\tiny{$\pm$0.033}} & 0.929{\tiny{$\pm$0.054}} & \textbf{0.978{\tiny{$\pm$0.019}}} & 0.975{\tiny{$\pm$0.018}} & 0.966{\tiny{$\pm$0.015}} & 0.956{\tiny{$\pm$0.020}} & 0.847{\tiny{$\pm$0.157}} & 0.838{\tiny{$\pm$0.079}} \\
Wisconsin & 0.995{\tiny{$\pm$0.003}} & 0.912{\tiny{$\pm$0.214}} & \textbf{0.998{\tiny{$\pm$0.002}}} & 0.997{\tiny{$\pm$0.002}} & 0.995{\tiny{$\pm$0.006}} & 0.996{\tiny{$\pm$0.004}} & 0.993{\tiny{$\pm$0.014}} & 0.997{\tiny{$\pm$0.002}} \\
Yeast3 & \textbf{0.879{\tiny{$\pm$0.032}}} & 0.657{\tiny{$\pm$0.140}} & 0.861{\tiny{$\pm$0.070}} & 0.853{\tiny{$\pm$0.063}} & 0.846{\tiny{$\pm$0.091}} & 0.850{\tiny{$\pm$0.038}} & 0.793{\tiny{$\pm$0.104}} & 0.752{\tiny{$\pm$0.133}} \\
\bottomrule
\end{tabular}%
}
\end{table}
\subsection{CIFAR-10N: Label-Noise Robustness}\label{sec:results:cifar10n}
\Cref{tab:results:cifar10n} shows AUC under three noise levels on CIFAR-10N. Under \emph{clean} labels all methods reach AUC $\geq 0.972$, with \textsc{Roll}-AoC at the top (0.980). As noise increases the advantage of \textsc{Roll}-AoC grows: under \emph{aggre} noise it leads by 0.2~pp over the next-best method (GCE, 0.932), and under \emph{worse} noise the margin widens to 1.2~pp over MAE (0.809) and 4.7~pp over BCE-W (0.738). This suggests that the KDE-smoothed loss surface provides implicit robustness to label noise.
\Cref{tab:results:cifar10n} shows AUC under three noise levels on CIFAR-10N. Under \emph{clean} labels all methods reach AUC $\geq 0.972$, with \textsc{Roll}-AoC at the top (0.980). As noise increases the advantage of \textsc{Roll}-AoC grows: under \emph{aggre} noise it leads by 0.002 over the next-best method (GCE, 0.932) while other methods degrade more significantly, and under \emph{worse} noise, the margin widens to 0.012 over MAE (0.809) and 0.047 over BCE-W (0.738). Observing degredation w.r.t the clean scenario, we can see that \testxc{Roll}-AoC degraded the least (-0.159 relative to clean results) relative to other configurations, with the other least-degrading result being MAE (-0.167)
\begin{table}[htbp]
\centering
@@ -124,8 +227,11 @@ Worse & \textbf{0.821{\tiny{$\pm$0.001}}} & 0.738{\tiny{$\pm$0.009}} & 0.734{\ti
}
\end{table}
% TODO: Remove Roll-TPR column from tab:results:large and update the discussion accordingly.
% Roll-TPR is not the focus for these datasets and its results (e.g. 0.438 on Bank Marketing) distract from the main story.
\subsection{Large Real-World Datasets}\label{sec:results:large}
% TODO: Reword Bank Marketing discussion after rerun — current results (Roll-AoC 0.511, Roll-TPR 0.438) are likely a bug. Update numbers and narrative once confirmed.
\Cref{tab:results:large} covers three large-scale datasets. On Credit Card Fraud \textsc{Roll}-TPR is the top performer (0.974) with \textsc{Roll}-AoC close behind (0.968), and both outperform all non-ROLL baselines. On Home Credit \textsc{Roll}-AoC edges LibAUC (0.712 vs.\ 0.711). On Bank Marketing, however, GCE achieves the highest AUC (0.746) while \textsc{Roll}-AoC and \textsc{Roll}-TPR underperform (0.511 and 0.438 respectively); the cause of this degradation on this dataset is an open question and a direction for future investigation.
\begin{table}[htbp]
@@ -146,32 +252,46 @@ Home Credit & \textbf{0.712{\tiny{$\pm$0.001}}} & 0.687{\tiny{$\pm$0.004}} & 0.6
\end{table}
\section{Label-Noise Poisoning Experiment}\label{sec:experiments:poisoning}
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{ניסוי הרעלת תוויות}}
\subsection{NIH ChestX-ray14}\label{sec:results:chestxray14}
% A controlled experiment that isolates ROLL's label-noise robustness from confounders
% (dataset size, imbalance ratio). Protocol: start from a clean dataset, inject
% \emph{asymmetric} noise --- flip a fraction of positive labels to negative, leave
% negative labels intact --- and track AUC degradation as the flip rate increases.
% This mirrors a worst-case practical scenario: rare true positives become false negatives,
% compounding the imbalance. Empirical complement to the structural argument in
% \Cref{sec:noisy-labels}.
\Cref{tab:results:chestxray14} reports test AUC on the ChestX-ray14 Effusion task.
\textsc{Roll}-AoC achieves the highest AUC ($0.839\pm0.001$), a clear margin above the
next-best methods GCE ($0.829\pm0.004$) and \textsc{Roll}-TPR ($0.828\pm0.002$).
Standard cross-entropy variants trail substantially: BCE-W reaches $0.788\pm0.008$,
Focal $0.798\pm0.009$, and Asym.\ $0.729\pm0.007$. MAE ($0.810\pm0.001$) benefits from
its noise-robust design but still falls behind the distributional losses. LibAUC results
are excluded due to an optimizer instability on this dataset (to be resolved in a
follow-up run).
\subsection{Protocol}\label{sec:experiments:poisoning:protocol}
The ${\approx}5$~pp gap between \textsc{Roll}-AoC and BCE-W, on a dataset whose labels
carry substantial organic NLP noise~\cite{oakdenrayner2020exploring}, is consistent with the findings from the controlled
KEEL noise experiment (\Cref{sec:results:keel}) and CIFAR-10N
(\Cref{sec:results:cifar10n}). The result also demonstrates that ROLL-AOC generalises
beyond from-scratch training: the DenseNet-121 embeddings are frozen, so ROLL operates
entirely within the MLP head, fitting score distributions over a fixed representation.
% Specify: which dataset(s) were poisoned (one or two from KEEL or large-scale, chosen
% for mid-range difficulty where differences are visible), the range of flip rates tested
% (e.g.\ 0\% to 40\% in steps of 10\%), and number of independent episodes per rate.
% Justify asymmetric noise as the harder, more realistic case compared to symmetric flipping.
\begin{table}[htbp]
\centering
\caption{Mean test AUC ($\pm$\,std, $n=5$ episodes) on NIH ChestX-ray14 (Effusion
vs.\ rest, IR~${\approx}7.5$:1). Labels carry substantial organic NLP noise~\cite{oakdenrayner2020exploring}.
LibAUC excluded due to optimizer instability. Best in \textbf{bold}.}
\label{tab:results:chestxray14}
\begin{tabular}{lc}
\toprule
Method & AUC \\
\midrule
\textsc{Roll}-AoC & \textbf{0.839{\tiny{$\pm$0.001}}} \\
\textsc{Roll}-TPR & 0.828{\tiny{$\pm$0.002}} \\
GCE & 0.829{\tiny{$\pm$0.004}} \\
MAE & 0.810{\tiny{$\pm$0.001}} \\
Focal & 0.798{\tiny{$\pm$0.009}} \\
BCE-W & 0.788{\tiny{$\pm$0.008}} \\
Asym. & 0.729{\tiny{$\pm$0.007}} \\
LibAUC & --- \\
\bottomrule
\end{tabular}
\end{table}
\subsection{Results}\label{sec:experiments:poisoning:results}
% Present AUC vs.\ noise-rate curves for all methods. Key finding: ROLL-AoC degrades
% more gracefully than pointwise losses (BCE-W, Focal, Asym.) because a distributional
% objective spreads the influence of each mislabeled sample over the estimated density
% rather than producing a single large gradient. Cite structural explanation from
% \Cref{sec:noisy-labels}; note consistency with CIFAR-10N results in
% \Cref{sec:results:cifar10n}.
\section{Ablation Study}\label{sec:experiments:ablation}
@@ -227,12 +347,15 @@ Home Credit & \textbf{0.712{\tiny{$\pm$0.001}}} & 0.687{\tiny{$\pm$0.004}} & 0.6
\section{Summary}\label{sec:experiments:summary}
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{סיכום}}
% Synthesise the empirical findings in 2--3 paragraphs.
% (1) Where ROLL-AoC wins clearly: CIFAR-10N noise robustness, Credit Card Fraud,
% Home Credit --- relate to the distributional objective and KDE smoothing.
% (2) Where it underperforms: Bank Marketing --- flag as an open question; discuss
% possible causes (feature scale, label proxy quality, optimizer interaction).
% (3) What the ablation reveals: which components are load-bearing vs.\ incidental.
% Close with a sentence bridging to \Cref{chapter:discussion_and_conclusions}.
% TODO: write 2--3 paragraphs synthesising findings.
% (1) Noise robustness is the primary result: ROLL-AoC is best at 20\% KEEL noise
% and best at all three CIFAR-10N noise levels. The story is consistent across
% very different settings (small tabular vs. image).
% (2) Large-dataset picture: strong on Credit Card Fraud and Home Credit; clear
% failure on Bank Marketing (0.511 vs 0.746 for GCE) -- flag as open question.
% (3) ROLL-TPR is inconsistent: best on Credit Card Fraud (0.974), poor elsewhere.
% Its value is as a targeted tool when a specific FPR budget is fixed.
% Bridge: Discussion chapter \Cref{chapter:discussion_and_conclusions} addresses
% the Bank Marketing failure and the mechanism behind noise robustness.
TODO write this section.
content/method/method.tex
+94 -80
View File
@@ -3,10 +3,10 @@
\label{sec:problem-formulation}
% Motivation for this section — why accuracy fails under class imbalance, why
% TPR@FPR gives direct operating-point control, and the Neyman-Pearson framing
% that underpins ROLL — is developed in \Cref{chapter:related_work}
% (\Cref{sec:imbalanced-tpr-fpr,sec:neyman-pearson}). This section establishes
% only the notation and objective used throughout the chapter.
% AUC is the natural objective under label noise and imbalance, and the
% Neyman-Pearson framing for single-threshold optimization — is developed in
% \Cref{chapter:related_work} (\Cref{sec:imbalanced-tpr-fpr,sec:neyman-pearson}).
% This section establishes the notation and primary objective used throughout the chapter.
Given a dataset $\mathcal{D} = \{(\mathbf{x}_i, y_i)\}_{i=1}^{n}$ with inputs
$\mathbf{x}_i \in \mathcal{X}$ and binary labels $y_i \in \{0, 1\}$, we learn a
scoring function $f_\theta : \mathcal{X} \to \mathbb{R}$ by fitting.
@@ -17,50 +17,51 @@ The positive and negative subsets of $\mathcal{D}$ are:
\mathcal{D}_0 = \{(\mathbf{x}_i, y_i) \in \mathcal{D} \mid y_i = 0\}.
\]
Applying a threshold $\tau$ produces predictions $\hat{y} = \mathbf{1}[f_\theta(\mathbf{x}) > \tau]$.
The \emph{true positive rate} (TPR) and \emph{false positive rate} (FPR) at $\tau$ are:
The \emph{true positive rate} (TPR) and \emph{false positive rate} (FPR) at threshold $\tau$ are:
\begin{align}
\operatorname{TPR}(\alpha) &= P\!\bigl(f_\theta(\mathbf{x}) > \tau \mid y = 1\bigr), \\
\operatorname{FPR}(\alpha) &= P\!\bigl(f_\theta(\mathbf{x}) > \tau \mid y = 0\bigr).
\operatorname{TPR}(\tau) &= P\!\bigl(f_\theta(\mathbf{x}) > \tau \mid y = 1\bigr), \\
\operatorname{FPR}(\tau) &= P\!\bigl(f_\theta(\mathbf{x}) > \tau \mid y = 0\bigr).
\end{align}
In our scenario, we either wish to learn $f_{\theta}$ in order to maximize $\operatorname{TPR}$
subject to a fixed $\operatorname{FPR}$, or vice versa. Our fixed $\operatorname{TPR}$ or $\operatorname{FPR}$
will be fixed to a value $\alpha \in (0, 1)$.
Therefor, when maximizing our $\operatorname{TPR}$ subject to a fixed $\operatorname{FPR}$, we are solving for:
For $\alpha \in (0,1)$, let $\tau^*(\alpha)$ denote the threshold achieving
$\operatorname{FPR}(\tau^*(\alpha)) = \alpha$, and write
$\operatorname{TPR}(\alpha) = \operatorname{TPR}(\tau^*(\alpha))$ for the true positive rate
at that operating point. The curve $\alpha \mapsto \operatorname{TPR}(\alpha)$ is the ROC curve.
The primary learning objective is to maximize its area:
\begin{equation}
\max_\theta \; \operatorname{TPR}\!\bigl(\tau^*(\theta)\bigr)
\quad \text{s.t.} \quad \operatorname{FPR}\!\bigl(\tau^*(\theta)\bigr) = \alpha,
\label{eq:objective}
\max_\theta\;\operatorname{AUC}(f_\theta) = \int_0^1 \operatorname{TPR}(\alpha)\,d\alpha.
\label{eq:objective-auc}
\end{equation}
Maximizing AUC is the natural objective under class imbalance and label noise, where no
single operating threshold is fixed in advance.
where $\tau^*(\theta)$ is the threshold that achieves exactly $\operatorname{FPR} = \alpha$
for the current model $f_\theta$.
Conversley, when we are minimizing $\operatorname{FPR}$ subject to a fixed $\operatorname{TPR}$ we are solving for:
\begin{equation}
\min_\theta \; \operatorname{FPR}\!\bigl(\tau^*(\theta)\bigr)
\quad \text{s.t.} \quad \operatorname{TPR}\!\bigl(\tau^*(\theta)\bigr) = \alpha,
\label{eq:objective}
\end{equation}
The ROLL framework, introduced in
\Cref{sec:roll-framework}, provides a method of differentiable surrogate for these objectives.
As a special case, one may instead optimize a single operating point: maximize
$\operatorname{TPR}$ at a fixed $\operatorname{FPR} = \alpha_0$, or minimize
$\operatorname{FPR}$ at a fixed $\operatorname{TPR} = \alpha_0$.
The ROLL framework, introduced in \Cref{sec:roll-framework}, provides differentiable
surrogates for both the AUC objective \eqref{eq:objective-auc} and these point objectives.
%------------------------------------------------
\section{The ROLL Framework}
\label{sec:roll-framework}
In order to allow gradient-based learning methods to optimize TPR or FPR directly, we introduce
ROLL - Rate Optimized Likelyhood Loss.
In order to allow gradient-based learning methods to optimize AUC directly, we introduce
ROLL --- Rate Optimized Likelihood Loss.
The core idea of ROLL is as follows: when training, the user selects that they would like to optimize
TPR given some fixed FPR. Then, during training, on every training step, the actual TPR is calculated for
the given FPR, and the fitting algorithm tries to maximize the TPR directly. This way, while training, we gain direct control over the performance at the threshold that we care about - avoiding needing to pick out a threshold later.
The core idea of ROLL is as follows: at each training step, differentiable distributions
are fitted to the positive- and negative-class score outputs of $f_\theta$. AUC is then
approximated by evaluating the estimated $\operatorname{TPR}$ at 99 evenly-spaced
$\operatorname{FPR}$ values across $(0,1)$ and averaging --- a numerical integration of
the ROC curve. Because each per-threshold evaluation is differentiable through the fitted
distributions, this AUC approximation can be maximized directly by gradient descent, with
no threshold selection required at inference time.
In Neyman-Pearson terms~\cite{tong2016neyman}, our algorithm wishes to directly minimize the Type-1 error given a fixed Type-2 error, or vice versa.
ROLL also supports single-threshold optimization: selecting a target
$\operatorname{FPR} = \alpha_0$ and maximizing $\operatorname{TPR}$ at exactly that point.
In Neyman-Pearson terms~\cite{tong2016neyman}, this corresponds to minimizing Type-I error
at a fixed Type-II error bound.
In practice, however, things are not so simple.
In practice, however, making these evaluations differentiable is not straightforward.
\subsection{The Differentiability Problem}
\label{sec:differentiability-problem}
@@ -116,15 +117,14 @@ every step. Contiguous probability distributions are contiguous w.r.t their defi
Additionally, we can calculate the TPR, FPR given the probability distributions CDF and ICDF functions. If we can gaurantee these functions to be differentiable, which proved to be the main challenge of this work, we can gaurantee the entire process to be differentiable. Once we do so, a derivative of the scores themselves w.r.t. the loss can be acheived.
\subsection{General ROLL Formulation and Derivation}
\subsection{Per-Threshold ROLL Loss}
\label{sec:roll-formulation}
% Present the abstract ROLL loss in terms of a generic distribution $\mathcal{F}$
% fitted to model scores, without yet committing to a specific family. Derive the
% expression for TPR@FPR$_0$ as a function of $\mathcal{F}$'s parameters, and show
% that minimizing the loss is equivalent to maximizing TPR subject to the FPR
% constraint. Include the full derivation here; this is the core theoretical
% contribution of the chapter.
% expression for TPR@FPR as a function of $\mathcal{F}$'s parameters. This
% per-threshold loss is the building block that ROLL-AOC averages over 99 FPR values
% to obtain the primary AUC objective.
Using the dataset $\mathcal{D} = \{(\mathbf{x}_i, y_i)\}_{i=1}^{n}$ as introduced in
\Cref{sec:problem-formulation}, let $\mathcal{B} \subseteq \mathcal{D}$ denote a training batch
@@ -255,49 +255,38 @@ Unifying both cases, the general gradient applicable to all ROLL instantiations
The concrete form of each partial derivative depends on the choice of distribution family
$\mathcal{F}$, and is derived for each instantiation in \Cref{sec:roll-instantiations}.
\subsection{ROLL variant - direct AUC optimisation}
In addition to optimising for a specific rate, ROLL provides us an opportunity to
optimize the AUC of the ROC directly.
The AUC is defined as the area under the curve of the ROC. The ROC is a function that
for every FPR gives us the performing TPR of the model we have fitted.
Formally, for the function $TPR(\alpha)$ as defined above, we may define AUC as:
\subsection{ROLL-AOC: AUC Optimization}
\label{sec:roll-aoc}
The primary objective of ROLL is to maximize AUC directly, using the per-threshold
loss \eqref{eq:roll-tpr-at-fpr} as a building block. Recall from
\Cref{sec:problem-formulation} that AUC is the integral of $\operatorname{TPR}(\alpha)$
over all FPR values:
\begin{equation}
\operatorname{AUC} = \int_{0}^{1}\operatorname{TPR}(\alpha) d \alpha
\operatorname{AUC} = \int_{0}^{1}\operatorname{TPR}(\alpha)\,d\alpha.
\end{equation}
The AUC is something we would like to maximize. In order to do so, we would like to
minimize the area over the curve, which is $1-AUC$.
We note that $1-\operatorname{TPR}(\alpha) = \mathcal{L}(TPR@FPR)$.
From here, we would like to minimize:
Maximizing AUC is equivalent to minimizing the \emph{Area Over the Curve} (AOC),
$1 - \operatorname{AUC}$. Since $1 - \operatorname{TPR}(\alpha) =
\mathcal{L}_{\text{ROLL-TPR@FPR}}(\alpha)$ by definition \eqref{eq:roll-tpr-at-fpr}:
\begin{equation}
\mathcal{L}_{AOC} = \int_0^1\mathcal{L}_{\text{ROLL-TPR@FPR}}(\alpha) d \alpha
\mathcal{L}_{\text{AOC}} = \int_0^1\mathcal{L}_{\text{ROLL-TPR@FPR}}(\alpha)\,d\alpha.
\end{equation}
A direct integration is not part of this work. However, integration can be approximated. Instead
of integrating directly, it is possible to evaluate the loss for every FPR between 0 and 1, sum them, and re-normalize.
Therefor, subdividing the interval $(0, 1)$ to 99 steps (every alpha starting from $0.01$ and ending with $0.99$, since a value of $0$ or $1$ would be illegal), our loss for minimizing AOC becomes:
Direct integration is not tractable, but the integral is well-approximated by a Riemann
sum. Evaluating at 99 evenly-spaced FPR values $\alpha = i/100$ for $i = 1, \ldots, 99$
(excluding the degenerate endpoints 0 and 1), the ROLL-AOC loss is:
\begin{equation}
\mathcal{L}_{ROLL-AOC} = \frac{1}{99}\sum_{i=1}^{99}\mathcal{L}_{ROLL-TPR@FPR}\left(\frac{i}{100}\right)
\mathcal{L}_{\text{ROLL-AOC}} = \frac{1}{99}\sum_{i=1}^{99}\mathcal{L}_{\text{ROLL-TPR@FPR}}\!\left(\frac{i}{100}\right)
\label{eq:roll-aoc}
\end{equation}
The gradient per score then simply becomes the average gradient over every threshold.
The gradient follows by linearity of the sum:
\begin{equation}
\frac{\partial \mathcal{L}_{ROLL-AOC}(f_\theta(\mathbf{X}))}
{\partial f_\theta(\mathbf{x}_i)} = \frac{1}{99}\sum_{i=1}^{99}\left(\frac{\partial \mathcal{L}_{\text{ROLL-TPR@FPR}}\left(f_\theta(\mathbf{X}) ; \frac{i}{100}\right)}
{\partial f_\theta(\mathbf{x}_i)} \right)
\frac{\partial \mathcal{L}_{\text{ROLL-AOC}}(f_\theta(\mathbf{X}))}
{\partial f_\theta(\mathbf{x}_i)} = \frac{1}{99}\sum_{i=1}^{99}\frac{\partial \mathcal{L}_{\text{ROLL-TPR@FPR}}\!\left(f_\theta(\mathbf{X}) ; \frac{i}{100}\right)}
{\partial f_\theta(\mathbf{x}_i)}
\label{eq:roll-aoc-gradient}
\end{equation}
The concrete per-threshold gradient used in the sum is derived for each distribution
family in \Cref{sec:roll-instantiations}.
%------------------------------------------------
@@ -327,6 +316,8 @@ While the distributions of scores cannot be gauranteed to follow any form, there
some instances where the score distributions might be similar or exactly gaussian.
For example, when finding linear separability between gaussian distributions, the resulting
scores will be distributed normally, and this method will fit exactly.
In ROLL-AOC, the Gaussian instantiation evaluates \eqref{eq:roll-tpr-at-fpr-gaussian}
at each of the 99 FPR grid points and averages the results.
\subsubsection{Forward}
\label{sec:roll-gaussian-forward}
@@ -401,6 +392,9 @@ Beta ROLL applies when model scores are constrained to the open interval $(0,1)$
as is the case when $f_\theta$ uses a sigmoid output activation. The Beta distribution
is supported on $(0,1)$ and its two shape parameters allow it to capture a wide range
of unimodal and skewed score distributions that a Gaussian cannot represent.
In ROLL-AOC, Beta ROLL evaluates \eqref{eq:roll-tpr-at-fpr-beta} at each of the 99
FPR grid points and averages the results, with gradients propagated via automatic
differentiation.
\subsubsection{Forward}
\label{sec:roll-beta-forward}
@@ -470,6 +464,9 @@ are therefore propagated via automatic differentiation through the estimator
Unfortunately, the score output of trained models remains unpredictable. A solution for this is to use a Kernel Density Estimation (KDE) for probability
estimation. This benefits from being far more adaptable to real-world distributions of outputs of our trained models. The tradeoff of course is computational
complexity - computing the gradient derivation in such a case is non-trivial.
KDE ROLL is the primary instantiation used in ROLL-AOC: its non-parametric flexibility
makes it the most accurate estimator of the ROC curve when score distributions are
unknown, at the cost of evaluating the custom backward pass 99 times per training step.
\subsubsection{Forward}
\label{sec:roll-kde-forward}
@@ -632,13 +629,20 @@ The Beta distribution only accepts values in the interval $\[0, 1\]$. Since it
is mapped only over an interval, the translation-invariance assumption does not hold
for this distribution.
\begin{corollary}[Gradient balance under ROLL-AOC]
\label{cor:grad-balance-aoc}
$\mathcal{L}_{\emph{\text{ROLL-AOC}}}$ also satisfies gradient balance: since it is a uniform
average of 99 per-threshold objectives each satisfying \eqref{eq:grad-balance}, the property
is preserved by linearity of summation.
\end{corollary}
\paragraph{Gradient locality.}
\label{para:gradient-locality}
A second structural consequence of the ROLL-KDE formulation is that gradient mass is
\emph{concentrated near the operating threshold $\tau$}. Samples whose scores lie
close to $\tau$ receive larger gradient updates, while samples
already far on the correct side contribute very little.
A second structural consequence of the ROLL-KDE formulation, when optimizing at a
single operating threshold, is that gradient mass is \emph{concentrated near that
threshold $\tau$}. Samples whose scores lie close to $\tau$ receive larger gradient
updates, while samples already far on the correct side contribute very little.
This follows directly from \Cref{eq:kde-grad-combined}. For the positive class ($y_i = 1$),
the gradient magnitude is
@@ -663,12 +667,12 @@ This stands in contrast to standard losses such as cross-entropy, where the per-
gradient is determined by the predicted probability, with no direct dependence on an
operating threshold.
\begin{observation}[Gradient locality]
\begin{observation}[Gradient locality, single-threshold]
\label{obs:gradient-locality}
For KDE-ROLL, the gradient magnitude of sample $\mathbf{x}_i$ is proportional to
the kernel PDF evaluated at the distance between $f_\theta(\mathbf{x}_i)$ and the
operating threshold $\tau$. Samples nearest to $\tau$ receive the largest updates;
samples far from $\tau$ in either direction contribute negligible gradient.
For KDE-ROLL at a single operating threshold, the gradient magnitude of sample
$\mathbf{x}_i$ is proportional to the kernel PDF evaluated at the distance between
$f_\theta(\mathbf{x}_i)$ and $\tau$. Samples nearest to $\tau$ receive the largest
updates; samples far from $\tau$ in either direction contribute negligible gradient.
\end{observation}
\Cref{fig:threshold-locality} illustrates this for a simulated two-class setting. The
@@ -689,6 +693,16 @@ concentration of gradient mass around $\tau$, with smooth decay on either side.
\label{fig:threshold-locality}
\end{figure}
\begin{remark}[Gradient coverage under ROLL-AOC]
\label{rem:aoc-coverage}
\Cref{obs:gradient-locality} describes gradient concentration at a single threshold.
Under ROLL-AOC, 99 distinct thresholds spanning $\operatorname{FPR} \in \{0.01,\ldots,0.99\}$
are active simultaneously, so the gradient is a superposition of 99 bell-shaped kernels
spread across the full score range. This broadens gradient coverage relative to
single-threshold optimization, making ROLL-AOC more suitable when no specific operating
point is fixed in advance.
\end{remark}
%------------------------------------------------
\section{Implementation Considerations}
main.tex
+25
View File
@@ -130,6 +130,26 @@
\usepackage{titling}
\usepackage[titletoc]{appendix}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Code listings:
\usepackage{listings}
\lstset{
basicstyle=\ttfamily\footnotesize,
breaklines=true,
breakatwhitespace=false,
showstringspaces=false,
frame=single,
numbers=left,
numberstyle=\tiny\ttfamily,
numbersep=6pt,
xleftmargin=18pt,
framexleftmargin=18pt,
tabsize=4,
captionpos=b,
keepspaces=true,
columns=flexible,
}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%Not included since 'hyperref' and 'bookmark' are too problematic with Hebrew
% \usepackage[hidelinks]{hyperref}
@@ -266,6 +286,11 @@
\addtocontents{tocheb}{\contentsline{chapter}{\protect\numberline{נספח ו}{\quad\qquadKDE ROLL: היפוך ניוטון-רפסון}}{{\beginL\thepage\endL}}}
\input{content/appendixs/appendix_kde_nr}
\chapter{ROLL Loss: Implementation Listing}
\label{appendix:roll-code}
\addtocontents{tocheb}{\contentsline{chapter}{\protect\numberline{נספח ז}{\quad\qquadROLL: קוד מימוש}}{{\beginL\thepage\endL}}}
\input{content/appendixs/appendix_roll_code}
\end{appendices}
%~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~%