From d217b2356a5d7197be68c5557a9af2e7fa60f220 Mon Sep 17 00:00:00 2001 From: Aner Zakobar Date: Mon, 31 Aug 2026 23:58:01 +0300 Subject: [PATCH] It's done first draft wahoo --- bibliography/thesis.bib | 32 ++++-------- content/appendixs/appendix_kde_grad.tex | 2 +- content/appendixs/appendix_kde_sigmoid.tex | 3 +- content/appendixs/appendix_keel_noise.tex | 10 ++-- .../discussion_and_conclusions.tex | 46 ++++++++--------- content/experiments/experiments.tex | 49 ++++++++++--------- content/introduction/introduction.tex | 6 +-- content/method/method.tex | 23 +++++---- content/related_work/related_work.tex | 15 +++--- main.tex | 4 +- 10 files changed, 90 insertions(+), 100 deletions(-) diff --git a/bibliography/thesis.bib b/bibliography/thesis.bib index 8c1d77d..e5277f0 100644 --- a/bibliography/thesis.bib +++ b/bibliography/thesis.bib @@ -71,16 +71,6 @@ year={2022} } -@article{demaine2007jigsaw, - title={Jigsaw puzzles, edge matching, and polyomino packing: Connections and complexity}, - author={Demaine, Erik D. and Demaine, Martin L.}, - journal={Graphs and Combinatorics}, - volume={23}, - number={1}, - pages={195--208}, - year={2007}, - publisher={Springer} -} @article{xie2024wsauc, title={Weakly Supervised {AUC} Optimization: A Unified Partial {AUC} Approach}, @@ -101,16 +91,6 @@ archivePrefix={arXiv} } -@article{altman1989solving, - title={Solving the jigsaw puzzle problem in linear time}, - author={Altman, Tom}, - journal={Applied Artificial Intelligence an International Journal}, - volume={3}, - number={4}, - pages={453--462}, - year={1989}, - publisher={Taylor \& Francis} -} @inproceedings{ghosh2017robust, title={Robust Loss Functions under Label Noise for Deep Neural Networks}, @@ -169,7 +149,7 @@ @inproceedings{feng2020can, title={Can Cross Entropy Loss Be Robust to Label Noise?}, - author={Feng, Lei and Shu, Senlin and Lin, Zhuoyi and Lv, Fengmei and Li, Li and An, Bo}, + author={Feng, Lei and Shu, Senlin and Lin, Zhuoyi and Lv, Fengmao and Li, Li and An, Bo}, booktitle={Proceedings of the Twenty-Ninth International Joint Conference on Artificial Intelligence}, pages={2206--2212}, year={2020} @@ -223,7 +203,7 @@ @article{oakdenrayner2020exploring, title={Exploring Large-scale Public Medical Image Datasets}, - author={Oakden-Rayner, Lauren}, + author={Oakden-Rayner, Luke}, journal={Academic Radiology}, volume={27}, number={1}, @@ -328,3 +308,11 @@ year={2017}, doi={10.1109/ICCV.2017.324} } + +@inproceedings{lee2018cleannet, + title={{CleanNet}: Transfer Learning for Scalable Image Classifier Training with Label Noise}, + author={Lee, Kuang-Huai and He, Xiaodong and Zhang, Lei and Yang, Linjun}, + booktitle={Proceedings of the {IEEE} Conference on Computer Vision and Pattern Recognition ({CVPR})}, + pages={5447--5456}, + year={2018} +} diff --git a/content/appendixs/appendix_kde_grad.tex b/content/appendixs/appendix_kde_grad.tex index b20ea72..aba0213 100644 --- a/content/appendixs/appendix_kde_grad.tex +++ b/content/appendixs/appendix_kde_grad.tex @@ -117,7 +117,7 @@ then multiplying numerator and denominator by $\sigma_0'(\tau - f_\theta(\mathbf \paragraph{Efficient computation via the sigmoid identity.} Direct evaluation of \eqref{eq:kde-grad-tau} requires computing $\sigma_0'$ for every -point — potentially unstable when a point is far from $\tau$. For the sigmoid kernel, +point, potentially unstable when a point is far from $\tau$. For the sigmoid kernel, the identity \begin{equation} diff --git a/content/appendixs/appendix_kde_sigmoid.tex b/content/appendixs/appendix_kde_sigmoid.tex index e62b23e..63843d8 100644 --- a/content/appendixs/appendix_kde_sigmoid.tex +++ b/content/appendixs/appendix_kde_sigmoid.tex @@ -27,8 +27,7 @@ Substituting \eqref{eq:sigmoid-kernel} into the KDE CDF: \label{eq:kde-sigmoid-cdf} \end{equation} % -The derivative of $\hat{F}_k$ with respect to $\tau$ — used in Newton--Raphson inversion -and in the gradient — is: +The derivative of $\hat{F}_k$ with respect to $\tau$ is: % \begin{equation} \frac{\partial \hat{F}_{k}(\tau)}{\partial \tau} diff --git a/content/appendixs/appendix_keel_noise.tex b/content/appendixs/appendix_keel_noise.tex index 16a2af0..493b37b 100644 --- a/content/appendixs/appendix_keel_noise.tex +++ b/content/appendixs/appendix_keel_noise.tex @@ -1,14 +1,14 @@ -The tables below report mean test AUC at $\varepsilon = 5\%$ and $\varepsilon = 10\%$ asymmetric label noise on the KEEL benchmark. +The tables below report mean test AUC at $\varepsilon = 5\%$ and $\varepsilon = 10\%$ symmetric label noise on the KEEL benchmark. Results at $\varepsilon = 20\%$ noise, where differentiation between methods is strongest, are discussed and tabled in the main text (\Cref{sec:results:keel}). \begin{table}[htbp] \centering -\caption{Mean test AUC ($\pm$\,std, $n=10$ episodes) on KEEL datasets at $\epsilon=5\%$ asymmetric label noise. Best per row in \textbf{bold}.} +\caption{Mean test AUC ($\pm$\,std, $n=10$ episodes) on KEEL datasets at $\epsilon=5\%$ symmetric label noise. Best per row in \textbf{bold}.} \label{tab:results:keel-noise-05} \resizebox{\textwidth}{!}{% \begin{tabular}{lccccccc} \toprule -Dataset & \textsc{Roll}-AOC & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\ +Dataset & \textsc{ROLL}-AOC & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\ \midrule Glass0 & 0.797{\tiny{$\pm$0.046}} & 0.786{\tiny{$\pm$0.019}} & \textbf{0.806{\tiny{$\pm$0.030}}} & 0.792{\tiny{$\pm$0.041}} & 0.794{\tiny{$\pm$0.029}} & 0.791{\tiny{$\pm$0.056}} & 0.792{\tiny{$\pm$0.058}} \\ Glass1 & 0.728{\tiny{$\pm$0.037}} & 0.736{\tiny{$\pm$0.032}} & 0.730{\tiny{$\pm$0.074}} & 0.704{\tiny{$\pm$0.028}} & 0.717{\tiny{$\pm$0.059}} & 0.694{\tiny{$\pm$0.060}} & \textbf{0.738{\tiny{$\pm$0.047}}} \\ @@ -29,12 +29,12 @@ Yeast3 & 0.900{\tiny{$\pm$0.020}} & 0.914{\tiny{$\pm$0.028}} & 0.915{\tiny \begin{table}[htbp] \centering -\caption{Mean test AUC ($\pm$\,std, $n=10$ episodes) on KEEL datasets at $\epsilon=10\%$ asymmetric label noise. Best per row in \textbf{bold}.} +\caption{Mean test AUC ($\pm$\,std, $n=10$ episodes) on KEEL datasets at $\epsilon=10\%$ symmetric label noise. Best per row in \textbf{bold}.} \label{tab:results:keel-noise-10} \resizebox{\textwidth}{!}{% \begin{tabular}{lccccccc} \toprule -Dataset & \textsc{Roll}-AOC & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\ +Dataset & \textsc{ROLL}-AOC & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\ \midrule Glass0 & 0.773{\tiny{$\pm$0.070}} & 0.789{\tiny{$\pm$0.037}} & 0.771{\tiny{$\pm$0.037}} & \textbf{0.821{\tiny{$\pm$0.026}}} & 0.786{\tiny{$\pm$0.057}} & 0.800{\tiny{$\pm$0.032}} & 0.807{\tiny{$\pm$0.050}} \\ Glass1 & 0.719{\tiny{$\pm$0.040}} & 0.716{\tiny{$\pm$0.079}} & 0.718{\tiny{$\pm$0.056}} & 0.710{\tiny{$\pm$0.049}} & 0.729{\tiny{$\pm$0.048}} & 0.675{\tiny{$\pm$0.070}} & \textbf{0.738{\tiny{$\pm$0.060}}} \\ diff --git a/content/discussion_and_conclusions/discussion_and_conclusions.tex b/content/discussion_and_conclusions/discussion_and_conclusions.tex index 02cff2f..e91fa9e 100644 --- a/content/discussion_and_conclusions/discussion_and_conclusions.tex +++ b/content/discussion_and_conclusions/discussion_and_conclusions.tex @@ -3,7 +3,7 @@ \label{sec:discussion} \addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{דיון}} -\subsection{Noise Robustness of \textsc{Roll}-AOC} +\subsection{Noise Robustness of \textsc{ROLL}-AOC} \label{sec:discussion:noise} \addcontentsline{tocheb}{subsection}{\protect\numberline{\subsecnumforhebrewtoc}{עמידות לרעש של ROLL-AOC}} @@ -23,9 +23,9 @@ % strengthening the hypothesis that this is a property of the objective rather than an artifact % of a specific dataset. -The data shows that \textsc{Roll}-AOC is a loss robust to label noise. In KEEL, we see that under the heaviest noise rate, \textsc{Roll}-AOC degrades least. In CIFAR-10N, \textsc{Roll}-AOC suffers the least as label noise gets worse. Across other large datasets, \textsc{Roll}-AOC shows to be either competitive or leading. The experiments used a wide variety of data, both tabular and image-based, with varying levels of noise. +The data shows that \textsc{ROLL}-AOC is a loss robust to label noise. In KEEL, we see that under the heaviest noise rate, \textsc{ROLL}-AOC degrades least. In CIFAR-10N, \textsc{ROLL}-AOC suffers the least as label noise gets worse. Across other large datasets, \textsc{ROLL}-AOC shows to be either competitive or leading. The experiments used a wide variety of data, both tabular and image-based, with varying levels of noise. -Having the ability to optimize the desired metric (AUC) directly, rather than in a round about way, allows +Having the ability to optimize the desired metric (AUC) directly, rather than in a roundabout way, allows our method to overcome challenging data. When using a per-sample loss, optimizing for a mislabeled sample might decrease the overall metric, but would do so at the cost of the performance of the model. Using AUC based methods allows this to be factored in already at the point of gradient weighting, so that the importance @@ -34,7 +34,7 @@ the fitted score distribution by $1/n$, diluting its influence smoothly across t than producing a concentrated gradient outlier as occurs in per-sample losses. This noise-robustness pattern is consistent across all three benchmarks used in this work: KEEL (controlled -asymmetric label noise), CIFAR-10N (real-world human annotation noise), and ChestXray14 (structural noise +symmetric label noise), CIFAR-10N (real-world human annotation noise), and ChestXray14 (structural noise from NLP-extracted labels). The same advantage appearing across tabular data, natural images, and medical images strengthens the case that noise robustness is a structural property of the distributional objective, rather than an artifact of any particular dataset or noise type. @@ -44,12 +44,12 @@ rather than an artifact of any particular dataset or noise type. \addcontentsline{tocheb}{subsection}{\protect\numberline{\subsecnumforhebrewtoc}{ROLL-AOC מול GCE ו-MAE}} \textsc{GCE} and \textsc{MAE} are used specifically for their robustness to noise, yet under 20\% -asymmetric noise on KEEL both degrade more than \textsc{Roll}-AOC (\textsc{GCE}: $-0.056$, -\textsc{MAE}: $-0.071$, \textsc{Roll}-AOC: $-0.033$). Whether this reflects a need for domain-specific -re-calibration or a fundamental limitation of per-sample objectives under label noise, \textsc{Roll}-AOC +symmetric noise on KEEL both degrade more than \textsc{ROLL}-AOC (\textsc{GCE}: $-0.056$, +\textsc{MAE}: $-0.071$, \textsc{ROLL}-AOC: $-0.033$). Whether this reflects a need for domain-specific +re-calibration or a fundamental limitation of per-sample objectives under label noise, \textsc{ROLL}-AOC overcomes these challenges without any noise-specific adjustment. -\subsection{\textsc{Roll}-AOC Versus \textsc{LibAUC}: Stability Under Noise} +\subsection{\textsc{ROLL}-AOC Versus \textsc{LibAUC}: Stability Under Noise} \label{sec:discussion:libauc} \addcontentsline{tocheb}{subsection}{\protect\numberline{\subsecnumforhebrewtoc}{ROLL-AOC מול LibAUC: יציבות תחת רעש}} @@ -66,8 +66,8 @@ overcomes these challenges without any noise-specific adjustment. % Frame as a practical stability advantage: in noisy or heterogeneous settings, % PESG-based optimization may be brittle in ways that a standard Adam + KDE loss is not. -While \textsc{Roll}-AOC shows improvement over per-sample losses on noisy datasets, it also outperforms -our LibAUC baseline, which also optimizes for AUC directly. +While \textsc{ROLL}-AOC shows improvement over per-sample losses on noisy datasets, it also outperforms +the LibAUC baseline, which also optimizes for AUC directly. This advantage can be explained by how closely each method's loss value tracks the actual AUC. LibAUC replaces the intractable pairwise indicator $\mathbf{1}[f(x^+) > f(x^-)]$ with a squared surrogate: @@ -76,15 +76,15 @@ LibAUC replaces the intractable pairwise indicator $\mathbf{1}[f(x^+) > f(x^-)]$ \] The squared surrogate and the indicator it replaces follow fundamentally different patterns: the squared loss is sensitive to the magnitude of score differences, not only to their sign. As a result, the aggregate -loss value gives little information about the AUC actually achieved --- a model with large score margins +loss value gives little information about the AUC actually achieved, a model with large score margins on easy pairs can have a low surrogate loss yet mediocre ranking quality on hard pairs. -\textsc{Roll}-AOC does not track AUC exactly either, but its error is bounded. The loss is the numerical +\textsc{ROLL}-AOC does not track AUC exactly either, but its error is bounded. The loss is the numerical integral of $1 - \operatorname{TPR}(\alpha)$ over 99 FPR grid points, directly approximating the area over the ROC curve ($\mathrm{AOC} = 1 - \mathrm{AUC}$). As long as the KDE bandwidth is well-estimated and 99 thresholds provide sufficient coverage, the loss value stays close to the true AOC. The practical consequence is twofold: gradient weighting is more accurately aligned with ranking quality, -and best-epoch model selection --- which picks the epoch with the lowest loss --- selects the epoch with +and best-epoch model selection, which picks the epoch with the lowest loss, selects the epoch with the highest estimated AUC rather than the lowest surrogate value. \subsection{Failure of the Single-Threshold Variant (\textsc{Roll}-TPR)} @@ -107,7 +107,7 @@ the highest estimated AUC rather than the lowest surrogate value. % threshold distribution and annealing toward the target operating point — or curriculum % strategies that introduce the fixed threshold gradually. -The \textsc{Roll} framework allows for a variant that specifically optimizes for a single FPR or TPR instead of the whole curve. This would allow models to trade off performance in certain parts of the curve for performance at a given threshold. Optimization such as this is motivated. Sometimes only a specific working point is interesting, and optimizing for it directly is desired. +The \textsc{ROLL} framework allows for a variant that specifically optimizes for a single FPR or TPR instead of the whole curve. This would allow models to trade off performance in certain parts of the curve for performance at a given threshold. Optimization such as this is motivated. Sometimes only a specific working point is interesting, and optimizing for it directly is desired. In experimentation, this did not work as well as anticipated. While on the train set, optimization was achieved for a certain working point, this benefit rarely translated to the test or even the validation set. @@ -118,7 +118,7 @@ of its score space, which exacerbated already existing overfitting issues and ma to random initialization. On datasets such as Wisconsin and New-Thyroid1, this manifested as extreme run-to-run variance ($\pm 0.441$ and $\pm 0.241$ AUC respectively under clean labels), where the outcome was determined more by initialization than by the optimization itself. -This stands in direct contrast to \textsc{Roll}-AOC: as noted in \Cref{rem:aoc-coverage}, the 99 +This stands in direct contrast to \textsc{ROLL}-AOC: as noted in \Cref{rem:aoc-coverage}, the 99 simultaneously active thresholds spread gradient coverage across the full score range, making convergence far more robust to initialization. @@ -142,7 +142,7 @@ far more robust to initialization. % computation and gradient derivations carry through with per-sample bandwidths with only % local changes to the kernel evaluation. -The \textsc{Roll} framework relies on good bandwidth estimation throughout training. +The \textsc{ROLL} framework relies on good bandwidth estimation throughout training. As described in \Cref{sec:kde-bandwidth}, the Improved Sheather-Jones (ISJ) estimator is the primary bandwidth selection method, with Silverman's rule as a closed-form fallback. In practice, ISJ is sensitive to small or degenerate batches early in training: when scores are not yet @@ -175,13 +175,13 @@ can perform poorly wherever the score distribution exhibits multiple scales of v The number of FPR thresholds evaluated per forward pass is itself a trade-off. More thresholds yield a finer approximation of the ROC curve and, as noted in \Cref{rem:aoc-coverage}, denser gradient coverage -across the score space --- both of which contribute to the robustness of \textsc{Roll}-AOC. Fewer +across the score space, both of which contribute to the robustness of \textsc{ROLL}-AOC. Fewer thresholds reduce training cost proportionally but produce a coarser AUC estimate and sparser gradient signal. Each threshold evaluation requires a KDE CDF query and a bisection inversion, each $O(n)$ in the batch size, making the total per-batch cost $O(kn)$ where $k$ is the number of thresholds. In the regime used in this work ($k = 99$) the cost is manageable, but it is a genuine overhead relative to -per-sample losses such as BCE, focal, and GCE (all $O(n)$). Treating $k$ as a hyperparameter --- for -example, using $k = 25$ during early training and increasing it as the model stabilises --- is a +per-sample losses such as BCE, focal, and GCE (all $O(n)$). Treating $k$ as a hyperparameter, for +example, using $k = 25$ during early training and increasing it as the model stabilises, is a straightforward mitigation that has not yet been explored. % ----------------------------------------------------------------------- @@ -209,7 +209,7 @@ straightforward mitigation that has not yet been explored. In this work, we present \textsc{ROLL} as a general framework for differentiable, ROC-based optimization via parametric score distribution fitting. As the primary instantiation, we develop the KDE-based variant and provide its full forward pass and gradient derivations. \textsc{ROLL}-AOC directly integrates the -estimated ROC curve over 99 evenly-spaced FPR thresholds --- a fundamentally different approach from +estimated ROC curve over 99 evenly-spaced FPR thresholds, a fundamentally different approach from \textsc{LibAUC}'s pairwise formulation, which optimizes a squared surrogate over score pairs rather than the curve itself. @@ -222,9 +222,9 @@ design objective. \label{sec:conclusions:limitations} \addcontentsline{tocheb}{subsection}{\protect\numberline{\subsecnumforhebrewtoc}{מגבלות}} -\textsc{Roll}-AOC has several practical limitations worth noting. +\textsc{ROLL}-AOC has several practical limitations worth noting. The KDE instantiation uses a single bandwidth per class, which cannot accurately represent multimodal or heavy-tailed score distributions. -Computational cost is significant: evaluating 99 FPR thresholds per forward pass makes \textsc{Roll}-AOC considerably more expensive per training step than per-sample losses or \textsc{LibAUC}'s pairwise formulation. +Computational cost is significant: evaluating 99 FPR thresholds per forward pass makes \textsc{ROLL}-AOC considerably more expensive per training step than per-sample losses or \textsc{LibAUC}'s pairwise formulation. The single-threshold variant (\textsc{Roll}-TPR) is brittle in practice; gradient locality concentrates updates near one operating point and the benefit rarely transfers to held-out data. Finally, the framework currently lacks formal convergence guarantees; the empirical results are encouraging but not theoretically grounded. @@ -242,4 +242,4 @@ Variable-bandwidth (adaptive) KDE methods address the single-bandwidth limitatio Formal convergence guarantees for KDE-ROLL under standard assumptions (batch size, bandwidth decay schedule, smoothness) remain an open problem, and would also clarify how the bandwidth scheduler interacts with gradient variance across training. \paragraph{Partial and weighted AUC.} -Restricting or reweighting the FPR grid to emphasize specific operating regions is a straightforward modification to \textsc{Roll}-AOC, and a natural fit for settings such as medical imaging where only a narrow FPR range is clinically actionable. +Restricting or reweighting the FPR grid to emphasize specific operating regions is a straightforward modification to \textsc{ROLL}-AOC, and a natural fit for settings such as medical imaging where only a narrow FPR range is clinically actionable. diff --git a/content/experiments/experiments.tex b/content/experiments/experiments.tex index 3ac0c34..2d47f71 100644 --- a/content/experiments/experiments.tex +++ b/content/experiments/experiments.tex @@ -31,7 +31,7 @@ Yeast3 & 1484 & 163 &1321 & 8.1 & Yeast protein localisation (UCI) \\ \end{table} \paragraph{CIFAR-10N.} -CIFAR-10 is a widely used image dataset of 60{,}000 32x32 color images belonging to 10 mutually exclusive classes. CIFAR-10N~\cite{wei2022cifar10n} augments the 50{,}000 training set labels with real human-annotated noisy labels collected from Amazon Mechanical Turk~\cite{wei2022cifar10n}. We form a binary task (class~1, automobile, vs.\ rest; IR~$\approx 9$:1) and train under three noise conditions: +CIFAR-10 is a widely used image dataset of 60{,}000 $32\times32$ colour images belonging to 10 mutually exclusive classes. CIFAR-10N~\cite{wei2022cifar10n} augments the 50{,}000 training set labels with real human-annotated noisy labels collected from Amazon Mechanical Turk. We form a binary task (class~1, automobile, vs.\ rest; IR~$\approx 9$:1) and train under three noise conditions: \emph{clean} (original CIFAR-10 labels, 0\% noise), \emph{aggre} (majority-vote aggregate of crowdworker annotations, $\approx$9\% noise), and \emph{worse} (single worst-annotator labels, $\approx$40\% noise). @@ -39,7 +39,7 @@ This allows us to measure the degradation of the model's performance under incre The architecture tested is a three-block convolutional network trained from scratch. \paragraph{Large datasets.} -We include four large-scale datasets spanning diverse imbalanced real-world settings. +We include five large-scale datasets spanning diverse imbalanced real-world settings. \begin{itemize} \item \textbf{Bank Marketing}~\cite{moro2014bank}, UCI repository; Portuguese bank telephone marketing campaigns (Moro et al., 2014). $n = 41{,}188$ samples, 48 features after one-hot encoding. Positive class: client subscribed to term deposit. IR~$\approx 8$:1 ($\approx$11.3\% positive). No label noise. @@ -48,7 +48,9 @@ We include four large-scale datasets spanning diverse imbalanced real-world sett \item \textbf{Home Credit}~\cite{homecredit2018kaggle}, Kaggle Home Credit Default Risk competition (2018). $n = 307{,}511$ loan applications, $\approx$120 numerical features from the primary application table. Positive class: loan default. IR~$\approx 11.4$:1 ($\approx$8\% default rate). Labels are proxy labels (repayment outcome); no injected noise. - \item \textbf{NIH ChestX-ray14}~\cite{wang2017chestxray14}, a large-scale medical chest radiograph dataset of $112{,}120$ X-rays from $30{,}805$ patients annotated with 14 pathology labels. Labels were extracted from radiological reports via NLP; the dataset authors estimate accuracy at $>$90\%~\cite{wang2017chestxray14}, though subsequent audits have found the true error rate substantially higher~\cite{oakdenrayner2020exploring}, making this a naturally noisy benchmark. We form a binary task targeting \emph{pleural effusion} detection (Effusion vs.\ rest; IR~$\approx 7.5$:1). Unlike the other three datasets, inputs here are not raw tabular features: we extract $1{,}024$-dimensional latent representations using a \mbox{DenseNet-121} pretrained on the dataset via the TorchXRayVision library~\cite{cohen2022torchxrayvision}, with global average pooling applied to the penultimate feature map, and pass these frozen embeddings to a lightweight $1024{\to}512{\to}128{\to}1$ MLP with dropout. This transfer-learning setup lets us evaluate ROLL without GPU-intensive end-to-end training. + \item \textbf{NIH ChestXray14}~\cite{wang2017chestxray14}, a large-scale medical chest radiograph dataset of $112{,}120$ X-rays from $30{,}805$ patients annotated with 14 pathology labels. Labels were extracted from radiological reports via NLP; the dataset authors estimate accuracy at $>$90\%~\cite{wang2017chestxray14}, though subsequent audits have found the true error rate substantially higher~\cite{oakdenrayner2020exploring}, making this a naturally noisy benchmark. We form a binary task targeting \emph{pleural effusion} detection (Effusion vs.\ rest; IR~$\approx 7.5$:1). Unlike the other three datasets, inputs here are not raw tabular features: we extract $1{,}024$-dimensional latent representations using a \mbox{DenseNet-121} pretrained on the dataset via the TorchXRayVision library~\cite{cohen2022torchxrayvision}, with global average pooling applied to the penultimate feature map, and pass these frozen embeddings to a lightweight $1024{\to}512{\to}128{\to}1$ MLP with dropout. This transfer-learning setup lets us evaluate ROLL without GPU-intensive end-to-end training. + + \item \textbf{Food-101N}~\cite{lee2018cleannet}, a large-scale web-crawled image dataset of $\approx$310{,}000 images spanning 101 food categories, with $\approx$18.4\% label noise from automated web collection. We form a binary task (class~0 vs.\ rest; IR~$\approx$100:1) and train a convolutional network end-to-end on 64$\times$64 RGB images, making this the only large-scale benchmark where ROLL operates on raw pixel inputs rather than pre-extracted features. \end{itemize} \section{Performance Measures}\label{sec:experiments:performance_measures} @@ -60,7 +62,7 @@ We report the \emph{area under the ROC curve} (AUC) averaged over independent tr \label{sec:experiments:configurations} \addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{קונפיגורציה ופרמטרים}} -We compare \textsc{Roll}-AOC (the primary proposed loss, optimising the area under the ROC curve via KDE) against the following baselines: +We compare \textsc{ROLL}-AOC (the primary proposed loss, optimising the area under the ROC curve via KDE) against the following baselines: \begin{itemize} \item \textbf{BCE-W}: binary cross-entropy with class-frequency inverse weighting. @@ -71,7 +73,7 @@ We compare \textsc{Roll}-AOC (the primary proposed loss, optimising the area und \item \textbf{MAE}: mean absolute error (symmetric noise-robust loss). \end{itemize} -All methods use the same network architecture, optimiser (Adam for non-LibAUC methods), and early stopping. The best model per-episode is chosen from the epoch with lowest loss. Each configuration is run for $n = 5$ independent episodes on KEEL and large datasets, and $n = 3$ on CIFAR-10N. Reported AUC is evaluated on the test split, averaged between episodes. +All methods use the same network architecture, optimiser (Adam for non-LibAUC methods), and early stopping. The best model per-episode is chosen from the epoch with lowest loss. Each configuration is run for $n = 5$ independent episodes on KEEL and the tabular large datasets, $n = 3$ on CIFAR-10N and Food-101N. Reported AUC is evaluated on the test split, averaged between episodes. \section{Results}\label{sec:experiments:results} \addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{תוצאות}} @@ -81,17 +83,17 @@ All methods use the same network architecture, optimiser (Adam for non-LibAUC me \Cref{tab:results:keel} reports mean test AUC across the 12 KEEL datasets under clean labels. No single method dominates: on easy datasets (Iris0, Wisconsin, Vowel0, Vehicle2, Glass6) nearly all methods converge to similar AUCs, leaving little room for differentiation. On the harder, more imbalanced datasets the picture is more varied. -\textsc{Roll}-AOC competes well but does not claim the top position on most individual datasets, establishing it as a competitive approach. +\textsc{ROLL}-AOC competes well but does not claim the top position on most individual datasets, establishing it as a competitive approach. To assess label-noise robustness, we inject symmetric label noise. A fraction of all class labels are randomly flipped. We evaluate at noise levels of $5\%, 10\%, 20\%$, running 10 independent episodes per dataset per noise level. The randomly flipped labels change each episode, yet are consistent between configurations, ensuring every configuration, per episode, sees the same labels, changed in the same way. Full per-dataset results at 5\% and 10\% noise are given in \Cref{appendix:keel-noise} (\Cref{tab:results:keel-noise-05,tab:results:keel-noise-10}); the 20\% results, where differentiation is strongest, are shown in \Cref{tab:results:keel-noise-20} below. -At $0\%$ noise, results are quite similar across configurations. \textsc{Roll}-AOC trails right behind the leader in most configurations. Since datasets are small, the \textbf{std} given for each point is relatively high, and quite a bit of variability is expected. Still, with few exceptions, results between configurations are within throwing distance of each other. -As noise increases the ranking shifts markedly: at $20\%$ noise \textsc{Roll}-AOC reaches the top on average (0.847), while traditionally noise-robust losses --- \textsc{GCE} (0.831) and \textsc{MAE} (0.804) --- degrade substantially. -The per-dataset results in \Cref{tab:results:keel-noise-20} reveal that this trend is consistent across most datasets rather than driven by a small subset: \textsc{Roll}-AOC leads on 6 of 12 datasets (Glass0, Glass1, Glass2, Pima, Vehicle2, Yeast3), while no other method leads on more than 3. -From $0\%$ to $20\%$ noise, \textsc{Roll}-AOC drops 0.033 in mean AUC (3.7\%), compared to 0.056 for \textsc{GCE} (6.3\%), 0.071 for \textsc{MAE} (8.1\%), and 0.073 for \textsc{LibAUC} (8.3\%). +At $0\%$ noise, results are quite similar across configurations. \textsc{ROLL}-AOC trails right behind the leader in most configurations. Since datasets are small, the \textbf{std} given for each point is relatively high, and quite a bit of variability is expected. Still, with few exceptions, results between configurations are within one standard deviation of each other. +As noise increases the ranking shifts markedly: at $20\%$ noise \textsc{ROLL}-AOC reaches the top on average (0.847), while traditionally noise-robust losses, \textsc{GCE} (0.831) and \textsc{MAE} (0.804), degrade substantially. +The per-dataset results in \Cref{tab:results:keel-noise-20} reveal that this trend is consistent across most datasets rather than driven by a small subset: \textsc{ROLL}-AOC leads on 6 of 12 datasets (Glass0, Glass1, Glass2, Pima, Vehicle2, Yeast3), while no other method leads on more than 3. +From $0\%$ to $20\%$ noise, \textsc{ROLL}-AOC drops 0.033 in mean AUC (3.7\%), compared to 0.056 for \textsc{GCE} (6.3\%), 0.071 for \textsc{MAE} (8.1\%), and 0.073 for \textsc{LibAUC} (7.7\%). This supports the hypothesis that ROLL's distributional objective provides implicit noise robustness: mislabelled samples shift the fitted score distribution gradually rather than producing isolated large-gradient outliers. \begin{table}[htbp] @@ -101,7 +103,7 @@ This supports the hypothesis that ROLL's distributional objective provides impli \resizebox{\textwidth}{!}{% \begin{tabular}{lccccccc} \toprule -Dataset & \textsc{Roll}-AOC & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\ +Dataset & \textsc{ROLL}-AOC & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\ \midrule Glass0 & 0.796{\tiny{$\pm$0.046}} & 0.839{\tiny{$\pm$0.033}} & 0.802{\tiny{$\pm$0.020}} & 0.814{\tiny{$\pm$0.034}} & 0.818{\tiny{$\pm$0.032}} & \textbf{0.843{\tiny{$\pm$0.035}}} & 0.826{\tiny{$\pm$0.037}} \\ Glass1 & 0.722{\tiny{$\pm$0.033}} & 0.710{\tiny{$\pm$0.018}} & 0.712{\tiny{$\pm$0.036}} & 0.744{\tiny{$\pm$0.044}} & 0.742{\tiny{$\pm$0.039}} & 0.723{\tiny{$\pm$0.054}} & \textbf{0.769{\tiny{$\pm$0.031}}} \\ @@ -122,12 +124,12 @@ Yeast3 & 0.906{\tiny{$\pm$0.024}} & 0.894{\tiny{$\pm$0.008}} & 0.908{\tiny{$\pm$ \begin{table}[htbp] \centering -\caption{Mean test AUC ($\pm$\,std, $n=10$ episodes) on KEEL datasets at $\epsilon=20\%$ asymmetric label noise. Best per row in \textbf{bold}.} +\caption{Mean test AUC ($\pm$\,std, $n=10$ episodes) on KEEL datasets at $\epsilon=20\%$ symmetric label noise. Best per row in \textbf{bold}.} \label{tab:results:keel-noise-20} \resizebox{\textwidth}{!}{% \begin{tabular}{lccccccc} \toprule -Dataset & \textsc{Roll}-AOC & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\ +Dataset & \textsc{ROLL}-AOC & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\ \midrule Glass0 & \textbf{0.801{\tiny{$\pm$0.042}}} & 0.767{\tiny{$\pm$0.042}} & 0.796{\tiny{$\pm$0.040}} & 0.793{\tiny{$\pm$0.051}} & 0.765{\tiny{$\pm$0.052}} & 0.745{\tiny{$\pm$0.089}} & 0.791{\tiny{$\pm$0.044}} \\ Glass1 & \textbf{0.725{\tiny{$\pm$0.036}}} & 0.718{\tiny{$\pm$0.054}} & 0.714{\tiny{$\pm$0.036}} & 0.700{\tiny{$\pm$0.039}} & 0.706{\tiny{$\pm$0.041}} & 0.635{\tiny{$\pm$0.052}} & 0.689{\tiny{$\pm$0.060}} \\ @@ -148,7 +150,7 @@ Yeast3 & \textbf{0.879{\tiny{$\pm$0.032}}} & 0.861{\tiny{$\pm$0.070}} & 0. \subsection{CIFAR-10N: Label-Noise Robustness}\label{sec:results:cifar10n} -\Cref{tab:results:cifar10n} shows AUC under three noise levels on CIFAR-10N. Under \emph{clean} labels all methods reach AUC $\geq 0.972$, with \textsc{Roll}-AOC at the top (0.980). As noise increases the advantage of \textsc{Roll}-AOC grows: under \emph{aggre} noise it leads by 0.002 over the next-best method (GCE, 0.932) while other methods degrade more significantly, and under \emph{worse} noise, the margin widens to 0.012 over MAE (0.809) and 0.047 over BCE-W (0.738). Observing degradation relative to the clean scenario, \textsc{Roll}-AOC degraded the least ($-0.159$ relative to clean results), with MAE second ($-0.167$). +\Cref{tab:results:cifar10n} shows AUC under three noise levels on CIFAR-10N. Under \emph{clean} labels all methods reach AUC $\geq 0.972$, with \textsc{ROLL}-AOC at the top (0.980). As noise increases the advantage of \textsc{ROLL}-AOC grows: under \emph{aggre} noise it leads by 0.002 over the next-best method (GCE, 0.932) while other methods degrade more significantly, and under \emph{worse} noise, the margin widens to 0.012 over MAE (0.809) and 0.083 over BCE-W (0.738). Observing degradation relative to the clean scenario, \textsc{ROLL}-AOC degraded the least ($-0.159$ relative to clean results), with MAE second ($-0.167$). \begin{table}[htbp] \centering @@ -157,7 +159,7 @@ Yeast3 & \textbf{0.879{\tiny{$\pm$0.032}}} & 0.861{\tiny{$\pm$0.070}} & 0. \resizebox{\textwidth}{!}{% \begin{tabular}{lccccccc} \toprule -Noise & \textsc{Roll}-AOC & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\ +Noise & \textsc{ROLL}-AOC & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\ \midrule Clean & \textbf{0.980{\tiny{$\pm$0.001}}} & 0.977{\tiny{$\pm$0.002}} & 0.977{\tiny{$\pm$0.000}} & 0.972{\tiny{$\pm$0.003}} & 0.976{\tiny{$\pm$0.002}} & 0.979{\tiny{$\pm$0.000}} & 0.976{\tiny{$\pm$0.002}} \\ Aggre. & \textbf{0.934{\tiny{$\pm$0.002}}} & 0.924{\tiny{$\pm$0.003}} & 0.912{\tiny{$\pm$0.014}} & 0.910{\tiny{$\pm$0.004}} & 0.932{\tiny{$\pm$0.005}} & 0.921{\tiny{$\pm$0.004}} & 0.930{\tiny{$\pm$0.002}} \\ @@ -169,27 +171,30 @@ Worse & \textbf{0.821{\tiny{$\pm$0.001}}} & 0.738{\tiny{$\pm$0.009}} & 0.734{\ti \subsection{Large Datasets}\label{sec:results:large} -\Cref{tab:results:large} covers the four large-scale datasets. On Bank Marketing, LibAUC leads (0.936) with \textsc{Roll}-AOC close behind (0.931), and both substantially outperform all other baselines. On Credit Card Fraud \textsc{Roll}-AOC is the top performer (0.968), outperforming all baselines. On Home Credit \textsc{Roll}-AOC edges LibAUC (0.712 vs.\ 0.711). +\Cref{tab:results:large} covers the five large-scale datasets. On Bank Marketing, LibAUC leads (0.936) with \textsc{ROLL}-AOC close behind (0.931), and both substantially outperform all other baselines. On Credit Card Fraud \textsc{ROLL}-AOC is the top performer (0.968), outperforming all baselines. On Home Credit \textsc{ROLL}-AOC edges LibAUC (0.712 vs.\ 0.711). -NIH ChestX-ray14 is qualitatively distinct from the other three: inputs are frozen DenseNet-121 embeddings from radiograph images rather than raw tabular features, making this a transfer-learning evaluation under naturally noisy labels~\cite{oakdenrayner2020exploring}. -\textsc{Roll}-AOC achieves the highest AUC ($0.839\pm0.001$), a clear margin above the next-best method GCE ($0.829\pm0.004$). +NIH ChestXray14 is qualitatively distinct from the other three: inputs are frozen DenseNet-121 embeddings from radiograph images rather than raw tabular features, making this a transfer-learning evaluation under naturally noisy labels~\cite{oakdenrayner2020exploring}. +\textsc{ROLL}-AOC achieves the highest AUC ($0.839\pm0.001$), a clear margin above the next-best method GCE ($0.829\pm0.004$). Standard cross-entropy variants trail substantially: BCE-W reaches $0.788\pm0.008$, Focal $0.798\pm0.009$, and Asym.\ $0.729\pm0.007$. MAE ($0.810\pm0.001$) and LibAUC ($0.835\pm0.002$) both fall short despite their respective noise-robust and AUC-optimising designs. -The ${\approx}5$~pp gap between \textsc{Roll}-AOC and BCE-W, consistent with the controlled KEEL noise experiment (\Cref{sec:results:keel}) and CIFAR-10N (\Cref{sec:results:cifar10n}), confirms that the noise-robustness advantage holds on real-world label noise. The result also shows that \textsc{Roll}-AOC generalises beyond from-scratch training: with embeddings frozen, ROLL operates entirely within the MLP head, fitting score distributions over a fixed representation. +The ${\approx}5$~pp gap between \textsc{ROLL}-AOC and BCE-W, consistent with the controlled KEEL noise experiment (\Cref{sec:results:keel}) and CIFAR-10N (\Cref{sec:results:cifar10n}), confirms that the noise-robustness advantage holds on real-world label noise. The result also shows that \textsc{ROLL}-AOC generalises beyond from-scratch training: with embeddings frozen, ROLL operates entirely within the MLP head, fitting score distributions over a fixed representation. + +On Food-101N, \textsc{ROLL}-AOC again achieves the highest AUC ($0.867\pm0.004$), outperforming the next-best method LibAUC ($0.857\pm0.002$) by $\approx$1~pp. Classification-based losses (BCE-W, Focal, Asym.) cluster between 0.843 and 0.848, while the noise-robust losses MAE and GCE reach 0.855 and 0.842 respectively. Food-101N is the only benchmark where ROLL trains on raw pixel inputs end-to-end rather than pre-extracted features, and the lead over LibAUC persists, reinforcing that ROLL's advantage is not artefactual of a specific architecture or feature representation. \begin{table}[htbp] \centering -\caption{Mean test AUC ($\pm$\,std, $n=5$ episodes) on large real-world datasets. Best per row in \textbf{bold}. ChestX-ray14 uses frozen DenseNet-121 embeddings rather than raw tabular features.} +\caption{Mean test AUC ($\pm$\,std) on large real-world datasets ($n=5$ episodes; Food-101N and ChestXray14: $n=3$). Best per row in \textbf{bold}. ChestXray14 uses frozen DenseNet-121 embeddings; Food-101N trains end-to-end on raw pixels. LibAUC on Food-101N uses no sigmoid on model outputs.} \label{tab:results:large} \resizebox{\textwidth}{!}{% \begin{tabular}{lccccccc} \toprule -Dataset & \textsc{Roll}-AOC & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\ +Dataset & \textsc{ROLL}-AOC & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\ \midrule Bank Marketing & 0.931{\tiny{$\pm$0.002}} & 0.580{\tiny{$\pm$0.040}} & 0.573{\tiny{$\pm$0.044}} & 0.684{\tiny{$\pm$0.033}} & 0.746{\tiny{$\pm$0.022}} & \textbf{0.936{\tiny{$\pm$0.003}}} & 0.707{\tiny{$\pm$0.035}} \\ Credit Card Fraud & \textbf{0.968{\tiny{$\pm$0.001}}} & 0.949{\tiny{$\pm$0.015}} & 0.942{\tiny{$\pm$0.020}} & 0.940{\tiny{$\pm$0.013}} & 0.948{\tiny{$\pm$0.014}} & 0.966{\tiny{$\pm$0.006}} & 0.954{\tiny{$\pm$0.009}} \\ Home Credit & \textbf{0.712{\tiny{$\pm$0.001}}} & 0.693{\tiny{$\pm$0.002}} & 0.694{\tiny{$\pm$0.002}} & 0.688{\tiny{$\pm$0.003}} & 0.698{\tiny{$\pm$0.002}} & 0.711{\tiny{$\pm$0.001}} & 0.699{\tiny{$\pm$0.004}} \\ \midrule -ChestX-ray14 (Effusion) & \textbf{0.839{\tiny{$\pm$0.001}}} & 0.788{\tiny{$\pm$0.008}} & 0.798{\tiny{$\pm$0.009}} & 0.729{\tiny{$\pm$0.007}} & 0.829{\tiny{$\pm$0.004}} & 0.835{\tiny{$\pm$0.002}} & 0.810{\tiny{$\pm$0.001}} \\ +ChestXray14 (Effusion) & \textbf{0.839{\tiny{$\pm$0.001}}} & 0.788{\tiny{$\pm$0.008}} & 0.798{\tiny{$\pm$0.009}} & 0.729{\tiny{$\pm$0.007}} & 0.829{\tiny{$\pm$0.004}} & 0.835{\tiny{$\pm$0.002}} & 0.810{\tiny{$\pm$0.001}} \\ +Food-101N & \textbf{0.867{\tiny{$\pm$0.004}}} & 0.848{\tiny{$\pm$0.025}} & 0.847{\tiny{$\pm$0.009}} & 0.845{\tiny{$\pm$0.032}} & 0.842{\tiny{$\pm$0.006}} & 0.857{\tiny{$\pm$0.002}} & 0.855{\tiny{$\pm$0.013}} \\ \bottomrule \end{tabular}% } diff --git a/content/introduction/introduction.tex b/content/introduction/introduction.tex index 39603e4..f883888 100644 --- a/content/introduction/introduction.tex +++ b/content/introduction/introduction.tex @@ -1,10 +1,10 @@ \section{Problem Statement} -Real-world data is hardly ever perfect. Many real-world datasets suffer a myriad of problems. One such common problem is improper labeling. While we would like our models to learn based off of an exact ground truth, often the labels with which we train our classifiers have errors themselves. Label-noise plagues many datasets for a variety of reasons. In datasets such as CIFAR-10N, a benchmark of natural images across 10 object classes, labels were collected from crowdworker annotators, who are prone to mistakes. In ChestX-ray14, a dataset of medical chest X-rays, labels were given by a natural language processing algorithm, since labelling each image by hand was deemed impractical. Such automatic labeling gave way to relatively high label error rates. +Real-world data is hardly ever perfect. Many real-world datasets suffer a myriad of problems. One such common problem is improper labeling. While we would like our models to learn based on an exact ground truth, often the labels with which we train our classifiers have errors themselves. Label-noise plagues many datasets for a variety of reasons. In datasets such as CIFAR-10N, labels were collected from crowdworker annotators, who are often prone to mistakes. In ChestXray14, a dataset of medical chest X-rays, labels were given by a natural language processing algorithm, since labelling each image by hand was deemed impractical. Such automatic labeling gave way to relatively high label error rates. In other cases, labels may only be assigned to part of the data, due to high effort needed to label, or the labels might be based on other data entirely, causing some misalignment. -Gradient based learning methods, like those used to train neural networks, have trouble dealing with such datasets. Standard, per-sample losses (BCE, focal-loss), are confused easily by mislabeled points, as they produce a very large gradient which throws off training. Current attempts at noise-robust losses (GCE, MAE) help smooth these large gradients, yet still optimize on a per-sample basis, targeting per-sample accuracy rather than ranking quality --- leaving them misaligned with AUC even when they do reduce noise sensitivity. +Gradient based learning methods, like those used to train neural networks, have trouble dealing with such datasets. Mislabeled data points are often assigned a score that is very far from the label itself. While this makes sense, the data point is very different from the class it is assigned, this mismatch can create a very large corrective gradient for training, a corrective gradient steering the model in the wrong direction. Current attempts at noise-robust losses (GCE, MAE) help smooth these large gradients, yet still optimize on a per-sample basis, targeting per-sample accuracy rather than ranking quality. This leaves them misaligned with AUC even when they do reduce noise sensitivity. -Direct AUC optimization is a natural direction here: ranking-based objectives are determined by the relative ordering of scores rather than their absolute values, making them structurally less sensitive to isolated mislabeled samples. Per-sample losses optimize each sample independently, while AUC losses optimize sample ranking across the whole population, allowing the model to make principled tradeoffs even in the presence of label errors. +Direct AUC optimization is a natural direction here: ranking-based objectives are determined by the relative ordering of scores rather than their absolute values, making them less sensitive to isolated mislabeled samples. Per-sample losses optimize each sample independently, while AUC losses optimize sample ranking across the whole population, allowing the model to make principled tradeoffs even in the presence of label errors. \section{Contribution} diff --git a/content/method/method.tex b/content/method/method.tex index 8e6487e..7cae3ec 100644 --- a/content/method/method.tex +++ b/content/method/method.tex @@ -9,7 +9,7 @@ % This section establishes the notation and primary objective used throughout the chapter. Given a dataset $\mathcal{D} = \{(\mathbf{x}_i, y_i)\}_{i=1}^{n}$ with inputs $\mathbf{x}_i \in \mathcal{X}$ and binary labels $y_i \in \{0, 1\}$, we learn a -scoring function $f_\theta : \mathcal{X} \to \mathbb{R}$ by fitting. +scoring function $f_\theta : \mathcal{X} \to \mathbb{R}$ by minimising a surrogate loss on $\mathcal{D}$. The positive and negative subsets of $\mathcal{D}$ are: \[ \mathcal{D}_1 = \{(\mathbf{x}_i, y_i) \in \mathcal{D} \mid y_i = 1\}, @@ -46,12 +46,12 @@ surrogates for both the AUC objective \eqref{eq:objective-auc} and these point o \label{sec:roll-framework} In order to allow gradient-based learning methods to optimize AUC directly, we introduce -ROLL --- Rate Optimized Likelihood Loss. +ROLL, Rate Optimized Likelihood Loss. The core idea of ROLL is as follows: at each training step, differentiable distributions are fitted to the positive- and negative-class score outputs of $f_\theta$. AUC is then approximated by evaluating the estimated $\operatorname{TPR}$ at 99 evenly-spaced -$\operatorname{FPR}$ values across $(0,1)$ and averaging --- a numerical integration of +$\operatorname{FPR}$ values across $(0,1)$ and averaging, a numerical integration of the ROC curve. Because each per-threshold evaluation is differentiable through the fitted distributions, this AUC approximation can be maximized directly by gradient descent, with no threshold selection required at inference time. @@ -75,7 +75,7 @@ In practice, however, making these evaluations differentiable is not straightfor Directly optimizing TPR for FPR, or vice versa, is a non-differentiable problem. -For this sub-section, we will reffer to an example of maximizing TPR given a fixed FPR, +For this sub-section, we will refer to an example of maximizing TPR given a fixed FPR, though this can be reversed. For any loss $\mathcal{L}(\theta)$, in order @@ -99,7 +99,7 @@ by some $\varepsilon$ will not affect either TPR or FPR at all, and the derivati if $f_\theta(\mathbf{x}_i) = \tau$, the derivative of TPR/FPR w.r.t.\ $f_\theta(\mathbf{x}_i)$ becomes infinite. This derivative cannot be used to continuously fit our model. -This necessitates a different approach, to make these calculations differentiable w.r.t the scores themselves, in order to fit this type of optimization into a learning platform. +This necessitates a different approach, to make these calculations differentiable w.r.t.\ the scores themselves, in order to fit this type of optimization into a learning platform. \subsection{Score Distribution Fitting as a Solution} \label{sec:score-distribution-fitting} @@ -198,9 +198,8 @@ inner structure: each evaluates one empirical CDF at the $(1-\alpha)$-quantile o The two objectives are interchangeable at the implementation level: replacing $f_\theta(\mathbf{x})$ with $-f_\theta(\mathbf{x})$ and $y$ with $1 - y$ swaps the roles of the two populations, and passing target rate $1-\alpha$ to the resulting -TPR@FPR objective recovers the FPR@TPR objective at rate $\alpha$. Because -the two objectives are therefore interchangeable at the implementation level, all derivations -and implementation details below are given for \eqref{eq:roll-tpr-at-fpr} only; +TPR@FPR objective recovers the FPR@TPR objective at rate $\alpha$. All derivations +and implementation details below are therefore given for \eqref{eq:roll-tpr-at-fpr} only; \eqref{eq:roll-fpr-at-tpr} follows by the same transformation applied to the inputs, with the target rate complemented to $1-\alpha$. \Cref{fig:roll-principle} provides a geometric view of \eqref{eq:roll-tpr-at-fpr}. @@ -213,7 +212,7 @@ the target rate complemented to $1-\alpha$. threshold $\tau$ (dashed) is placed at the $(1-\alpha)$-quantile of the negative class, so that exactly $\alpha$ fraction of negative scores exceed it ($\operatorname{FPR}=\alpha$). The shaded area under the positive-class PDF to the left of $\tau$ is - $\hat{F}_1(\tau)$ --- the ROLL loss we minimize. Tick marks along the bottom represent + $\hat{F}_1(\tau)$, the ROLL loss we minimize. Tick marks along the bottom represent observed positive-class scores. \textit{Lower}: corresponding CDFs $\hat{F}_0$ (blue) and $\hat{F}_1$ (orange). Reading horizontally at height $1-\alpha$ on $\hat{F}_0$ gives $\tau$; reading $\hat{F}_1(\tau)$ off $\hat{F}_1$ at the same $\tau$ gives the @@ -406,7 +405,7 @@ The derivation is given in Appendix~\ref{appendix:kde-grad}; letting $\tau = \ha where $\tau = \hat{F}_0^{-1}(1-\alpha ; \mathcal{B}_0)$. -The explicit sigmoid-kernel form of \eqref{eq:kde-grad-combined} — forward pass CDF, efficient gradient, and implementation notes — is given in Appendix~\ref{appendix:kde-sigmoid}. +The explicit sigmoid-kernel form of \eqref{eq:kde-grad-combined}, forward pass CDF, efficient gradient, and implementation notes, is given in Appendix~\ref{appendix:kde-sigmoid}. Positive-class samples are weighted by their individual kernel PDF at $\tau$; negative-class samples are weighted by the mean positive-class kernel PDF scaled by each sample's share of the negative-class kernel mass. @@ -561,7 +560,7 @@ The scaling has no effect on the loss value itself or derived gradients. Silverman's rule derives the bandwidth proportionally to the data standard deviation, $h \propto \hat\sigma \cdot n^{-1/5}$, so dividing all scores by $\hat\sigma_\mathcal{B}$ causes the bandwidth to scale by the same factor. -The product $v \cdot (\tau - s_j)$ — the only quantity entering the sigmoid — is +The product $v \cdot (\tau - s_j)$ (the only quantity entering the sigmoid) is therefore invariant under the rescaling. In the backward pass, gradients with respect to the normalised scores are multiplied by $1/\hat\sigma_\mathcal{B}$ to apply the chain rule, providing gradients mathematically @@ -586,7 +585,7 @@ nearest-threshold samples. We estimate the bandwidth using the Improved Sheather-Jones (ISJ) method~\cite{botev2010kde}, which minimises the asymptotic mean integrated squared error via an iterative plug-in procedure. ISJ tracks the true score distribution more -closely than closed-form rules, as illustrated in \Cref{fig:density-fit-comparison}. +closely than closed-form rules. Because ISJ is iterative, it can fail to converge on small or degenerate batches early in training. In those cases we fall back to Silverman's rule of thumb~\cite{silverman1986density}, diff --git a/content/related_work/related_work.tex b/content/related_work/related_work.tex index e64b04b..9df82e7 100644 --- a/content/related_work/related_work.tex +++ b/content/related_work/related_work.tex @@ -30,11 +30,8 @@ \subsection{AUC and ROC definitions} -% TODO add formalizations -% Formalize ROC and integration into AUC - A Receiver Operating Characteristic (ROC) is a graph that represents the performance of a classifier. -The classifier in question is evaluated on binary-labeled data. The labeled data is ordered based off of the +The classifier in question is evaluated on binary-labeled data. The labeled data is ordered based on the scores given by the classifier. Given any numerical threshold, it is then easy to see which labeled data points fall above and below. For each threshold, the true positive rate and the false positive rate is given. The results for each threshold, and so for each possible TPR and FPR, are put on a graph. The resulting curve is the ROC curve. @@ -66,16 +63,16 @@ The AUC is a direct representation of the performance of a classifier, as oppose An example of this is a result for which all true samples receive a score of $0.51$ and all false samples receive a score of $0.49$. Using many losses, including BCE, this would result in a relatively high loss value. On this score however, the AUC is $1$. The model perfectly separates the samples, so further training is not necessary. -Direct AUC optimization can be ideal with optimizing for imbalanced datasets. With most losses such as BCE or focal, the loss is calculated per-sample and then summed (or averaged). In order to properly train for both true and false populations, the scores are weighted, so that equal weight falls on both the true and the false populations. With direct AUC optimization, the imbalance does not affect the calculation, and the reweighting can be forgone. +Direct AUC optimization can be ideal with optimizing for imbalanced datasets. With most losses, such as BCE or focal, the loss is calculated per-sample and then summed (or averaged). In order to properly train for both true and false populations, the scores are weighted, so that equal weight falls on both the true and the false populations. With direct AUC optimization, the imbalance does not affect the calculation, and the reweighting can be forgone. The noise-robustness properties of ranking-based objectives are discussed in \Cref{sec:noisy-labels}. \subsection{AUC optimization in literature} -While AUC as a metric for model performance has existed for a while, methods to optimize for it directly are relatively recent. At it's core, it is a non-trivial problem to solve. Existing efforts to directly optimize AUC have been based on the Wilcoxon-Mann-Whitney interpretation of AUC. However, directly calculating the aforementioned probability of a random positive score being higher than a negative one introduces two complications. The first is that one must iterate over $n^2$ pairs to cover every possible pair. Additionally, the pairwise function one must evaluate ($\mathbb{P}(f(x_1) > f(x_0))$) is a discontinuous stepwise function, and is therefore non-differentiable. +While AUC as a metric for model performance has existed for a while, methods to optimize for it directly are relatively recent. At its core, it is a non-trivial problem to solve. Existing efforts to directly optimize AUC have been based on the Wilcoxon-Mann-Whitney interpretation of AUC. However, directly calculating the aforementioned probability of a random positive score being higher than a negative one introduces two complications. The first is that one must iterate over $n^2$ pairs to cover every possible pair. Additionally, the pairwise function one must evaluate ($\mathbb{P}(f(x_1) > f(x_0))$) is a discontinuous stepwise function, and is therefore non-differentiable. The discontinuous stepwise nature of the evaluated pairwise function is generally addressed by using a surrogate, either sigmoid-like or hinge functions are applied to the score difference~\cite{cortes2003auc}. -\citet{yuan2021auc} proposed a compositional stochastic gradient framework that reformulates AUC maximization as a min-max saddle-point problem, enabling mini-batch training without maintaining all pairwise scores in memory. Their PESG optimizer and the LibAUC library are the primary practical tools for deep AUC maximization and serve as the main AUC baseline in this thesis. +\citet{yuan2021auc} proposed a formulation of the loss that, using specific pairwise surrogate losses, allows training without separately evaluating every pair, but rather, evaluating all given samples once. Their PESG optimizer and the LibAUC library are the primary practical tools for deep AUC maximization and serve as the main AUC baseline in this thesis. % ============================================================ % PLACEMENT: Section 2.2, after ROC/AUC. Core differentiating @@ -92,7 +89,7 @@ Many data collection methods lead to imperfect labeling, resulting in a muddy gr This could be due to a number of factors. The labels might be based on human labeling, which is error prone. The labeling might be true, yet the signal might be noisy or wrong, creating a mismatch between signal and label. The labeling could be a resource-intensive process, and so it might only be done for a portion of the data. -And in some cases, the labeling is done based off a different learned model, which too could be faulty. +And in some cases, the labeling is done based on a different learned model, which too could be faulty. \subsection{Label noise formalization} @@ -319,7 +316,7 @@ which maximizes the \emph{likelihood ratio test}: \] Where $\phi^*(\mathbf{x})$ is our theoretical classifier, and $\eta \geq 0$ is chosen so that the FPR equals exactly $\alpha$. -Given these assumptions, and the target FPR that gives $\eta$, the classifier$\phi^*$ is proven by the lemma to be optimal. +Given these assumptions, and the target FPR that gives $\eta$, the classifier~$\phi^*$ is proven by the lemma to be optimal. \subsection{The practical gap} diff --git a/main.tex b/main.tex index 0fda90d..51320cc 100644 --- a/main.tex +++ b/main.tex @@ -61,6 +61,7 @@ \newtheorem{proposition}[theorem]{Proposition} \newtheorem{observation}[theorem]{Observation} \newtheorem{assumption}[theorem]{Assumption} +\newtheorem{remark}[theorem]{Remark} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %proof definition: @@ -165,6 +166,7 @@ \Crefname{observation}{Observation}{Observations} \Crefname{assumption}{Assumption}{Assumptions} +\Crefname{remark}{Remark}{Remarks} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %For Hebrew table of contents: (\addcontentsline is used to add section and subsections to the Hebrew toc. For some reason, the page number is reversed only for chapter, thus we use \addtocontents for them) \makeatletter @@ -271,7 +273,7 @@ \addtocontents{tocheb}{\contentsline{chapter}{\protect\numberline{נספח ג}{\quad\qquadנגזרת KDE של ROLL}}{{\beginL\thepage\endL}}} \input{content/appendixs/appendix_kde_grad} -\chapter{KDE ROLL: Sigmoid Kernel — Explicit Equations} +\chapter{KDE ROLL: Sigmoid Kernel - Explicit Equations} \label{appendix:kde-sigmoid} \addtocontents{tocheb}{\contentsline{chapter}{\protect\numberline{נספח ד}{\quad\qquadKDE ROLL: ניסוח מפורש עם גרעין סיגמואיד}}{{\beginL\thepage\endL}}} \input{content/appendixs/appendix_kde_sigmoid}