Compare commits
16 Commits
5933cda8c2
..
master
| Author | SHA1 | Date | |
|---|---|---|---|
 aner
|
86291e948e | ||
|
aner
|
1a8a31c158 | ||
|
aner
|
a85bb2a551 | ||
|
aner
|
ce726d938f | ||
|
aner
|
96d74ef901 | ||
|
aner
|
a078f44838 | ||
|
aner
|
c142ff98f4 | ||
|
aner
|
21f0427fba | ||
|
aner
|
e57efde395 | ||
|
aner
|
fa20fe0eb7 | ||
|
aner
|
10f1bd53f1 | ||
|
aner
|
7ff2ef6835 | ||
|
aner
|
a82885fe2d | ||
|
aner
|
32aa829603 | ||
|
aner
|
342efd836d | ||
|
aner
|
06687667d5 |
@@ -0,0 +1,2 @@
|
||||
((latex-mode . ((TeX-engine . xetex)
|
||||
(TeX-master . "/home/aner/projects/research-projects/roll/thesis/main.tex"))))
|
||||
@@ -25,3 +25,6 @@ _region_.prv/
|
||||
|
||||
# Auto-downloaded LaTeX package files
|
||||
*.sty
|
||||
*.pdf
|
||||
.#*
|
||||
*.xdv
|
||||
|
||||
@@ -1,3 +1,52 @@
|
||||
@article{tong2016neyman,
|
||||
title={A survey on {Neyman--Pearson} classification and suggestions for future research},
|
||||
author={Tong, Xin and Feng, Yang and Zhao, Anqi},
|
||||
journal={{WIREs} Computational Statistics},
|
||||
volume={8},
|
||||
pages={64--81},
|
||||
year={2016},
|
||||
doi={10.1002/wics.1376},
|
||||
publisher={Wiley}
|
||||
}
|
||||
|
||||
@book{silverman1986density,
|
||||
title={Density Estimation for Statistics and Data Analysis},
|
||||
author={Silverman, B. W.},
|
||||
year={1986},
|
||||
publisher={Chapman and Hall},
|
||||
address={London},
|
||||
series={Chapman \& Hall/CRC Monographs on Statistics and Applied Probability}
|
||||
}
|
||||
|
||||
@article{botev2010kde,
|
||||
title={Kernel density estimation via diffusion},
|
||||
author={Botev, Z. I. and Grotowski, J. F. and Kroese, D. P.},
|
||||
journal={The Annals of Statistics},
|
||||
volume={38},
|
||||
number={5},
|
||||
pages={2916--2957},
|
||||
year={2010},
|
||||
doi={10.1214/10-aos799},
|
||||
publisher={Institute of Mathematical Statistics}
|
||||
}
|
||||
|
||||
@book{press2007numerical,
|
||||
title={Numerical Recipes: The Art of Scientific Computing},
|
||||
author={Press, William H. and Teukolsky, Saul A. and Vetterling, William T. and Flannery, Brian P.},
|
||||
edition={3rd},
|
||||
year={2007},
|
||||
publisher={Cambridge University Press},
|
||||
address={Cambridge},
|
||||
isbn={978-0-521-88068-8}
|
||||
}
|
||||
|
||||
@article{chen2022beta,
|
||||
title={Novel closed-form point estimators for the beta distribution},
|
||||
author={Chen, Piao and Xiao, Xun},
|
||||
journal={arXiv preprint arXiv:2210.05536},
|
||||
year={2022}
|
||||
}
|
||||
|
||||
@article{demaine2007jigsaw,
|
||||
title={Jigsaw puzzles, edge matching, and polyomino packing: Connections and complexity},
|
||||
author={Demaine, Erik D. and Demaine, Martin L.},
|
||||
@@ -9,6 +58,25 @@
|
||||
publisher={Springer}
|
||||
}
|
||||
|
||||
@article{xie2024wsauc,
|
||||
title={Weakly Supervised {AUC} Optimization: A Unified Partial {AUC} Approach},
|
||||
author={Xie, Zheng and Liu, Yu and He, Hao-Yuan and Li, Ming and Zhou, Zhi-Hua},
|
||||
journal={IEEE Transactions on Pattern Analysis and Machine Intelligence},
|
||||
year={2024},
|
||||
doi={10.1109/TPAMI.2024.3357814},
|
||||
eprint={2305.14258},
|
||||
archivePrefix={arXiv}
|
||||
}
|
||||
|
||||
@inproceedings{wu2025preserving,
|
||||
title={Preserving {AUC} Fairness in Learning with Noisy Protected Groups},
|
||||
author={Wu, Mingyang and Lin, Li and Zhang, Wenbin and Wang, Xin and Yang, Zhenhuan and Hu, Shu},
|
||||
booktitle={Proceedings of the 42nd International Conference on Machine Learning},
|
||||
year={2025},
|
||||
eprint={2505.18532},
|
||||
archivePrefix={arXiv}
|
||||
}
|
||||
|
||||
@article{altman1989solving,
|
||||
title={Solving the jigsaw puzzle problem in linear time},
|
||||
author={Altman, Tom},
|
||||
@@ -19,3 +87,74 @@
|
||||
year={1989},
|
||||
publisher={Taylor \& Francis}
|
||||
}
|
||||
|
||||
@inproceedings{ghosh2017robust,
|
||||
title={Robust Loss Functions under Label Noise for Deep Neural Networks},
|
||||
author={Ghosh, Aritra and Kumar, Himanshu and Sastry, P. S.},
|
||||
booktitle={Proceedings of the Thirty-First {AAAI} Conference on Artificial Intelligence},
|
||||
pages={1919--1925},
|
||||
year={2017}
|
||||
}
|
||||
|
||||
@inproceedings{arpit2017memorization,
|
||||
title={A Closer Look at Memorization in Deep Networks},
|
||||
author={Arpit, Devansh and Jastrz\k{e}bski, Stanis{\l}aw and Ballas, Nicolas and Krueger, David and Bengio, Emmanuel and Kanwal, Maxinder S. and Maharaj, Tegan and Fischer, Asja and Courville, Aaron and Bengio, Yoshua and Lacoste-Julien, Simon},
|
||||
booktitle={Proceedings of the 34th International Conference on Machine Learning},
|
||||
pages={233--242},
|
||||
year={2017}
|
||||
}
|
||||
|
||||
@inproceedings{zhang2018generalized,
|
||||
title={Generalized Cross Entropy Loss for Training Deep Neural Networks with Noisy Labels},
|
||||
author={Zhang, Zhilu and Sabuncu, Mert},
|
||||
booktitle={Advances in Neural Information Processing Systems},
|
||||
volume={31},
|
||||
year={2018}
|
||||
}
|
||||
|
||||
@inproceedings{patrini2017making,
|
||||
title={Making Deep Neural Networks Robust to Label Noise: {A} Loss Correction Approach},
|
||||
author={Patrini, Giorgio and Rozza, Alessandro and Krishna Menon, Aditya and Nock, Richard and Qu, Lizhen},
|
||||
booktitle={Proceedings of the {IEEE} Conference on Computer Vision and Pattern Recognition},
|
||||
pages={1944--1952},
|
||||
year={2017}
|
||||
}
|
||||
|
||||
@inproceedings{han2018coteaching,
|
||||
title={Co-teaching: Robust Training of Deep Neural Networks with Extremely Noisy Labels},
|
||||
author={Han, Bo and Yao, Quanming and Yu, Xingrui and Niu, Gang and Xu, Miao and Hu, Weihua and Tsang, Ivor and Sugiyama, Masashi},
|
||||
booktitle={Advances in Neural Information Processing Systems},
|
||||
volume={31},
|
||||
year={2018}
|
||||
}
|
||||
|
||||
@inproceedings{li2020dividemix,
|
||||
title={{DivideMix}: Learning with Noisy Labels as Semi-Supervised Learning},
|
||||
author={Li, Junnan and Socher, Richard and Hoi, Steven C. H.},
|
||||
booktitle={International Conference on Learning Representations},
|
||||
year={2020}
|
||||
}
|
||||
|
||||
@inproceedings{ren2018learning,
|
||||
title={Learning to Reweight Examples for Robust Deep Learning},
|
||||
author={Ren, Mengye and Zeng, Wenyuan and Yang, Bin and Urtasun, Raquel},
|
||||
booktitle={Proceedings of the 35th International Conference on Machine Learning},
|
||||
pages={4334--4343},
|
||||
year={2018}
|
||||
}
|
||||
|
||||
@inproceedings{feng2020can,
|
||||
title={Can Cross Entropy Loss Be Robust to Label Noise?},
|
||||
author={Feng, Lei and Shu, Senlin and Lin, Zhuoyi and Lv, Fengmei and Li, Li and An, Bo},
|
||||
booktitle={Proceedings of the Twenty-Ninth International Joint Conference on Artificial Intelligence},
|
||||
pages={2206--2212},
|
||||
year={2020}
|
||||
}
|
||||
|
||||
@inproceedings{zhou2021asymmetric,
|
||||
title={Asymmetric Loss Functions for Learning with Noisy Labels},
|
||||
author={Zhou, Xiong and Liu, Xianming and Jiang, Junjun and Gao, Xin and Ji, Xiangyang},
|
||||
booktitle={Proceedings of the 38th International Conference on Machine Learning},
|
||||
pages={12846--12856},
|
||||
year={2021}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
We derive the gradient of \eqref{eq:roll-tpr-at-fpr-gaussian} with respect to
|
||||
$f_\theta(\mathbf{x}_i)$, instantiating the general structure~\eqref{eq:roll-gradient}
|
||||
for the Gaussian case. Throughout, $\tau = \hat{F}_0^{-1}(1-\alpha) =
|
||||
\mu_0 + \sigma_0\sqrt{2}\,\operatorname{ierf}(1-2\alpha)$.
|
||||
|
||||
\paragraph{Case $y_i = 1$.}
|
||||
|
||||
The score $f_\theta(\mathbf{x}_i)$ enters through $\mu_1$ and $\sigma_1$.
|
||||
Differentiating $\hat{F}_1(\tau) = \frac{1}{2}\bigl[1+\operatorname{erf}\!\bigl(\frac{\tau-\mu_1}{\sigma_1\sqrt{2}}\bigr)\bigr]$:
|
||||
|
||||
\[
|
||||
\dfrac{\partial \hat{F}_1(\hat{F}_0^{-1}(1-\alpha))}{\partial \mu_1} =
|
||||
-\frac{1}{\sigma_1\sqrt{2\pi}}\exp\!\left(-\frac{(\tau - \mu_1)^2}{2\sigma_1^2}\right)
|
||||
\]
|
||||
|
||||
\[
|
||||
\dfrac{\partial \hat{F}_1(\hat{F}_0^{-1}(1-\alpha))}{\partial \sigma_1} =
|
||||
\frac{\mu_1 - \tau}{\sigma_1^2\sqrt{2\pi}}\exp\!\left(-\frac{(\tau - \mu_1)^2}{2\sigma_1^2}\right)
|
||||
\]
|
||||
|
||||
Combined with $\frac{\partial \mu_1}{\partial f_\theta(\mathbf{x}_i)} = \frac{1}{|\mathcal{B}_1|}$
|
||||
and $\frac{\partial \sigma_1}{\partial f_\theta(\mathbf{x}_i)} = \frac{f_\theta(\mathbf{x}_i)-\mu_1}{|\mathcal{B}_1|\sigma_1}$,
|
||||
the chain rule gives:
|
||||
|
||||
\begin{equation}
|
||||
\dfrac{\partial \hat{F}_1(\hat{F}_0^{-1}(1-\alpha))}{\partial f_{\theta}(\mathbf{x}_i)}
|
||||
= -\frac{1}{|\mathcal{B}_1|\,\sigma_1\sqrt{2\pi}}
|
||||
\exp\!\left(-\frac{(\tau - \mu_1)^2}{2\sigma_1^2}\right)
|
||||
\left(1 + \frac{(\tau - \mu_1)(f_\theta(\mathbf{x}_i) - \mu_1)}{\sigma_1^2}\right)
|
||||
\label{eq:gauss-grad-y1}
|
||||
\end{equation}
|
||||
|
||||
\paragraph{Case $y_i = 0$.}
|
||||
|
||||
The score $f_\theta(\mathbf{x}_i)$ affects the loss only through $\tau$. By the chain rule:
|
||||
|
||||
\[
|
||||
\frac{\partial \hat{F}_1(\tau)}{\partial f_\theta(\mathbf{x}_i)}
|
||||
= \underbrace{\frac{\partial \hat{F}_1(\tau)}{\partial \tau}}_{\text{PDF of class 1 at }\tau}
|
||||
\cdot \frac{\partial \tau}{\partial f_\theta(\mathbf{x}_i)}
|
||||
\]
|
||||
|
||||
The first factor is the Gaussian PDF at $\tau$:
|
||||
\[
|
||||
\frac{\partial \hat{F}_1(\tau)}{\partial \tau}
|
||||
= \frac{1}{\sigma_1\sqrt{2\pi}}\exp\!\left(-\frac{(\tau - \mu_1)^2}{2\sigma_1^2}\right)
|
||||
\]
|
||||
|
||||
For the second factor, since $\tau = \mu_0 + \sigma_0\sqrt{2}\,\operatorname{ierf}(1-2\alpha)$:
|
||||
$\frac{\partial\tau}{\partial\mu_0}=1$ and
|
||||
$\frac{\partial\tau}{\partial\sigma_0}=\frac{\tau-\mu_0}{\sigma_0}$.
|
||||
Combined with $\frac{\partial\mu_0}{\partial f_\theta(\mathbf{x}_i)}=\frac{1}{|\mathcal{B}_0|}$
|
||||
and $\frac{\partial\sigma_0}{\partial f_\theta(\mathbf{x}_i)}=\frac{f_\theta(\mathbf{x}_i)-\mu_0}{|\mathcal{B}_0|\sigma_0}$:
|
||||
\[
|
||||
\frac{\partial\tau}{\partial f_\theta(\mathbf{x}_i)}
|
||||
= \frac{1}{|\mathcal{B}_0|}\!\left(1+\frac{(\tau-\mu_0)(f_\theta(\mathbf{x}_i)-\mu_0)}{\sigma_0^2}\right)
|
||||
\]
|
||||
|
||||
Therefore:
|
||||
\begin{equation}
|
||||
\dfrac{\partial \hat{F}_1(\hat{F}_0^{-1}(1-\alpha))}{\partial f_\theta(\mathbf{x}_i)}
|
||||
= \frac{1}{|\mathcal{B}_0|\,\sigma_1\sqrt{2\pi}}
|
||||
\exp\!\left(-\frac{(\tau - \mu_1)^2}{2\sigma_1^2}\right)
|
||||
\left(1 + \frac{(\tau - \mu_0)(f_\theta(\mathbf{x}_i) - \mu_0)}{\sigma_0^2}\right)
|
||||
\label{eq:gauss-grad-y0}
|
||||
\end{equation}
|
||||
|
||||
Substituting \eqref{eq:gauss-grad-y1} and \eqref{eq:gauss-grad-y0} into \eqref{eq:roll-gradient} yields \eqref{eq:gauss-grad-combined}.
|
||||
@@ -0,0 +1,30 @@
|
||||
We describe the bisection procedure used to compute $\tau = \hat{F}_0^{-1}(1-\alpha;\mathcal{B}_0)$
|
||||
in the KDE-ROLL forward pass (\Cref{sec:roll-kde-forward}).
|
||||
|
||||
\paragraph{Bracket initialisation.}
|
||||
Because $\sigma(u;\,v) \to 0$ exponentially as $u \to -\infty$ and $\sigma(u;\,v) \to 1$
|
||||
as $u \to +\infty$, with tails decaying within roughly $10/v_0$ of the data range, the
|
||||
interval
|
||||
\[
|
||||
\mathrm{lo} = \min(\mathbf{s}^{(0)}) - \tfrac{10}{v_0},
|
||||
\qquad
|
||||
\mathrm{hi} = \max(\mathbf{s}^{(0)}) + \tfrac{10}{v_0}
|
||||
\]
|
||||
is guaranteed to satisfy $\hat{F}_0(\mathrm{lo}) < 1-\alpha < \hat{F}_0(\mathrm{hi})$.
|
||||
|
||||
\paragraph{Bisection iterations.}
|
||||
At each step we evaluate the midpoint,
|
||||
\[
|
||||
\tau_{\mathrm{mid}} = \tfrac{1}{2}(\mathrm{lo} + \mathrm{hi}),
|
||||
\]
|
||||
and update the bracket:
|
||||
\[
|
||||
\begin{cases}
|
||||
\mathrm{lo} \leftarrow \tau_{\mathrm{mid}} & \text{if } \hat{F}_0(\tau_{\mathrm{mid}};\,\mathcal{B}_0) < 1-\alpha, \\
|
||||
\mathrm{hi} \leftarrow \tau_{\mathrm{mid}} & \text{otherwise.}
|
||||
\end{cases}
|
||||
\]
|
||||
Iteration continues until $\lvert\hat{F}_0(\tau_{\mathrm{mid}};\,\mathcal{B}_0)-(1-\alpha)\rvert < \varepsilon$
|
||||
(with $\varepsilon = 10^{-4}$), or until a maximum iteration count is reached.
|
||||
The method converges unconditionally and requires no derivative evaluation, in contrast
|
||||
to the Newton--Raphson alternative documented in Appendix~\ref{appendix:kde-nr}.
|
||||
@@ -0,0 +1,142 @@
|
||||
We derive the KDE gradient, instantiating the general form~\eqref{eq:roll-gradient}.
|
||||
Throughout, $\tau = \hat{F}_0^{-1}(1-\alpha;\mathcal{B}_0)$ and bandwidths $v_0, v_1$
|
||||
are as defined in \Cref{sec:kde-bandwidth}.
|
||||
|
||||
\paragraph{Case $y_i = 1$: expanding the KDE sum.}
|
||||
|
||||
When $\mathbf{x}_i \in \mathcal{B}_1$, the score $f_\theta(\mathbf{x}_i)$ appears directly
|
||||
in $\hat{F}_1$ and has no effect on $\tau$. Expanding the KDE CDF:
|
||||
%
|
||||
\[
|
||||
\dfrac{\partial \hat{F}_1\!\left(\hat{F}_0^{-1}(1-\alpha)\right)}{\partial f_{\theta}(\mathbf{x}_i)}
|
||||
= \frac{\partial}{\partial f_{\theta}(\mathbf{x}_i)}
|
||||
\frac{1}{|\mathcal{B}_1|}\sum_{\mathbf{x}_j \in \mathcal{B}_1}
|
||||
\sigma_1\!\left(\tau - f_{\theta}(\mathbf{x}_j)\right)
|
||||
\]
|
||||
%
|
||||
Separating the $j = i$ term (the only one depending on $f_\theta(\mathbf{x}_i)$):
|
||||
%
|
||||
\[
|
||||
= \frac{\partial}{\partial f_{\theta}(\mathbf{x}_i)}\!\left(
|
||||
\sum_{\substack{\mathbf{x}_j \in \mathcal{B}_1 \\ j \neq i}}
|
||||
\frac{\sigma_1\!\left(\tau - f_{\theta}(\mathbf{x}_j)\right)}{|\mathcal{B}_1|}
|
||||
\;+\;
|
||||
\frac{\sigma_1\!\left(\tau - f_{\theta}(\mathbf{x}_i)\right)}{|\mathcal{B}_1|}
|
||||
\right)
|
||||
\]
|
||||
%
|
||||
The first sum is constant in $f_\theta(\mathbf{x}_i)$. Applying the chain rule to the
|
||||
last term with $\partial(\tau - f_\theta(\mathbf{x}_i))/\partial f_\theta(\mathbf{x}_i) = -1$:
|
||||
%
|
||||
\begin{equation}
|
||||
\frac{\partial \hat{F}_1(\hat{F}_0^{-1}(1-\alpha))}{\partial f_{\theta}(\mathbf{x}_i)}
|
||||
= -\frac{1}{|\mathcal{B}_1|}\sigma_1'(\tau - f_{\theta}(\mathbf{x}_i))
|
||||
\label{eq:kde-grad-y1}
|
||||
\end{equation}
|
||||
|
||||
\paragraph{Case $y_i = 0$: derivative of $\hat{F}_1$ with respect to $\tau$.}
|
||||
|
||||
Differentiating the KDE CDF with respect to $\tau$:
|
||||
%
|
||||
\[
|
||||
\frac{\partial \hat{F}_1\!\left(\hat{F}_0^{-1}(1-\alpha)\right)}{\partial \hat{F}_0^{-1}(1-\alpha)}
|
||||
= \frac{\partial}{\partial \tau}
|
||||
\frac{1}{|\mathcal{B}_1|}\sum_{\mathbf{x}_j \in \mathcal{B}_1}
|
||||
\sigma_1\!\left(\tau - f_{\theta}(\mathbf{x}_j)\right)
|
||||
\]
|
||||
%
|
||||
Since $\partial(\tau - f_\theta(\mathbf{x}_j))/\partial\tau = +1$ for every $j$,
|
||||
the chain rule gives:
|
||||
%
|
||||
\begin{equation}
|
||||
\frac{\partial \hat{F}_1(\hat{F}_0^{-1}(1-\alpha))}{\partial \hat{F}_0^{-1}(1-\alpha)}
|
||||
= +\frac{1}{|\mathcal{B}_1|}\sum_{\mathbf{x}_j \in \mathcal{B}_1}\sigma_1'(\tau - f_{\theta}(\mathbf{x}_j))
|
||||
\label{eq:kde-dF1-dtau}
|
||||
\end{equation}
|
||||
|
||||
\paragraph{Case $y_i = 0$: derivative of $\tau$ with respect to $f_\theta(\mathbf{x}_i)$.}
|
||||
|
||||
Since $\tau = \hat{F}_0^{-1}(1-\alpha;\mathcal{B}_0)$ is computed numerically, we apply the \emph{inverse function theorem}: if $h$ has inverse $g$,
|
||||
then $g'(y) = 1/h'(g(y))$.
|
||||
|
||||
We view $\hat{F}_0^{-1}(1-\alpha;\mathcal{B}_0)$ as a function of $f_\theta(\mathbf{x}_i)$
|
||||
and find its inverse. From the defining equation
|
||||
$|\mathcal{B}_0|\cdot(1-\alpha) = \sum_{\mathbf{x}_j \in \mathcal{B}_0}
|
||||
\sigma_0(\tau - f_\theta(\mathbf{x}_j))$, solving for $f_\theta(\mathbf{x}_i)$:
|
||||
%
|
||||
\[
|
||||
f_\theta(\mathbf{x}_i)
|
||||
= \tau - \sigma_0^{-1}\!\!\left(
|
||||
|\mathcal{B}_0|\cdot(1-\alpha)
|
||||
- \sum_{\substack{\mathbf{x}_j \in \mathcal{B}_0 \\ j \neq i}}
|
||||
\sigma_0\!\left(\tau - f_{\theta}(\mathbf{x}_j)\right)
|
||||
\right)
|
||||
\]
|
||||
%
|
||||
Differentiating this expression with respect to $\tau$ and applying the chain rule:
|
||||
%
|
||||
\[
|
||||
\frac{\partial f_\theta(\mathbf{x}_i)}{\partial \tau}
|
||||
= 1 + \sigma_0^{-1}{}'\!\!\left(\cdots\right)
|
||||
\cdot
|
||||
\sum_{\substack{\mathbf{x}_j \in \mathcal{B}_0 \\ j \neq i}}
|
||||
\sigma_0'\!\left(\tau - f_{\theta}(\mathbf{x}_j)\right)
|
||||
\]
|
||||
%
|
||||
(the leading $+1$ comes from differentiating $\tau$; the second term from differentiating
|
||||
through $\sigma_0^{-1}$). By the inverse function theorem:
|
||||
%
|
||||
\[
|
||||
\frac{\partial \hat{F}_0^{-1}(1-\alpha ; \mathcal{B}_0)}{\partial f_\theta(\mathbf{x}_i)}
|
||||
= \frac{1}{\displaystyle
|
||||
1 + \sigma_0^{-1}{}'\!\!\left(
|
||||
|\mathcal{B}_0|\cdot(1-\alpha)
|
||||
- \sum_{j \neq i}\sigma_0(\tau - f_\theta(\mathbf{x}_j))
|
||||
\right)
|
||||
\cdot \sum_{j \neq i}\sigma_0'(\tau - f_\theta(\mathbf{x}_j))}
|
||||
\]
|
||||
%
|
||||
Using $|\mathcal{B}_0|\cdot(1-\alpha) = \sum_j \sigma_0(\tau - f_\theta(\mathbf{x}_j))$,
|
||||
the argument of $\sigma_0^{-1}{}'$ simplifies:
|
||||
%
|
||||
\[
|
||||
|\mathcal{B}_0|\cdot(1-\alpha)
|
||||
- \sum_{j \neq i}\sigma_0(\tau - f_\theta(\mathbf{x}_j))
|
||||
= \sigma_0(\tau - f_\theta(\mathbf{x}_i))
|
||||
\]
|
||||
%
|
||||
Substituting and applying $\sigma_0^{-1}{}'(\sigma_0(u)) = 1/\sigma_0'(u)$,
|
||||
then multiplying numerator and denominator by $\sigma_0'(\tau - f_\theta(\mathbf{x}_i))$:
|
||||
%
|
||||
\begin{equation}
|
||||
\frac{\partial \hat{F}_0^{-1}(1-\alpha ; \mathcal{B}_0)}{\partial f_\theta(\mathbf{x}_i)} =
|
||||
\frac{\sigma_0'(\tau - f_{\theta}(\mathbf{x}_i))}{\displaystyle\sum_{\mathbf{x}_j \in \mathcal{B}_0}\sigma_0'(\tau - f_{\theta}(\mathbf{x}_j))}
|
||||
\label{eq:kde-grad-tau}
|
||||
\end{equation}
|
||||
|
||||
\paragraph{Efficient computation via the sigmoid identity.}
|
||||
|
||||
Direct evaluation of \eqref{eq:kde-grad-tau} requires computing $\sigma_0'$ for every
|
||||
point — potentially unstable when a point is far from $\tau$. For the sigmoid kernel,
|
||||
the identity
|
||||
|
||||
\begin{equation}
|
||||
\sigma(u;\,v)\,\bigl(1 - \sigma(u;\,v)\bigr) = \frac{\sigma'(u;\,v)}{v}
|
||||
\label{eq:sigmoid-identity}
|
||||
\end{equation}
|
||||
|
||||
(proved by direct substitution: both sides equal $\exp(-v|u|)/(1+\exp(-v|u|))^2$)
|
||||
allows $v_0$ to cancel between numerator and denominator, so the ratio is expressed
|
||||
entirely in terms of sigmoid values already cached from the forward pass:
|
||||
|
||||
\begin{equation}
|
||||
\frac{\partial \hat{F}_0^{-1}(1-\alpha ; \mathcal{B}_0)}{\partial f_\theta(\mathbf{x}_i)}
|
||||
=
|
||||
\frac{\sigma_0(\tau-f_\theta(\mathbf{x}_i))\,\bigl(1-\sigma_0(\tau-f_\theta(\mathbf{x}_i))\bigr)}
|
||||
{\displaystyle\sum_{\mathbf{x}_j \in \mathcal{B}_0}
|
||||
\sigma_0(\tau-f_\theta(\mathbf{x}_j))\,\bigl(1-\sigma_0(\tau-f_\theta(\mathbf{x}_j))\bigr)}
|
||||
\label{eq:kde-grad-tau-efficient}
|
||||
\end{equation}
|
||||
|
||||
Substituting \eqref{eq:kde-grad-y1}, \eqref{eq:kde-dF1-dtau}, and \eqref{eq:kde-grad-tau}
|
||||
into \eqref{eq:roll-gradient} yields \eqref{eq:kde-grad-combined}.
|
||||
@@ -0,0 +1,28 @@
|
||||
An earlier version of the implementation used Newton--Raphson to invert the KDE CDF
|
||||
instead of bisection (\Cref{sec:roll-kde-forward}). We document it here as a reference
|
||||
alternative.
|
||||
|
||||
\paragraph{Method.}
|
||||
To find $\tau = \hat{F}_0^{-1}(1-\alpha;\mathcal{B}_0)$, choose an initial guess $\tau_0$
|
||||
and iterate:
|
||||
\begin{equation}
|
||||
\tau_{n+1} = \tau_n
|
||||
- \frac{\hat{F}_0(\tau_n;\,\mathcal{B}_0) - (1-\alpha)}
|
||||
{\dfrac{\partial\hat{F}_0(\tau_n;\,\mathcal{B}_0)}{\partial\tau}},
|
||||
\label{eq:kde-nr-step}
|
||||
\end{equation}
|
||||
where the denominator is the KDE density evaluated at $\tau_n$:
|
||||
\[
|
||||
\frac{\partial\hat{F}_0(\tau_n;\,\mathcal{B}_0)}{\partial\tau}
|
||||
= \frac{1}{|\mathcal{B}_0|}\sum_{\mathbf{x}_j\in\mathcal{B}_0}
|
||||
\sigma_0'(\tau_n - f_\theta(\mathbf{x}_j)).
|
||||
\]
|
||||
Iteration continues until
|
||||
$\lvert\hat{F}_0(\tau_n;\,\mathcal{B}_0)-(1-\alpha)\rvert < \varepsilon$.
|
||||
|
||||
\paragraph{Limitations.}
|
||||
Newton--Raphson requires evaluating $\sigma_0'$ at every step and can diverge if
|
||||
$\tau_0$ is far from the root or if the KDE density is very small near the target
|
||||
quantile (e.g.\ early in training when scores cluster tightly and $v_0$ is large).
|
||||
These failure modes motivated the switch to bisection, which requires no derivative
|
||||
evaluation in the forward pass and is guaranteed to converge given a valid bracket.
|
||||
@@ -0,0 +1,70 @@
|
||||
We instantiate the KDE gradient~\eqref{eq:kde-grad-combined} for the logistic sigmoid
|
||||
kernel used in our implementation.
|
||||
|
||||
\paragraph{Kernel definition.}
|
||||
|
||||
\begin{equation}
|
||||
\sigma(x ; v) = \frac{1}{1 + \exp(-vx)},
|
||||
\qquad
|
||||
\sigma'(x ; v) = \frac{v\,\exp(-v|x|)}{\bigl(1 + \exp(-v|x|)\bigr)^{2}}
|
||||
\label{eq:sigmoid-kernel}
|
||||
\end{equation}
|
||||
|
||||
where $v > 0$ is the bandwidth parameter. The derivative uses $|x|$ rather than $x$:
|
||||
since $\sigma'$ is even, both forms are mathematically identical, but $\exp(-v|x|) \to 0$
|
||||
as $|x| \to \infty$ whereas $\exp(-vx) \to \infty$ for $x \to -\infty$, so the $|x|$
|
||||
form avoids floating-point overflow when scores fall well below $\tau$.
|
||||
|
||||
\paragraph{Forward pass.}
|
||||
|
||||
Substituting \eqref{eq:sigmoid-kernel} into the KDE CDF:
|
||||
%
|
||||
\begin{equation}
|
||||
\hat{F}_{k}(\tau ; \mathbf{s}^{(k)}) =
|
||||
\frac{1}{|\mathcal{B}_k|}
|
||||
\sum_{\mathbf{x}_j \in \mathcal{B}_k}
|
||||
\frac{1}{1 + \exp\!\bigl(-v_k\bigl(\tau - f_\theta(\mathbf{x}_j)\bigr)\bigr)}
|
||||
\label{eq:kde-sigmoid-cdf}
|
||||
\end{equation}
|
||||
%
|
||||
The derivative of $\hat{F}_k$ with respect to $\tau$ — used in Newton--Raphson inversion
|
||||
and in the gradient — is:
|
||||
%
|
||||
\begin{equation}
|
||||
\frac{\partial \hat{F}_{k}(\tau)}{\partial \tau}
|
||||
=
|
||||
\frac{1}{|\mathcal{B}_k|}
|
||||
\sum_{\mathbf{x}_j \in \mathcal{B}_k}
|
||||
\frac{v_k\,\exp\!\bigl(-v_k\bigl|\tau - f_\theta(\mathbf{x}_j)\bigr|\bigr)}
|
||||
{\Bigl(1 + \exp\!\bigl(-v_k\bigl|\tau - f_\theta(\mathbf{x}_j)\bigr|\bigr)\Bigr)^{2}}
|
||||
\label{eq:kde-sigmoid-pdf}
|
||||
\end{equation}
|
||||
|
||||
\paragraph{Gradient.}
|
||||
|
||||
Define the shorthand $\varphi_k(u) = \sigma'(u ; v_k)$:
|
||||
\begin{equation}
|
||||
\varphi_k(u) \;=\;
|
||||
\frac{v_k\,\exp(-v_k|u|)}{\bigl(1+\exp(-v_k|u|)\bigr)^{2}}
|
||||
\label{eq:sigmoid-phi}
|
||||
\end{equation}
|
||||
|
||||
Substituting \eqref{eq:sigmoid-kernel} into \eqref{eq:kde-grad-combined}:
|
||||
|
||||
\begin{equation}
|
||||
\frac{\partial \mathcal{L}_{\text{ROLL-TPR@FPR}}}{\partial f_\theta(\mathbf{x}_i)} =
|
||||
\begin{cases}
|
||||
-\dfrac{\varphi_1\!\left(\tau - f_\theta(\mathbf{x}_i)\right)}{|\mathcal{B}_1|}
|
||||
& \text{if } y_i = 1\\[14pt]
|
||||
+\,\dfrac{\displaystyle\sum_{\mathbf{x}_j \in \mathcal{B}_1}
|
||||
\varphi_1\!\left(\tau - f_\theta(\mathbf{x}_j)\right)}{|\mathcal{B}_1|}
|
||||
\;\cdot\;
|
||||
\dfrac{\varphi_0\!\left(\tau-f_\theta(\mathbf{x}_i)\right)}
|
||||
{\displaystyle\sum_{\mathbf{x}_j \in \mathcal{B}_0}
|
||||
\varphi_0\!\left(\tau-f_\theta(\mathbf{x}_j)\right)}
|
||||
& \text{if } y_i = 0
|
||||
\end{cases}
|
||||
\label{eq:kde-sigmoid-grad}
|
||||
\end{equation}
|
||||
|
||||
where $\tau = \hat{F}_0^{-1}(1-\alpha ; \mathcal{B}_0)$.
|
||||
@@ -2,16 +2,237 @@
|
||||
\section{Data Sets}\label{sec:experiments:data_sets}
|
||||
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{מערך נתונים}}
|
||||
|
||||
\paragraph{KEEL benchmark.}
|
||||
We evaluate on 12 binary imbalanced datasets from the KEEL repository~\cite{TODO} (Knowledge Extraction based on Evolutionary Learning). All labels are clean. \Cref{tab:datasets:keel} summarises the datasets; imbalance ratios range from 1.8 to 11.6, covering mild to moderate imbalance regimes commonly used in the literature.
|
||||
|
||||
\begin{table}[htbp]
|
||||
\centering
|
||||
\caption{KEEL benchmark datasets used in experiments. IR = negative/positive count ratio.}
|
||||
\label{tab:datasets:keel}
|
||||
\small
|
||||
\begin{tabular}{lrrrrl}
|
||||
\toprule
|
||||
Dataset & $n$ & Pos & Neg & IR & Source \\
|
||||
\midrule
|
||||
Glass0 & 214 & 70 & 144 & 2.1 & Glass identification (UCI) \\
|
||||
Glass1 & 214 & 76 & 138 & 1.8 & Glass identification (UCI) \\
|
||||
Glass2 & 214 & 17 & 197 & 11.6 & Glass identification (UCI) \\
|
||||
Glass6 & 214 & 29 & 185 & 6.4 & Glass identification (UCI) \\
|
||||
Haberman & 306 & 81 & 225 & 2.8 & Breast cancer survival, 1958--1970 \\
|
||||
Iris0 & 150 & 50 & 100 & 2.0 & \emph{Iris setosa} vs.\ others \\
|
||||
New-Thyroid1 & 215 & 35 & 180 & 5.1 & Thyroid function classification \\
|
||||
Pima & 768 & 268 & 500 & 1.9 & Pima Indian diabetes (UCI) \\
|
||||
Vehicle2 & 846 & 218 & 628 & 2.9 & Vehicle silhouette (UCI) \\
|
||||
Vowel0 & 988 & 90 & 898 & 10.0 & Vowel recognition, class 0 vs.\ rest \\
|
||||
Wisconsin & 683 & 239 & 444 & 1.9 & Wisconsin breast cancer (UCI) \\
|
||||
Yeast3 & 1484 & 163 &1321 & 8.1 & Yeast protein localisation (UCI) \\
|
||||
\bottomrule
|
||||
\end{tabular}
|
||||
\end{table}
|
||||
|
||||
\paragraph{CIFAR-10N.}
|
||||
CIFAR-10N~\cite{TODO} augments the 50{,}000-image CIFAR-10 training set ($32{\times}32$ RGB) with real human-annotated noisy labels collected from Amazon Mechanical Turk~\cite{TODO}. We form a binary task (class~1, automobile, vs.\ rest; IR~$\approx 9$:1) and train under three noise conditions:
|
||||
\emph{clean} (original CIFAR-10 labels, 0\% noise),
|
||||
\emph{aggre} (majority-vote aggregate of crowdworker annotations, $\approx$9\% noise), and
|
||||
\emph{worse} (single worst-annotator labels, $\approx$40\% noise).
|
||||
Architecture: a three-block convolutional network trained from scratch.
|
||||
|
||||
\paragraph{Large tabular datasets.}
|
||||
We include three large-scale tabular datasets spanning diverse imbalanced real-world settings. All labels are considered clean (no injected or crowd-sourced noise).
|
||||
|
||||
\begin{itemize}
|
||||
\item \textbf{Bank Marketing}~\cite{TODO} --- UCI repository; Portuguese bank telephone marketing campaigns (Moro et al., 2014). $n = 41{,}188$ samples, 48 features after one-hot encoding. Positive class: client subscribed to term deposit. IR~$\approx 8$:1 ($\approx$11.3\% positive). No label noise.
|
||||
|
||||
\item \textbf{Credit Card Fraud}~\cite{TODO} --- Kaggle / ULB (Worldline + MLG, 2013). $n = 284{,}807$ transactions, 30 features (V1--V28 are PCA-transformed; Time and Amount rescaled). Extreme imbalance: 492 confirmed frauds, IR~$\approx 577$:1. No label noise.
|
||||
|
||||
\item \textbf{Home Credit}~\cite{TODO} --- Kaggle Home Credit Default Risk competition (2018). $n = 307{,}511$ loan applications, $\approx$120 numerical features from the primary application table. Positive class: loan default. IR~$\approx 11.4$:1 ($\approx$8\% default rate). Labels are proxy labels (repayment outcome); no injected noise.
|
||||
\end{itemize}
|
||||
|
||||
\section{Performance Measures}\label{sec:experiments:performance_measures}
|
||||
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{מדדי ביצועים}}
|
||||
|
||||
To be continued\footnotemark.
|
||||
|
||||
\footnotetext{foot note text}
|
||||
We report the \emph{area under the ROC curve} (AUC) averaged over independent training episodes (see \Cref{sec:experiments:configurations}). AUC measures overall ranking quality and is a standard metric for imbalanced classification. All results are presented as mean $\pm$ standard deviation over episodes.
|
||||
|
||||
\section{Configurations and Parameter Settings}
|
||||
\label{sec:experiments:configurations}
|
||||
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{קונפיגורציה ופרמטרים}}
|
||||
|
||||
We compare \textsc{Roll}-AoC (the primary proposed loss, optimising the area under the ROC curve via KDE) and \textsc{Roll}-TPR (a variant targeting TPR at a fixed operating point) against the following baselines:
|
||||
|
||||
\begin{itemize}
|
||||
\item \textbf{BCE-W} --- binary cross-entropy with class-frequency inverse weighting.
|
||||
\item \textbf{Focal} --- focal loss~\cite{TODO} with focusing parameter $\gamma = 2$.
|
||||
\item \textbf{Asym.} --- asymmetric loss~\cite{TODO}.
|
||||
\item \textbf{GCE} --- generalised cross-entropy~\cite{TODO} with $q = 0.7$.
|
||||
\item \textbf{LibAUC} --- PESG-based AUC maximisation from the LibAUC library~\cite{TODO}.
|
||||
\item \textbf{MAE} --- mean absolute error (symmetric noise-robust loss).
|
||||
\end{itemize}
|
||||
|
||||
All methods use the same network architecture, optimiser (Adam for non-LibAUC methods), and early stopping. Each configuration is run for $n = 5$ independent episodes on KEEL and large datasets, and $n = 3$ on CIFAR-10N. Reported AUC is evaluated on the held-out test split.
|
||||
|
||||
\section{Results}\label{sec:experiments:results}
|
||||
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{תוצאות}}
|
||||
|
||||
\subsection{KEEL Benchmark}\label{sec:results:keel}
|
||||
|
||||
\Cref{tab:results:keel} reports mean test AUC across the 12 KEEL datasets. No single method dominates the benchmark: on easy datasets (Iris0, Wisconsin, Vowel0, Vehicle2, Glass6) nearly all methods converge to near-perfect AUC, leaving little room for differentiation. On the harder, more imbalanced datasets the picture is more varied. \textsc{Roll}-AoC competes well but does not claim the top position on most individual datasets; this is consistent with no-free-lunch intuitions for heterogeneous benchmarks.
|
||||
|
||||
\begin{table}[htbp]
|
||||
\centering
|
||||
\caption{Mean test AUC ($\pm$\,std, $n=5$ episodes) on KEEL benchmark datasets. Best per row in \textbf{bold}.}
|
||||
\label{tab:results:keel}
|
||||
\resizebox{\textwidth}{!}{%
|
||||
\begin{tabular}{lcccccccc}
|
||||
\toprule
|
||||
Dataset & \textsc{Roll}-AoC & \textsc{Roll}-TPR & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\
|
||||
\midrule
|
||||
Glass0 & 0.796{\tiny{$\pm$0.046}} & 0.826{\tiny{$\pm$0.032}} & 0.839{\tiny{$\pm$0.033}} & 0.802{\tiny{$\pm$0.020}} & 0.814{\tiny{$\pm$0.034}} & 0.818{\tiny{$\pm$0.032}} & \textbf{0.843{\tiny{$\pm$0.035}}} & 0.826{\tiny{$\pm$0.037}} \\
|
||||
Glass1 & 0.722{\tiny{$\pm$0.033}} & 0.698{\tiny{$\pm$0.042}} & 0.710{\tiny{$\pm$0.018}} & 0.712{\tiny{$\pm$0.036}} & 0.744{\tiny{$\pm$0.044}} & 0.742{\tiny{$\pm$0.039}} & 0.723{\tiny{$\pm$0.054}} & \textbf{0.769{\tiny{$\pm$0.031}}} \\
|
||||
Glass2 & 0.723{\tiny{$\pm$0.062}} & 0.701{\tiny{$\pm$0.055}} & 0.661{\tiny{$\pm$0.069}} & 0.677{\tiny{$\pm$0.072}} & \textbf{0.750{\tiny{$\pm$0.075}}} & 0.689{\tiny{$\pm$0.023}} & 0.719{\tiny{$\pm$0.084}} & 0.679{\tiny{$\pm$0.018}} \\
|
||||
Glass6 & 0.975{\tiny{$\pm$0.014}} & 0.975{\tiny{$\pm$0.027}} & 0.972{\tiny{$\pm$0.025}} & 0.974{\tiny{$\pm$0.020}} & 0.982{\tiny{$\pm$0.013}} & 0.971{\tiny{$\pm$0.020}} & 0.969{\tiny{$\pm$0.012}} & \textbf{0.985{\tiny{$\pm$0.006}}} \\
|
||||
Haberman & 0.696{\tiny{$\pm$0.045}} & 0.675{\tiny{$\pm$0.063}} & 0.736{\tiny{$\pm$0.012}} & \textbf{0.742{\tiny{$\pm$0.025}}} & 0.741{\tiny{$\pm$0.018}} & 0.737{\tiny{$\pm$0.016}} & 0.616{\tiny{$\pm$0.116}} & 0.671{\tiny{$\pm$0.102}} \\
|
||||
Iris0 & \textbf{1.000{\tiny{$\pm$0.000}}} & 0.981{\tiny{$\pm$0.027}} & 0.900{\tiny{$\pm$0.224}} & \textbf{1.000{\tiny{$\pm$0.000}}} & \textbf{1.000{\tiny{$\pm$0.000}}} & \textbf{1.000{\tiny{$\pm$0.000}}} & \textbf{1.000{\tiny{$\pm$0.000}}} & 0.900{\tiny{$\pm$0.224}} \\
|
||||
New-Thyroid1 & 0.989{\tiny{$\pm$0.010}} & 0.845{\tiny{$\pm$0.241}} & 0.992{\tiny{$\pm$0.005}} & 0.977{\tiny{$\pm$0.033}} & 0.988{\tiny{$\pm$0.007}} & 0.987{\tiny{$\pm$0.011}} & 0.985{\tiny{$\pm$0.012}} & \textbf{0.994{\tiny{$\pm$0.003}}} \\
|
||||
Pima & 0.778{\tiny{$\pm$0.024}} & 0.755{\tiny{$\pm$0.014}} & \textbf{0.787{\tiny{$\pm$0.014}}} & 0.775{\tiny{$\pm$0.020}} & 0.771{\tiny{$\pm$0.019}} & 0.785{\tiny{$\pm$0.005}} & 0.761{\tiny{$\pm$0.015}} & 0.778{\tiny{$\pm$0.005}} \\
|
||||
Vehicle2 & 0.982{\tiny{$\pm$0.003}} & 0.966{\tiny{$\pm$0.020}} & \textbf{0.984{\tiny{$\pm$0.004}}} & 0.982{\tiny{$\pm$0.003}} & 0.980{\tiny{$\pm$0.004}} & 0.982{\tiny{$\pm$0.004}} & 0.982{\tiny{$\pm$0.003}} & 0.983{\tiny{$\pm$0.004}} \\
|
||||
Vowel0 & 0.993{\tiny{$\pm$0.005}} & 0.987{\tiny{$\pm$0.002}} & 0.995{\tiny{$\pm$0.004}} & 0.996{\tiny{$\pm$0.003}} & 0.995{\tiny{$\pm$0.003}} & 0.997{\tiny{$\pm$0.003}} & 0.991{\tiny{$\pm$0.007}} & \textbf{0.998{\tiny{$\pm$0.001}}} \\
|
||||
Wisconsin & 0.997{\tiny{$\pm$0.002}} & 0.800{\tiny{$\pm$0.441}} & \textbf{0.999{\tiny{$\pm$0.000}}} & \textbf{0.999{\tiny{$\pm$0.001}}} & 0.999{\tiny{$\pm$0.001}} & 0.999{\tiny{$\pm$0.001}} & 0.999{\tiny{$\pm$0.000}} & 0.998{\tiny{$\pm$0.001}} \\
|
||||
Yeast3 & 0.906{\tiny{$\pm$0.024}} & 0.908{\tiny{$\pm$0.021}} & 0.894{\tiny{$\pm$0.008}} & 0.908{\tiny{$\pm$0.019}} & \textbf{0.935{\tiny{$\pm$0.014}}} & 0.927{\tiny{$\pm$0.006}} & \textbf{0.935{\tiny{$\pm$0.016}}} & 0.915{\tiny{$\pm$0.010}} \\
|
||||
\bottomrule
|
||||
\end{tabular}%
|
||||
}
|
||||
\end{table}
|
||||
|
||||
\subsection{CIFAR-10N: Label-Noise Robustness}\label{sec:results:cifar10n}
|
||||
|
||||
\Cref{tab:results:cifar10n} shows AUC under three noise levels on CIFAR-10N. Under \emph{clean} labels all methods reach AUC $\geq 0.972$, with \textsc{Roll}-AoC at the top (0.980). As noise increases the advantage of \textsc{Roll}-AoC grows: under \emph{aggre} noise it leads by 0.2~pp over the next-best method (GCE, 0.932), and under \emph{worse} noise the margin widens to 1.2~pp over MAE (0.809) and 4.7~pp over BCE-W (0.738). This suggests that the KDE-smoothed loss surface provides implicit robustness to label noise.
|
||||
|
||||
\begin{table}[htbp]
|
||||
\centering
|
||||
\caption{Mean test AUC ($\pm$\,std, $n=3$ episodes) on CIFAR-10N under three label-noise levels. Best per row in \textbf{bold}.}
|
||||
\label{tab:results:cifar10n}
|
||||
\resizebox{\textwidth}{!}{%
|
||||
\begin{tabular}{lccccccc}
|
||||
\toprule
|
||||
Noise & \textsc{Roll}-AoC & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\
|
||||
\midrule
|
||||
Clean & \textbf{0.980{\tiny{$\pm$0.001}}} & 0.977{\tiny{$\pm$0.002}} & 0.977{\tiny{$\pm$0.000}} & 0.972{\tiny{$\pm$0.003}} & 0.976{\tiny{$\pm$0.002}} & 0.979{\tiny{$\pm$0.000}} & 0.976{\tiny{$\pm$0.002}} \\
|
||||
Aggre. & \textbf{0.934{\tiny{$\pm$0.002}}} & 0.924{\tiny{$\pm$0.003}} & 0.912{\tiny{$\pm$0.014}} & 0.910{\tiny{$\pm$0.004}} & 0.932{\tiny{$\pm$0.005}} & 0.921{\tiny{$\pm$0.004}} & 0.930{\tiny{$\pm$0.002}} \\
|
||||
Worse & \textbf{0.821{\tiny{$\pm$0.001}}} & 0.738{\tiny{$\pm$0.009}} & 0.734{\tiny{$\pm$0.009}} & 0.720{\tiny{$\pm$0.004}} & 0.774{\tiny{$\pm$0.005}} & 0.752{\tiny{$\pm$0.006}} & 0.809{\tiny{$\pm$0.004}} \\
|
||||
\bottomrule
|
||||
\end{tabular}%
|
||||
}
|
||||
\end{table}
|
||||
|
||||
\subsection{Large Real-World Datasets}\label{sec:results:large}
|
||||
|
||||
\Cref{tab:results:large} covers three large-scale datasets. On Credit Card Fraud \textsc{Roll}-TPR is the top performer (0.974) with \textsc{Roll}-AoC close behind (0.968), and both outperform all non-ROLL baselines. On Home Credit \textsc{Roll}-AoC edges LibAUC (0.712 vs.\ 0.711). On Bank Marketing, however, GCE achieves the highest AUC (0.746) while \textsc{Roll}-AoC and \textsc{Roll}-TPR underperform (0.511 and 0.438 respectively); the cause of this degradation on this dataset is an open question and a direction for future investigation.
|
||||
|
||||
\begin{table}[htbp]
|
||||
\centering
|
||||
\caption{Mean test AUC ($\pm$\,std, $n=5$ episodes) on large real-world datasets. Best per row in \textbf{bold}.}
|
||||
\label{tab:results:large}
|
||||
\resizebox{\textwidth}{!}{%
|
||||
\begin{tabular}{lcccccccc}
|
||||
\toprule
|
||||
Dataset & \textsc{Roll}-AoC & \textsc{Roll}-TPR & BCE-W & Focal & Asym. & GCE & LibAUC & MAE \\
|
||||
\midrule
|
||||
Bank Marketing & 0.511{\tiny{$\pm$0.047}} & 0.438{\tiny{$\pm$0.044}} & 0.580{\tiny{$\pm$0.040}} & 0.573{\tiny{$\pm$0.044}} & 0.684{\tiny{$\pm$0.033}} & \textbf{0.746{\tiny{$\pm$0.022}}} & 0.652{\tiny{$\pm$0.031}} & 0.707{\tiny{$\pm$0.035}} \\
|
||||
Credit Card Fraud & 0.968{\tiny{$\pm$0.001}} & \textbf{0.974{\tiny{$\pm$0.003}}} & 0.949{\tiny{$\pm$0.015}} & 0.942{\tiny{$\pm$0.020}} & 0.940{\tiny{$\pm$0.013}} & 0.948{\tiny{$\pm$0.014}} & 0.966{\tiny{$\pm$0.006}} & 0.954{\tiny{$\pm$0.009}} \\
|
||||
Home Credit & \textbf{0.712{\tiny{$\pm$0.001}}} & 0.687{\tiny{$\pm$0.004}} & 0.693{\tiny{$\pm$0.002}} & 0.694{\tiny{$\pm$0.002}} & 0.688{\tiny{$\pm$0.003}} & 0.698{\tiny{$\pm$0.002}} & 0.711{\tiny{$\pm$0.001}} & 0.699{\tiny{$\pm$0.004}} \\
|
||||
\bottomrule
|
||||
\end{tabular}%
|
||||
}
|
||||
\end{table}
|
||||
|
||||
|
||||
\section{Label-Noise Poisoning Experiment}\label{sec:experiments:poisoning}
|
||||
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{ניסוי הרעלת תוויות}}
|
||||
|
||||
% A controlled experiment that isolates ROLL's label-noise robustness from confounders
|
||||
% (dataset size, imbalance ratio). Protocol: start from a clean dataset, inject
|
||||
% \emph{asymmetric} noise --- flip a fraction of positive labels to negative, leave
|
||||
% negative labels intact --- and track AUC degradation as the flip rate increases.
|
||||
% This mirrors a worst-case practical scenario: rare true positives become false negatives,
|
||||
% compounding the imbalance. Empirical complement to the structural argument in
|
||||
% \Cref{sec:noisy-labels}.
|
||||
|
||||
\subsection{Protocol}\label{sec:experiments:poisoning:protocol}
|
||||
|
||||
% Specify: which dataset(s) were poisoned (one or two from KEEL or large-scale, chosen
|
||||
% for mid-range difficulty where differences are visible), the range of flip rates tested
|
||||
% (e.g.\ 0\% to 40\% in steps of 10\%), and number of independent episodes per rate.
|
||||
% Justify asymmetric noise as the harder, more realistic case compared to symmetric flipping.
|
||||
|
||||
\subsection{Results}\label{sec:experiments:poisoning:results}
|
||||
|
||||
% Present AUC vs.\ noise-rate curves for all methods. Key finding: ROLL-AoC degrades
|
||||
% more gracefully than pointwise losses (BCE-W, Focal, Asym.) because a distributional
|
||||
% objective spreads the influence of each mislabeled sample over the estimated density
|
||||
% rather than producing a single large gradient. Cite structural explanation from
|
||||
% \Cref{sec:noisy-labels}; note consistency with CIFAR-10N results in
|
||||
% \Cref{sec:results:cifar10n}.
|
||||
|
||||
|
||||
\section{Ablation Study}\label{sec:experiments:ablation}
|
||||
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{ניתוח אבלציה}}
|
||||
|
||||
% Isolate the contribution of ROLL's individual design choices to determine which
|
||||
% components drive its performance and noise robustness.
|
||||
|
||||
\subsection{Effect of KDE Smoothing}\label{sec:experiments:ablation:kde}
|
||||
|
||||
% Compare ROLL-AoC against a hard-threshold variant (no KDE; threshold estimated
|
||||
% by sorting scores and selecting the rank corresponding to FPR $= \alpha$).
|
||||
% Expected finding: without smooth KDE the gradient signal collapses and training
|
||||
% is unstable, confirming that differentiability is not just a theoretical convenience.
|
||||
|
||||
\subsection{Effect of Kernel Choice}\label{sec:experiments:ablation:kernel}
|
||||
|
||||
% Compare Gaussian vs.\ sigmoid kernel in ROLL-AoC on representative datasets.
|
||||
% The sigmoid kernel admits closed-form gradients (see \Cref{appendix:kde-sigmoid});
|
||||
% this subsection assesses whether kernel choice materially affects accuracy or
|
||||
% whether performance is robust to kernel selection.
|
||||
|
||||
\subsection{ROLL-AoC vs.\ ROLL-TPR}\label{sec:experiments:ablation:variants}
|
||||
|
||||
% Systematically compare the two ROLL variants across all benchmarks. Discuss when
|
||||
% targeting a single operating point (ROLL-TPR) helps (e.g.\ Credit Card Fraud, where
|
||||
% a strict FPR budget is natural) vs.\ when whole-curve optimization (ROLL-AoC) is
|
||||
% preferable (heterogeneous or unconstrained evaluation).
|
||||
|
||||
|
||||
\section{Sensitivity Analysis}\label{sec:experiments:sensitivity}
|
||||
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{ניתוח רגישות}}
|
||||
|
||||
% Characterize ROLL's robustness to its key hyperparameters so practitioners can
|
||||
% apply it without exhaustive tuning.
|
||||
|
||||
\subsection{Bandwidth Sensitivity}\label{sec:experiments:sensitivity:bandwidth}
|
||||
|
||||
% Plot AUC vs.\ log-bandwidth $h$ for ROLL-AoC across a representative subset of
|
||||
% datasets (at least one from KEEL, CIFAR-10N, and large-scale). Report the range
|
||||
% of $h$ over which performance stays within 1\% of its peak --- this defines a
|
||||
% practical safe tuning zone. Show that Silverman's rule falls within this zone,
|
||||
% justifying it as a default.
|
||||
|
||||
\subsection{Operating-Point Sensitivity (ROLL-TPR)}\label{sec:experiments:sensitivity:alpha}
|
||||
|
||||
% For ROLL-TPR, sweep the target FPR budget $\alpha$ and measure TPR achieved at
|
||||
% each setting. Validate that the trained model's operating point tracks the
|
||||
% specified $\alpha$, demonstrating that ROLL-TPR reliably targets the operating
|
||||
% point it was trained for and generalises to nearby values.
|
||||
|
||||
|
||||
\section{Summary}\label{sec:experiments:summary}
|
||||
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{סיכום}}
|
||||
|
||||
% Synthesise the empirical findings in 2--3 paragraphs.
|
||||
% (1) Where ROLL-AoC wins clearly: CIFAR-10N noise robustness, Credit Card Fraud,
|
||||
% Home Credit --- relate to the distributional objective and KDE smoothing.
|
||||
% (2) Where it underperforms: Bank Marketing --- flag as an open question; discuss
|
||||
% possible causes (feature scale, label proxy quality, optimizer interaction).
|
||||
% (3) What the ablation reveals: which components are load-bearing vs.\ incidental.
|
||||
% Close with a sentence bridging to \Cref{chapter:discussion_and_conclusions}.
|
||||
|
||||
TODO write this section.
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 76 KiB |
@@ -0,0 +1,181 @@
|
||||
"""
|
||||
Comparison of density estimators on real model output scores.
|
||||
|
||||
For each class (negative / positive) plots four curves against the empirical CDF:
|
||||
- ECDF — ground truth step function
|
||||
- KDE (ISJ) — Gaussian-kernel KDE with Improved Sheather-Jones bandwidth
|
||||
- Gaussian fit — MLE normal distribution
|
||||
- Beta fit — sigmoid-transformed scores fitted with Beta MLE, CDF
|
||||
mapped back to the original logit axis
|
||||
|
||||
Data: test split of a ROLL model trained on the glass0 dataset.
|
||||
|
||||
Run from the impl directory (KDEpy lives in that flake):
|
||||
cd impl && nix develop --command python3 ../thesis/content/method/figures/density_fit_comparison.py
|
||||
"""
|
||||
|
||||
import sys, pickle
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
from scipy.stats import norm, beta as sp_beta
|
||||
from scipy.special import expit
|
||||
from KDEpy import FFTKDE
|
||||
from KDEpy.bw_selection import silvermans_rule, improved_sheather_jones
|
||||
|
||||
# Resolve paths relative to this script's location so the script runs from anywhere.
|
||||
_HERE = Path(__file__).resolve().parent # .../thesis/content/method/figures
|
||||
_THESIS = _HERE.parents[2] # .../thesis
|
||||
_IMPL = _THESIS.parent / "impl" # .../impl
|
||||
|
||||
sys.path.insert(0, str(_IMPL)) # pickle needs 'import src.experiment' → impl/src/experiment.py
|
||||
|
||||
# ── Config ──────────────────────────────────────────────────────────────────
|
||||
RESULT_PKL = _IMPL / "results/glass0/2026-07-08-22-25/roll/0/test-res.pkl"
|
||||
OUT_DIR = _HERE
|
||||
SPLIT = "test"
|
||||
|
||||
C_NEG = "#4477AA"
|
||||
C_POS = "#CC6633"
|
||||
|
||||
plt.rcParams.update({
|
||||
"text.usetex": False,
|
||||
"mathtext.fontset": "cm",
|
||||
"pdf.fonttype": 42,
|
||||
})
|
||||
|
||||
# ── Load data ────────────────────────────────────────────────────────────────
|
||||
with open(RESULT_PKL, "rb") as f:
|
||||
ep = pickle.load(f)
|
||||
|
||||
res = ep.split_results[SPLIT]
|
||||
yh = res.yh.astype(np.float64) # raw logit scores
|
||||
y = res.y
|
||||
|
||||
scores_neg = yh[y == 0]
|
||||
scores_pos = yh[y == 1]
|
||||
|
||||
print(f"Loaded {SPLIT} split: {len(scores_neg)} negatives, {len(scores_pos)} positives")
|
||||
print(f"Score range: [{yh.min():.3f}, {yh.max():.3f}]")
|
||||
|
||||
|
||||
# ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
def ecdf(scores):
|
||||
"""Return (x, p) for a step-function ECDF."""
|
||||
s = np.sort(scores)
|
||||
p = np.arange(1, len(s) + 1) / len(s)
|
||||
return s, p
|
||||
|
||||
|
||||
def _kde_pdf(scores, x_grid, h):
|
||||
pdf = FFTKDE(kernel="gaussian", bw=h).fit(scores).evaluate(x_grid)
|
||||
dx = x_grid[1] - x_grid[0]
|
||||
cdf = np.cumsum(pdf) * dx
|
||||
cdf /= cdf[-1]
|
||||
return cdf
|
||||
|
||||
def kde_isj_cdf(scores, x_grid):
|
||||
"""ISJ bandwidth; Silverman fallback on failure."""
|
||||
data = scores[:, np.newaxis].astype(np.float64)
|
||||
try:
|
||||
h = float(improved_sheather_jones(data))
|
||||
except Exception:
|
||||
h = float(silvermans_rule(data))
|
||||
return _kde_pdf(scores, x_grid, h)
|
||||
|
||||
def kde_silverman_cdf(scores, x_grid):
|
||||
h = float(silvermans_rule(scores[:, np.newaxis].astype(np.float64)))
|
||||
return _kde_pdf(scores, x_grid, h)
|
||||
|
||||
|
||||
def gauss_cdf(scores, x_grid):
|
||||
loc, scale = norm.fit(scores)
|
||||
return norm.cdf(x_grid, loc, scale)
|
||||
|
||||
|
||||
def beta_cdf(scores, x_grid):
|
||||
"""Fit Beta in sigmoid space; evaluate CDF back on the logit x_grid."""
|
||||
s01 = expit(scores).clip(1e-6, 1 - 1e-6)
|
||||
a, b, _, _ = sp_beta.fit(s01, floc=0, fscale=1)
|
||||
return sp_beta.cdf(expit(x_grid), a, b)
|
||||
|
||||
|
||||
# ── Shared x-grid (covers both classes with a small margin) ──────────────────
|
||||
margin = 0.5
|
||||
x_lo = yh.min() - margin
|
||||
x_hi = yh.max() + margin
|
||||
x_grid = np.linspace(x_lo, x_hi, 2048)
|
||||
|
||||
|
||||
# ── Compute all curves ───────────────────────────────────────────────────────
|
||||
neg_ecdf_x, neg_ecdf_p = ecdf(scores_neg)
|
||||
neg_cdf_kde = kde_isj_cdf(scores_neg, x_grid)
|
||||
neg_cdf_silverman = kde_silverman_cdf(scores_neg, x_grid)
|
||||
neg_cdf_gauss = gauss_cdf(scores_neg, x_grid)
|
||||
neg_cdf_beta = beta_cdf(scores_neg, x_grid)
|
||||
|
||||
pos_ecdf_x, pos_ecdf_p = ecdf(scores_pos)
|
||||
pos_cdf_kde = kde_isj_cdf(scores_pos, x_grid)
|
||||
pos_cdf_silverman = kde_silverman_cdf(scores_pos, x_grid)
|
||||
pos_cdf_gauss = gauss_cdf(scores_pos, x_grid)
|
||||
pos_cdf_beta = beta_cdf(scores_pos, x_grid)
|
||||
|
||||
|
||||
# ── Plot ─────────────────────────────────────────────────────────────────────
|
||||
fig, (ax0, ax1) = plt.subplots(
|
||||
2, 1, figsize=(5.5, 4.8), sharex=True,
|
||||
gridspec_kw={"height_ratios": [1, 1]},
|
||||
)
|
||||
fig.subplots_adjust(hspace=0.08)
|
||||
|
||||
|
||||
def _plot_class(ax, color, ecdf_x, ecdf_p,
|
||||
cdf_isj, cdf_silverman, cdf_gauss, cdf_beta, label_prefix):
|
||||
ax.step(ecdf_x, ecdf_p,
|
||||
color=color, lw=1.0, alpha=0.55, where="post",
|
||||
label="ECDF")
|
||||
ax.plot(x_grid, cdf_isj,
|
||||
color=color, lw=1.8, linestyle="-",
|
||||
label="KDE (ISJ)")
|
||||
ax.plot(x_grid, cdf_silverman,
|
||||
color=color, lw=1.4, linestyle="-.",
|
||||
label="KDE (Silverman)")
|
||||
ax.plot(x_grid, cdf_gauss,
|
||||
color=color, lw=1.4, linestyle="--",
|
||||
label="Gaussian")
|
||||
ax.plot(x_grid, cdf_beta,
|
||||
color=color, lw=1.6, linestyle=":",
|
||||
label="Beta")
|
||||
ax.set_ylabel("CDF", fontsize=9)
|
||||
ax.set_yticks([0.0, 0.25, 0.50, 0.75, 1.0])
|
||||
ax.set_yticklabels(["0", ".25", ".5", ".75", "1"], fontsize=7.5)
|
||||
ax.tick_params(labelsize=8)
|
||||
ax.spines[["top", "right"]].set_visible(False)
|
||||
ax.text(0.97, 0.05, label_prefix,
|
||||
transform=ax.transAxes, fontsize=8,
|
||||
ha="right", va="bottom", color=color)
|
||||
|
||||
|
||||
_plot_class(ax0, C_NEG,
|
||||
neg_ecdf_x, neg_ecdf_p,
|
||||
neg_cdf_kde, neg_cdf_silverman, neg_cdf_gauss, neg_cdf_beta,
|
||||
r"Negative class ($y=0$)")
|
||||
|
||||
_plot_class(ax1, C_POS,
|
||||
pos_ecdf_x, pos_ecdf_p,
|
||||
pos_cdf_kde, pos_cdf_silverman, pos_cdf_gauss, pos_cdf_beta,
|
||||
r"Positive class ($y=1$)")
|
||||
|
||||
ax0.legend(fontsize=7.5, loc="upper left", framealpha=0.88)
|
||||
ax1.set_xlabel(r"Score $f_\theta(\mathbf{x})$", fontsize=9)
|
||||
ax0.set_xlim(x_lo, x_hi)
|
||||
|
||||
for path_suffix in ("pdf", "png"):
|
||||
kw = {"bbox_inches": "tight"}
|
||||
if path_suffix == "png":
|
||||
kw["dpi"] = 180
|
||||
fig.savefig(OUT_DIR / f"density_fit_comparison.{path_suffix}", **kw)
|
||||
|
||||
print(f"Saved to {OUT_DIR}/density_fit_comparison.{{pdf,png}}")
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 84 KiB |
@@ -0,0 +1,125 @@
|
||||
"""
|
||||
Geometric picture of the ROLL TPR@FPR objective.
|
||||
|
||||
Upper panel: score PDFs for both classes; shaded area under the positive-class
|
||||
PDF to the left of τ equals the ROLL loss F̂₁(τ).
|
||||
Lower panel: CDFs showing how τ is read from F̂₀ and how the loss and TPR
|
||||
are read from F̂₁.
|
||||
|
||||
Run from the thesis root:
|
||||
nix develop --command python3 content/method/figures/roll_principle.py
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
from scipy.stats import norm
|
||||
|
||||
OUT_DIR = "content/method/figures"
|
||||
|
||||
np.random.seed(3)
|
||||
|
||||
# --- Parameters ------------------------------------------------------------
|
||||
mu0, sig0 = 0.0, 1.00 # negative class N(0, 1)
|
||||
mu1, sig1 = 1.5, 1.00 # positive class N(1.5, 1) — intentional overlap
|
||||
alpha = 0.20 # target FPR
|
||||
|
||||
# τ = (1-α)-quantile of the negative class → F̂₀(τ) = 1-α → FPR = α
|
||||
tau = norm.ppf(1 - alpha, mu0, sig0)
|
||||
F0_tau = norm.cdf(tau, mu0, sig0) # = 1 - alpha
|
||||
F1_tau = norm.cdf(tau, mu1, sig1) # ROLL loss
|
||||
tpr = 1.0 - F1_tau
|
||||
|
||||
# --- Sample points for positive class (rug) --------------------------------
|
||||
rug = np.random.normal(mu1, sig1, 35)
|
||||
|
||||
# --- Score axis ------------------------------------------------------------
|
||||
x = np.linspace(-3.6, 5.0, 900)
|
||||
|
||||
# --- Palette ---------------------------------------------------------------
|
||||
C_NEG = "#4477AA"
|
||||
C_POS = "#CC6633"
|
||||
C_TAU = "#333333"
|
||||
C_LOSS = "#7799BB"
|
||||
|
||||
# ── Figure ─────────────────────────────────────────────────────────────────
|
||||
fig, (ax1, ax2) = plt.subplots(
|
||||
2, 1, figsize=(5.5, 4.8), sharex=True,
|
||||
gridspec_kw={"height_ratios": [1.25, 1.0]},
|
||||
)
|
||||
fig.subplots_adjust(hspace=0.07)
|
||||
|
||||
# ══ Top: PDF ══════════════════════════════════════════════════════════════
|
||||
|
||||
pdf0 = norm.pdf(x, mu0, sig0)
|
||||
pdf1 = norm.pdf(x, mu1, sig1)
|
||||
|
||||
ax1.plot(x, pdf0, color=C_NEG, linewidth=1.8, label=r"Negative class ($y=0$)")
|
||||
ax1.plot(x, pdf1, color=C_POS, linewidth=1.8, label=r"Positive class ($y=1$)")
|
||||
ax1.axvline(tau, color=C_TAU, linestyle="--", linewidth=1.3,
|
||||
label=r"Threshold $\tau$")
|
||||
|
||||
# Shade: area under positive PDF to the left of τ = loss = F̂₁(τ)
|
||||
x_left = x[x <= tau]
|
||||
ax1.fill_between(x_left, norm.pdf(x_left, mu1, sig1),
|
||||
color=C_LOSS, alpha=0.30, label=r"Loss $= \hat{F}_1(\tau)$")
|
||||
|
||||
# Rug plot
|
||||
rug_y = np.full_like(rug, -0.006)
|
||||
ax1.scatter(rug, rug_y, color=C_POS, s=18, marker="|",
|
||||
zorder=3, alpha=0.65, clip_on=False)
|
||||
|
||||
ax1.set_ylabel("PDF", fontsize=9)
|
||||
ax1.set_yticks([])
|
||||
ax1.set_ylim(bottom=-0.025)
|
||||
ax1.legend(fontsize=7.5, loc="upper right", framealpha=0.88)
|
||||
|
||||
# ══ Bottom: CDF ═══════════════════════════════════════════════════════════
|
||||
|
||||
cdf0 = norm.cdf(x, mu0, sig0)
|
||||
cdf1 = norm.cdf(x, mu1, sig1)
|
||||
|
||||
ax2.plot(x, cdf0, color=C_NEG, linewidth=1.8, label=r"$\hat{F}_0$ (neg.)")
|
||||
ax2.plot(x, cdf1, color=C_POS, linewidth=1.8, label=r"$\hat{F}_1$ (pos.)")
|
||||
ax2.axvline(tau, color=C_TAU, linestyle="--", linewidth=1.3)
|
||||
|
||||
# Horizontal reference lines
|
||||
ax2.axhline(F0_tau, color=C_NEG, linestyle="-.", linewidth=0.9, alpha=0.7)
|
||||
ax2.axhline(F1_tau, color=C_POS, linestyle="-.", linewidth=0.9, alpha=0.7)
|
||||
|
||||
# Dots at the intercepts
|
||||
ax2.plot(tau, F0_tau, "o", color=C_NEG, markersize=5, zorder=4)
|
||||
ax2.plot(tau, F1_tau, "o", color=C_POS, markersize=5, zorder=4)
|
||||
|
||||
# Right-side labels for the intercepts (placed where CDFs have flattened to ~1)
|
||||
x_label_right = x[-1] - 0.2
|
||||
ax2.text(x_label_right, F0_tau, rf"$1-\alpha$",
|
||||
fontsize=8, color=C_NEG, va="bottom", ha="right")
|
||||
ax2.text(x_label_right, F1_tau, r"$\hat{F}_1(\tau)$" + "\n(loss)",
|
||||
fontsize=7.5, color=C_POS, va="top", ha="right")
|
||||
|
||||
# TPR annotation: double-headed arrow on the right + label
|
||||
x_bracket = tau + 1.9
|
||||
ax2.annotate("", xy=(x_bracket, 1.0), xytext=(x_bracket, F1_tau),
|
||||
arrowprops=dict(arrowstyle="<->", color="black",
|
||||
lw=0.8, mutation_scale=10))
|
||||
ax2.text(x_bracket + 0.12, (1.0 + F1_tau) / 2,
|
||||
f"TPR\n= {tpr*100:.0f}%",
|
||||
fontsize=7.5, va="center", ha="left", color="black")
|
||||
|
||||
ax2.set_ylabel("CDF", fontsize=9)
|
||||
ax2.set_xlabel(r"Score $f_\theta(\mathbf{x})$", fontsize=9)
|
||||
ax2.set_yticks([0.0, 0.25, 0.50, 0.75, 1.0])
|
||||
ax2.set_yticklabels(["0", "0.25", "0.5", "0.75", "1"], fontsize=7.5)
|
||||
ax2.set_xlim(x[0] - 0.3, x[-1])
|
||||
ax2.legend(fontsize=7.5, loc="upper left", framealpha=0.88)
|
||||
|
||||
for ax in (ax1, ax2):
|
||||
ax.tick_params(labelsize=8)
|
||||
ax.spines[["top", "right"]].set_visible(False)
|
||||
|
||||
fig.savefig(f"{OUT_DIR}/roll_principle.pdf", bbox_inches="tight")
|
||||
fig.savefig(f"{OUT_DIR}/roll_principle.png", bbox_inches="tight", dpi=180)
|
||||
print(f"Saved to {OUT_DIR}/roll_principle.{{pdf,png}}")
|
||||
print(f"tau={tau:.3f} F0(tau)={F0_tau:.3f} F1(tau)={F1_tau:.3f} TPR={tpr:.3f}")
|
||||
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 64 KiB |
@@ -0,0 +1,108 @@
|
||||
"""
|
||||
Illustrates the gradient locality property of KDE-ROLL.
|
||||
|
||||
Upper panel: score distributions for both classes with operating threshold.
|
||||
Lower panel: gradient magnitude per sample, showing bell-shaped decay from threshold.
|
||||
|
||||
Run from the thesis root:
|
||||
nix develop --command python3 content/method/figures/threshold_weighting.py
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
# --- Add these configuration rules right here ---
|
||||
plt.rcParams.update({
|
||||
"text.usetex": False, # Rely on Matplotlib's math parser instead of the system compiler
|
||||
"mathtext.fontset": "cm", # Emulate standard LaTeX Computer Modern styles flawlessly
|
||||
"pdf.fonttype": 42 # Ensure fonts are embedded as TrueType vector curves instead of Type 3 paths
|
||||
})
|
||||
|
||||
OUT_DIR = "content/method/figures"
|
||||
|
||||
np.random.seed(7)
|
||||
|
||||
# --- Data ------------------------------------------------------------------
|
||||
n = 120
|
||||
alpha = 0.25 # target FPR
|
||||
|
||||
neg_scores = np.sort(np.random.normal(-0.6, 0.75, n))
|
||||
pos_scores = np.sort(np.random.normal(1.1, 0.80, n))
|
||||
|
||||
# Threshold: alpha-quantile of the negative-class scores
|
||||
tau = np.percentile(neg_scores, 100 * (1 - alpha))
|
||||
|
||||
# --- Bandwidth (Silverman's rule per class) --------------------------------
|
||||
h_neg = 1.06 * neg_scores.std() * n ** (-1 / 5)
|
||||
h_pos = 1.06 * pos_scores.std() * n ** (-1 / 5)
|
||||
|
||||
|
||||
def sigma_prime(x, h):
|
||||
"""Logistic kernel PDF: derivative of the sigmoid-smoothed CDF."""
|
||||
e = np.exp(-np.abs(x) / h)
|
||||
return e / (h * (1 + e) ** 2)
|
||||
|
||||
|
||||
# --- KDE-ROLL gradients (eq:kde-grad-combined) ----------------------------
|
||||
# y=1: (1/n) * sigma_prime(score - tau)
|
||||
grad_pos = (1.0 / n) * sigma_prime(pos_scores - tau, h_pos)
|
||||
|
||||
# y=0: -(scale) * sigma_prime(tau - score_i) / sum_j[sigma_prime(tau - score_j)]
|
||||
# where scale = (1/n) * sum_j[sigma_prime(pos_j - tau)]
|
||||
neg_kernel = sigma_prime(tau - neg_scores, h_neg)
|
||||
scale = (1.0 / n) * np.sum(sigma_prime(pos_scores - tau, h_pos))
|
||||
grad_neg_signed = -scale * neg_kernel / neg_kernel.sum()
|
||||
grad_neg_abs = np.abs(grad_neg_signed)
|
||||
|
||||
# --- Smooth reference bell curves -----------------------------------------
|
||||
x_ref = np.linspace(neg_scores.min() - 0.3, pos_scores.max() + 0.3, 400)
|
||||
bell_pos = sigma_prime(x_ref - tau, h_pos)
|
||||
bell_pos *= (grad_pos.max() / bell_pos.max())
|
||||
bell_neg = sigma_prime(tau - x_ref, h_neg)
|
||||
bell_neg *= (grad_neg_abs.max() / bell_neg.max())
|
||||
|
||||
# --- Plot -----------------------------------------------------------------
|
||||
C_NEG = "#4477AA"
|
||||
C_POS = "#CC6633"
|
||||
C_TAU = "#333333"
|
||||
|
||||
fig, (ax1, ax2) = plt.subplots(
|
||||
2, 1, figsize=(5.5, 4.2), sharex=True,
|
||||
gridspec_kw={"height_ratios": [1.3, 1.0]}
|
||||
)
|
||||
fig.subplots_adjust(hspace=0.06)
|
||||
|
||||
# — Top: score distributions —
|
||||
ax1.hist(neg_scores, bins=20, density=True, alpha=0.45, color=C_NEG,
|
||||
label=r"Negative class ($y=0$)")
|
||||
ax1.hist(pos_scores, bins=20, density=True, alpha=0.45, color=C_POS,
|
||||
label=r"Positive class ($y=1$)")
|
||||
ax1.axvline(tau, color=C_TAU, linestyle="--", linewidth=1.4,
|
||||
label=rf"Threshold $\tau$ (FPR $=\alpha={alpha}$)")
|
||||
ax1.set_ylabel("Density", fontsize=9)
|
||||
ax1.set_yticks([])
|
||||
ax1.legend(fontsize=7.5, loc="upper right", framealpha=0.85)
|
||||
|
||||
# — Bottom: gradient magnitudes —
|
||||
ax2.scatter(neg_scores, grad_neg_abs, color=C_NEG, s=12, zorder=3,
|
||||
alpha=0.75, label=r"$|\partial\mathcal{L}/\partial f_\theta(\mathbf{x})|$, $y=0$")
|
||||
ax2.scatter(pos_scores, grad_pos, color=C_POS, s=12, zorder=3,
|
||||
alpha=0.75, label=r"$\partial\mathcal{L}/\partial f_\theta(\mathbf{x})$, $y=1$")
|
||||
ax2.plot(x_ref, bell_neg, color=C_NEG, linewidth=1.2, alpha=0.5)
|
||||
ax2.plot(x_ref, bell_pos, color=C_POS, linewidth=1.2, alpha=0.5)
|
||||
ax2.axvline(tau, color=C_TAU, linestyle="--", linewidth=1.4)
|
||||
ax2.set_ylabel("Gradient magnitude", fontsize=9)
|
||||
ax2.set_xlabel(r"Score $f_\theta(\mathbf{x})$", fontsize=9)
|
||||
ax2.set_yticks([])
|
||||
ax2.legend(fontsize=7.5, loc="upper right", framealpha=0.85)
|
||||
|
||||
for ax in (ax1, ax2):
|
||||
ax.tick_params(labelsize=8)
|
||||
ax.spines[["top", "right"]].set_visible(False)
|
||||
|
||||
fig.savefig(f"{OUT_DIR}/threshold_weighting.pdf", bbox_inches="tight")
|
||||
fig.savefig(f"{OUT_DIR}/threshold_weighting.png", bbox_inches="tight", dpi=180)
|
||||
print(f"Saved to {OUT_DIR}/threshold_weighting.{{pdf,png}}")
|
||||
print(f"Sum neg: {np.sum(grad_neg_signed)}, pos: {np.sum(grad_pos)}")
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 86 KiB |
@@ -0,0 +1,123 @@
|
||||
"""
|
||||
Illustrates the gradient locality property of Gaussian-ROLL.
|
||||
|
||||
Upper panel: score distributions for both classes with fitted Gaussian PDFs
|
||||
and operating threshold.
|
||||
Lower panel: gradient magnitude per sample using the Gaussian gradient formula
|
||||
(eq:gauss-grad-combined).
|
||||
|
||||
Run from the thesis root:
|
||||
nix develop --command python3 content/method/figures/threshold_weighting_gaussian.py
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
from scipy import stats
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
plt.rcParams.update({
|
||||
"text.usetex": False,
|
||||
"mathtext.fontset": "cm",
|
||||
"pdf.fonttype": 42
|
||||
})
|
||||
|
||||
OUT_DIR = "content/method/figures"
|
||||
|
||||
np.random.seed(7)
|
||||
|
||||
# --- Data ------------------------------------------------------------------
|
||||
n = 120
|
||||
alpha = 0.25 # target FPR
|
||||
|
||||
neg_scores = np.sort(np.random.normal(-0.6, 0.75, n))
|
||||
pos_scores = np.sort(np.random.normal(1.1, 0.80, n))
|
||||
|
||||
n0 = len(neg_scores)
|
||||
n1 = len(pos_scores)
|
||||
|
||||
# --- Gaussian MLE parameters -----------------------------------------------
|
||||
mu0, sigma0 = neg_scores.mean(), neg_scores.std()
|
||||
mu1, sigma1 = pos_scores.mean(), pos_scores.std()
|
||||
|
||||
# Threshold: (1-alpha)-quantile of negative class so that FPR = alpha
|
||||
# (FPR = P(score > tau | y=0) = 1 - F_0(tau), so F_0(tau) = 1-alpha)
|
||||
tau = stats.norm.ppf(1 - alpha, loc=mu0, scale=sigma0)
|
||||
|
||||
# Shared scalar: pdf of positive class evaluated at the threshold
|
||||
common = (1.0 / (sigma1 * np.sqrt(2 * np.pi))) * np.exp(-0.5 * ((tau - mu1) / sigma1) ** 2)
|
||||
|
||||
# --- Gaussian ROLL gradients (eq:gauss-grad-combined) ---------------------
|
||||
# y=1: -(1/n1) * common * (1 + (tau-mu1)*(s-mu1)/sigma1^2)
|
||||
grad_pos_signed = -(1.0 / n1) * common * (
|
||||
1 + (tau - mu1) * (pos_scores - mu1) / sigma1 ** 2
|
||||
)
|
||||
grad_pos_abs = np.abs(grad_pos_signed)
|
||||
|
||||
# y=0: +(1/n0) * common * (1 + (tau-mu0)*(s-mu0)/sigma0^2)
|
||||
grad_neg_signed = (1.0 / n0) * common * (
|
||||
1 + (tau - mu0) * (neg_scores - mu0) / sigma0 ** 2
|
||||
)
|
||||
grad_neg_abs = np.abs(grad_neg_signed)
|
||||
|
||||
# Gradient balance check (should be ~0)
|
||||
print(f"Gradient balance: pos sum = {grad_pos_signed.sum():.6f}, "
|
||||
f"neg sum = {grad_neg_signed.sum():.6f}, "
|
||||
f"total = {(grad_pos_signed.sum() + grad_neg_signed.sum()):.2e}")
|
||||
|
||||
# --- Smooth reference curves (continuous gradient functions) ---------------
|
||||
x_ref = np.linspace(neg_scores.min() - 0.3, pos_scores.max() + 0.3, 400)
|
||||
smooth_pos_abs = np.abs(-(1.0 / n1) * common * (
|
||||
1 + (tau - mu1) * (x_ref - mu1) / sigma1 ** 2
|
||||
))
|
||||
smooth_neg_abs = np.abs((1.0 / n0) * common * (
|
||||
1 + (tau - mu0) * (x_ref - mu0) / sigma0 ** 2
|
||||
))
|
||||
|
||||
# --- Plot -----------------------------------------------------------------
|
||||
C_NEG = "#4477AA"
|
||||
C_POS = "#CC6633"
|
||||
C_TAU = "#333333"
|
||||
|
||||
fig, (ax1, ax2) = plt.subplots(
|
||||
2, 1, figsize=(5.5, 4.2), sharex=True,
|
||||
gridspec_kw={"height_ratios": [1.3, 1.0]}
|
||||
)
|
||||
fig.subplots_adjust(hspace=0.06)
|
||||
|
||||
# — Top: score distributions with fitted Gaussian PDFs —
|
||||
ax1.hist(neg_scores, bins=20, density=True, alpha=0.35, color=C_NEG,
|
||||
label=r"Negative class ($y=0$)")
|
||||
ax1.hist(pos_scores, bins=20, density=True, alpha=0.35, color=C_POS,
|
||||
label=r"Positive class ($y=1$)")
|
||||
ax1.plot(x_ref, stats.norm.pdf(x_ref, mu0, sigma0),
|
||||
color=C_NEG, linewidth=1.8, label=r"Fitted $\mathcal{N}(\mu_0,\sigma_0^2)$")
|
||||
ax1.plot(x_ref, stats.norm.pdf(x_ref, mu1, sigma1),
|
||||
color=C_POS, linewidth=1.8, label=r"Fitted $\mathcal{N}(\mu_1,\sigma_1^2)$")
|
||||
ax1.axvline(tau, color=C_TAU, linestyle="--", linewidth=1.4,
|
||||
label=rf"Threshold $\tau$ (FPR $=\alpha={alpha}$)")
|
||||
ax1.set_ylabel("Density", fontsize=9)
|
||||
ax1.set_yticks([])
|
||||
ax1.legend(fontsize=7, loc="upper right", framealpha=0.85)
|
||||
|
||||
# — Bottom: gradient magnitudes —
|
||||
ax2.scatter(neg_scores, grad_neg_abs, color=C_NEG, s=12, zorder=3,
|
||||
alpha=0.75, label=r"$|\partial\mathcal{L}/\partial f_\theta(\mathbf{x})|$, $y=0$")
|
||||
ax2.scatter(pos_scores, grad_pos_abs, color=C_POS, s=12, zorder=3,
|
||||
alpha=0.75, label=r"$|\partial\mathcal{L}/\partial f_\theta(\mathbf{x})|$, $y=1$")
|
||||
ax2.plot(x_ref, smooth_neg_abs, color=C_NEG, linewidth=1.2, alpha=0.55)
|
||||
ax2.plot(x_ref, smooth_pos_abs, color=C_POS, linewidth=1.2, alpha=0.55)
|
||||
ax2.axvline(tau, color=C_TAU, linestyle="--", linewidth=1.4)
|
||||
ax2.set_ylabel("Gradient magnitude", fontsize=9)
|
||||
ax2.set_xlabel(r"Score $f_\theta(\mathbf{x})$", fontsize=9)
|
||||
ax2.set_yticks([])
|
||||
ax2.legend(fontsize=7.5, loc="upper right", framealpha=0.85)
|
||||
|
||||
for ax in (ax1, ax2):
|
||||
ax.tick_params(labelsize=8)
|
||||
ax.spines[["top", "right"]].set_visible(False)
|
||||
|
||||
out_base = f"{OUT_DIR}/threshold_weighting_gaussian"
|
||||
fig.savefig(f"{out_base}.pdf", bbox_inches="tight")
|
||||
fig.savefig(f"{out_base}.png", bbox_inches="tight", dpi=180)
|
||||
print(f"Saved to {out_base}.{{pdf,png}}")
|
||||
+459
-165
@@ -2,100 +2,49 @@
|
||||
\section{Problem Formulation}
|
||||
\label{sec:problem-formulation}
|
||||
|
||||
\subsection{Imbalanced Classification and The TPR/FPR Objective}
|
||||
\label{sec:imbalanced-tpr-fpr}
|
||||
% Motivation for this section — why accuracy fails under class imbalance, why
|
||||
% TPR@FPR gives direct operating-point control, and the Neyman-Pearson framing
|
||||
% that underpins ROLL — is developed in \Cref{chapter:related_work}
|
||||
% (\Cref{sec:imbalanced-tpr-fpr,sec:neyman-pearson}). This section establishes
|
||||
% only the notation and objective used throughout the chapter.
|
||||
Given a dataset $\mathcal{D} = \{(\mathbf{x}_i, y_i)\}_{i=1}^{n}$ with inputs
|
||||
$\mathbf{x}_i \in \mathcal{X}$ and binary labels $y_i \in \{0, 1\}$, we learn a
|
||||
scoring function $f_\theta : \mathcal{X} \to \mathbb{R}$ by fitting.
|
||||
The positive and negative subsets of $\mathcal{D}$ are:
|
||||
\[
|
||||
\mathcal{D}_1 = \{(\mathbf{x}_i, y_i) \in \mathcal{D} \mid y_i = 1\},
|
||||
\qquad
|
||||
\mathcal{D}_0 = \{(\mathbf{x}_i, y_i) \in \mathcal{D} \mid y_i = 0\}.
|
||||
\]
|
||||
Applying a threshold $\tau$ produces predictions $\hat{y} = \mathbf{1}[f_\theta(\mathbf{x}) > \tau]$.
|
||||
The \emph{true positive rate} (TPR) and \emph{false positive rate} (FPR) at $\tau$ are:
|
||||
\begin{align}
|
||||
\operatorname{TPR}(\alpha) &= P\!\bigl(f_\theta(\mathbf{x}) > \tau \mid y = 1\bigr), \\
|
||||
\operatorname{FPR}(\alpha) &= P\!\bigl(f_\theta(\mathbf{x}) > \tau \mid y = 0\bigr).
|
||||
\end{align}
|
||||
In our scenario, we either wish to learn $f_{\theta}$ in order to maximize $\operatorname{TPR}$
|
||||
subject to a fixed $\operatorname{FPR}$, or vice versa. Our fixed $\operatorname{TPR}$ or $\operatorname{FPR}$
|
||||
will be fixed to a value $\alpha \in (0, 1)$.
|
||||
|
||||
% IMPROVEMENT SUGGESTIONS:
|
||||
% - Section title promises TPR/FPR but neither is formally defined here; the section
|
||||
% must land on a concrete statement of the TPR@FPR objective with proper notation.
|
||||
% - The inseparability tangent (paragraphs 2-3) dilutes the argument; trim to one
|
||||
% sentence or move to Related Work. The cascade example in particular reads as
|
||||
% background material unrelated to the TPR/FPR framing.
|
||||
% - Narrative arc is currently backwards: the comment block near the end (lines
|
||||
% starting 'Motivate why standard loss functions...') is a better outline than
|
||||
% the prose above it. Drive the section with: accuracy is gameable under imbalance
|
||||
% -> class weighting is the standard fix -> it gives no direct control over the
|
||||
% operating point -> therefore we need TPR@FPR directly.
|
||||
% - $\mathcal{D}_0$ and $\mathcal{D}_1$ are used but never defined; add definitions
|
||||
% for the positive/negative subsets of $\mathcal{D}$ before first use.
|
||||
% - Suggested arc: (1) standard setup + accuracy objective; (2) why accuracy fails
|
||||
% under imbalance -- the trivial majority-class classifier; (3) standard remedies
|
||||
% and why they give only indirect control; (4) formally define TPR, FPR, ROC, and
|
||||
% state the target objective: maximise TPR at a fixed FPR budget $\alpha$.
|
||||
Therefor, when maximizing our $\operatorname{TPR}$ subject to a fixed $\operatorname{FPR}$, we are solving for:
|
||||
\begin{equation}
|
||||
\max_\theta \; \operatorname{TPR}\!\bigl(\tau^*(\theta)\bigr)
|
||||
\quad \text{s.t.} \quad \operatorname{FPR}\!\bigl(\tau^*(\theta)\bigr) = \alpha,
|
||||
\label{eq:objective}
|
||||
\end{equation}
|
||||
|
||||
When fitting a classifier, it is most common to try and maximize the probability
|
||||
of success. That is, given some dataset
|
||||
$\mathcal{D} = \{(\mathbf{x}_i, y_i)\}_{i=1}^{n}$
|
||||
with inputs $\mathbf{x}_i \in \mathcal{X}$ and binary labels $y_i \in \{0, 1\}$, we learn a scoring function $f_\theta : \mathcal{X} \to \mathbb{R}$ and apply a threshold $\tau$ to produce predictions $\hat{y} = \mathbf{1}[f_\theta(\mathbf{x}) > \tau]$. The standard objective is to minimize the probability of classification error:
|
||||
where $\tau^*(\theta)$ is the threshold that achieves exactly $\operatorname{FPR} = \alpha$
|
||||
for the current model $f_\theta$.
|
||||
Conversley, when we are minimizing $\operatorname{FPR}$ subject to a fixed $\operatorname{TPR}$ we are solving for:
|
||||
|
||||
$$\mathcal{L}(\theta) = \frac{1}{n} \sum_{i=1}^{n} \mathbf{1}[\hat{y}_i \neq y_i] = P(\hat{y} \neq y)$$
|
||||
|
||||
which is equivalent to maximizing classification accuracy $P(\hat{y} = y)$.
|
||||
|
||||
For most applications, this optimization is warranted. However, there are many real-world settings in which
|
||||
this type of fitting is ill-suited.
|
||||
|
||||
One such application is in the case of inseparable classes. In many cases, no meaningful
|
||||
separation boundary can be found between $\mathcal{D}_0$ and $\mathcal{D}_1$. In many cases,
|
||||
especially in tabular datasets, there can be two data elements $i,j$ for which $\mathbf{x}_i = \mathbf{x}_j$,
|
||||
however, $y_i \neq y_j$. Any deterministic classifier will have to decide whether to classify
|
||||
inseparable points as one class or the other, but it remains hard to control the behavior of
|
||||
a classifier while it is fitting to suggest one outcome or the other.
|
||||
|
||||
The issue of inseparability becomes more apparent in certain applications where runtime is a key
|
||||
and limiting factor. In many classification pipelines, a ``quick and dirty'' classifier may
|
||||
filter out many candidates before an ``in depth'' classifier gives the final answer. This may be done
|
||||
in events where running the full in-depth classification on all data points is not a cost-effective or time-efficient solution.
|
||||
The ``quick and dirty'' classifier can be either low in parameters, making it run quickly. It may also only
|
||||
make a quick decision based on part of the data point. For example, in the case of a classifier looking to flag
|
||||
images of the ocean, a ``quick and dirty'' classifier may look at a very-low resolution version of the image and simply check if it is relatively ``blue''. It filters out many certain false candidates this way, then passes on the remainder to an in-depth, higher accuracy classifier. Solutions like this allow to run at near-optimal performance with lower compute resources in limited-time applications. These situations are common in large-volume image applications on the internet, as well as in astronomy/microscopy.
|
||||
|
||||
Another such application is a case of highly imbalanced data. Even with separability, it is very
|
||||
easy to a classifier to simply ``ignore'' the minority class. This may very likely be a local minimum
|
||||
while fitting, which may be inescapable. As was once said, ``a classifier that always returns 0 is
|
||||
well fitted for flagging pictures of Michael Jordan bathing in green M\&Ms''. Combined with inseparability, we see that in extreme cases, classifiers may choose to ignore, or otherwise penalize, the minority class.
|
||||
|
||||
For every trained classifier, the output per model is a score (TODO rewrite and formalize).
|
||||
|
||||
TODO finish this subsection
|
||||
|
||||
\begin{itemize}
|
||||
\item What usually happens
|
||||
\begin{itemize}
|
||||
\item Fitting
|
||||
\item Graphing ROC and picking threshold based on tradeoff
|
||||
\end{itemize}
|
||||
\item Usually we compensate for poor performance with
|
||||
\begin{itemize}
|
||||
\item Weighting between classes
|
||||
\item Possible other methods
|
||||
\end{itemize}
|
||||
\item Problem with this method is lack of control.
|
||||
\end{itemize}
|
||||
|
||||
|
||||
% Motivate why standard loss functions (cross-entropy, MSE) break down under class
|
||||
% imbalance: they optimize average accuracy, which a model can game by predicting
|
||||
% the majority class. Introduce concrete examples (e.g. medical diagnosis, fraud
|
||||
% detection) where the cost of a false negative vastly outweighs a false positive,
|
||||
% and where the practitioner needs to operate at a specific FPR budget. Argue that
|
||||
% what is actually needed is direct control over TPR at a fixed FPR, not a
|
||||
% surrogate that only loosely correlates with it.
|
||||
|
||||
|
||||
|
||||
|
||||
\subsection{Connection to Neyman-Pearson}
|
||||
\label{sec:neyman-pearson}
|
||||
|
||||
% Introduce the Neyman-Pearson lemma: among all tests at a given false-positive rate,
|
||||
% the likelihood-ratio test maximizes TPR. Frame ROLL as a learnable, gradient-based
|
||||
% realization of this principle — rather than assuming known class-conditional
|
||||
% distributions (as classical NP does), ROLL estimates them from model scores during
|
||||
% training. Briefly note that existing NP-inspired methods (covered in
|
||||
% \Cref{chapter:related_work}) do not support deep learning / gradient-based
|
||||
% optimization; this is the gap ROLL fills.
|
||||
\begin{equation}
|
||||
\min_\theta \; \operatorname{FPR}\!\bigl(\tau^*(\theta)\bigr)
|
||||
\quad \text{s.t.} \quad \operatorname{TPR}\!\bigl(\tau^*(\theta)\bigr) = \alpha,
|
||||
\label{eq:objective}
|
||||
\end{equation}
|
||||
|
||||
The ROLL framework, introduced in
|
||||
\Cref{sec:roll-framework}, provides a method of differentiable surrogate for these objectives.
|
||||
|
||||
%------------------------------------------------
|
||||
|
||||
@@ -109,7 +58,7 @@ The core idea of ROLL is as follows: when training, the user selects that they w
|
||||
TPR given some fixed FPR. Then, during training, on every training step, the actual TPR is calculated for
|
||||
the given FPR, and the fitting algorithm tries to maximize the TPR directly. This way, while training, we gain direct control over the performance at the threshold that we care about - avoiding needing to pick out a threshold later.
|
||||
|
||||
In Neyman-Pearson terms, our algorithm wishes to directly minimize the Type-1 error given a fixed Type-2 error, or vice versa.
|
||||
In Neyman-Pearson terms~\cite{tong2016neyman}, our algorithm wishes to directly minimize the Type-1 error given a fixed Type-2 error, or vice versa.
|
||||
|
||||
In practice, however, things are not so simple.
|
||||
|
||||
@@ -163,7 +112,8 @@ This necessitates a different approach, to make these calculations differentiabl
|
||||
|
||||
The way to overcome this problem in differentiability is not to calculate the TPR and FPR
|
||||
directly, but to estimate them by fitting probability distributions to the score output of the model at
|
||||
every step. Contiguous probability distributions are contiguous w.r.t their inputs. Additionally, we can calculate the TPR, FPR given the probability distributions CDF and ICDF functions. If we can gaurantee these functions to be differentiable, which proved to be the main challenge of this work, we can gaurantee this process to be differentiable.
|
||||
every step. Contiguous probability distributions are contiguous w.r.t their defining parameters, and a derivation of the parameters w.r.t the distributions themselves can be made.
|
||||
Additionally, we can calculate the TPR, FPR given the probability distributions CDF and ICDF functions. If we can gaurantee these functions to be differentiable, which proved to be the main challenge of this work, we can gaurantee the entire process to be differentiable. Once we do so, a derivative of the scores themselves w.r.t. the loss can be acheived.
|
||||
|
||||
|
||||
\subsection{General ROLL Formulation and Derivation}
|
||||
@@ -208,11 +158,11 @@ Let $\hat{F}_1$ and $\hat{F}_0$ denote the CDFs of $P_{\hat{\phi}_1}$ and $P_{\h
|
||||
respectively. Specific choices of $\mathcal{F}$ (Gaussian, Beta, KDE) are discussed in
|
||||
\Cref{sec:roll-instantiations}.
|
||||
|
||||
Using this notation, at a threshold $\alpha$:
|
||||
Using this notation, at a threshold $\tau$:
|
||||
\begin{align}
|
||||
\operatorname{TPR}(\alpha) &= 1 - \hat{F}_1(\alpha), \label{eq:tpr-cdf} \\
|
||||
\operatorname{FPR}(\alpha) &= \hat{F}_0(\alpha). \label{eq:fpr-cdf}
|
||||
\end{align}
|
||||
\operatorname{TPR}(\tau) &= 1 - \hat{F}_1(\tau), \label{eq:tpr-cdf} \\
|
||||
\operatorname{FPR}(\tau) &= 1 - \hat{F}_0(\tau). \label{eq:fpr-cdf}
|
||||
\end{align}t
|
||||
|
||||
Let $\hat{F}_1^{-1}$ and $\hat{F}_0^{-1}$ denote the quantile functions of $P_{\hat{\phi}_1}$ and
|
||||
$P_{\hat{\phi}_0}$, respectively. The threshold achieving a target TPR of $\alpha$ follows directly
|
||||
@@ -223,45 +173,70 @@ from \eqref{eq:tpr-cdf}:
|
||||
\]
|
||||
and similarly for a target FPR of $\alpha$, from \eqref{eq:fpr-cdf}:
|
||||
\[
|
||||
\operatorname{FPR}(\tau) = \alpha \;\Longrightarrow\; \hat{F}_0(\tau) = \alpha
|
||||
\;\Longrightarrow\; \tau = \hat{F}_0^{-1}(\alpha)
|
||||
\operatorname{FPR}(\tau) = \alpha \;\Longrightarrow\; 1 - \hat{F}_0(\tau) = \alpha
|
||||
\;\Longrightarrow\; \tau = \hat{F}_0^{-1}(1 - \alpha)
|
||||
\]
|
||||
|
||||
This yields two versions of our objective. Substituting the ICDF threshold into the rate
|
||||
expressions, our estimated TPR at a fixed FPR of $\alpha$ is $1 - \hat{F}_1(\hat{F}_0^{-1}(\alpha))$.
|
||||
expressions, our estimated TPR at a fixed FPR of $\alpha$ is $1 - \hat{F}_1(\hat{F}_0^{-1}(1-\alpha))$.
|
||||
Therefore, to maximise TPR we minimise:
|
||||
\begin{equation}
|
||||
\mathcal{L}_{\text{ROLL-TPR@FPR}}(f_{\theta}(\mathbf{X}) ; \alpha) = \hat{F}_1(\hat{F}_0^{-1}(\alpha))
|
||||
\mathcal{L}_{\text{ROLL-TPR@FPR}}(f_{\theta}(\mathcal{B}) ; \alpha) = \hat{F}_1(\hat{F}_0^{-1}(1-\alpha))
|
||||
\label{eq:roll-tpr-at-fpr}
|
||||
\end{equation}
|
||||
|
||||
Our estimated FPR at a fixed TPR of $\alpha$ is $\hat{F}_0(\hat{F}_1^{-1}(1 - \alpha))$.
|
||||
Our estimated FPR at a fixed TPR of $\alpha$ is $1 - \hat{F}_0(\hat{F}_1^{-1}(1 - \alpha))$.
|
||||
Therefore, to minimise FPR:
|
||||
\begin{equation}
|
||||
\mathcal{L}_{\text{ROLL-FPR@TPR}}(f_{\theta}(\mathbf{X}) ; \alpha) = \hat{F}_0(\hat{F}_1^{-1}(1 - \alpha))
|
||||
\mathcal{L}_{\text{ROLL-FPR@TPR}}(f_{\theta}(\mathbf{X}) ; \alpha) = 1 - \hat{F}_0(\hat{F}_1^{-1}(1 - \alpha))
|
||||
\label{eq:roll-fpr-at-tpr}
|
||||
\end{equation}
|
||||
|
||||
TODO explain the trick where we can only use one version - base off code.
|
||||
Observe that \eqref{eq:roll-tpr-at-fpr} and \eqref{eq:roll-fpr-at-tpr} share the same
|
||||
inner structure: each evaluates one empirical CDF at the $(1-\alpha)$-quantile of the other.
|
||||
The two objectives are interchangeable at the implementation level: replacing
|
||||
$f_\theta(\mathbf{x})$ with $-f_\theta(\mathbf{x})$ and $y$ with $1 - y$ swaps the roles
|
||||
of the two populations, and passing target rate $1-\alpha$ to the resulting
|
||||
TPR@FPR objective recovers the FPR@TPR objective at rate $\alpha$. Because
|
||||
the two objectives are therefore interchangeable at the implementation level, all derivations
|
||||
and implementation details below are given for \eqref{eq:roll-tpr-at-fpr} only;
|
||||
\eqref{eq:roll-fpr-at-tpr} follows by the same transformation applied to the inputs, with
|
||||
the target rate complemented to $1-\alpha$.
|
||||
\Cref{fig:roll-principle} provides a geometric view of \eqref{eq:roll-tpr-at-fpr}.
|
||||
|
||||
\begin{figure}[H]
|
||||
\centering
|
||||
\includegraphics[width=0.85\textwidth]{content/method/figures/roll_principle.png}
|
||||
\caption{Geometric picture of the ROLL TPR@FPR objective. \textit{Upper}: score PDFs
|
||||
for the negative (blue, $y=0$) and positive (orange, $y=1$) classes. The operating
|
||||
threshold $\tau$ (dashed) is placed at the $(1-\alpha)$-quantile of the negative
|
||||
class, so that exactly $\alpha$ fraction of negative scores exceed it ($\operatorname{FPR}=\alpha$).
|
||||
The shaded area under the positive-class PDF to the left of $\tau$ is
|
||||
$\hat{F}_1(\tau)$ --- the ROLL loss we minimize. Tick marks along the bottom represent
|
||||
observed positive-class scores. \textit{Lower}: corresponding CDFs $\hat{F}_0$ (blue)
|
||||
and $\hat{F}_1$ (orange). Reading horizontally at height $1-\alpha$ on $\hat{F}_0$
|
||||
gives $\tau$; reading $\hat{F}_1(\tau)$ off $\hat{F}_1$ at the same $\tau$ gives the
|
||||
loss; the distance from $\hat{F}_1(\tau)$ to $1$ is the resulting TPR.}
|
||||
\label{fig:roll-principle}
|
||||
\end{figure}
|
||||
|
||||
We now derive the gradient of \eqref{eq:roll-tpr-at-fpr} with respect to $f_\theta(\mathbf{x}_i)$.
|
||||
As both loss forms are symmetric in structure, it is sufficient to treat the TPR@FPR case.
|
||||
|
||||
If $y_i = 1$, then $(\mathbf{x}_i, y_i) \in \mathcal{B}_1$ and $f_\theta(\mathbf{x}_i)$ has no
|
||||
effect on $\hat{F}_0^{-1}(\alpha)$. Therefore:
|
||||
effect on $\hat{F}_0^{-1}(1-\alpha)$. Therefore:
|
||||
\begin{equation}
|
||||
\left.\frac{\partial \mathcal{L}_{\text{ROLL-TPR@FPR}}(f_\theta(\mathbf{X}) ; \alpha)}
|
||||
\left.\frac{\partial \mathcal{L}_{\text{ROLL-TPR@FPR}}(f_\theta(\mathcal{B}) ; \alpha)}
|
||||
{\partial f_\theta(\mathbf{x}_i)}\right|_{y_i = 1}
|
||||
= \frac{\partial \hat{F}_1(\hat{F}_0^{-1}(\alpha))}{\partial f_\theta(\mathbf{x}_i)}
|
||||
= \frac{\partial \hat{F}_1(\hat{F}_0^{-1}(1-\alpha))}{\partial f_\theta(\mathbf{x}_i)}
|
||||
\end{equation}
|
||||
|
||||
If $y_i = 0$, then $(\mathbf{x}_i, y_i) \in \mathcal{B}_0$ and $f_\theta(\mathbf{x}_i)$ affects
|
||||
$\hat{F}_0^{-1}(\alpha)$ instead. Applying the chain rule:
|
||||
$\hat{F}_0^{-1}(1-\alpha)$ instead. Applying the chain rule:
|
||||
\begin{equation}
|
||||
\left.\frac{\partial \mathcal{L}_{\text{ROLL-TPR@FPR}}(f_\theta(\mathbf{X}) ; \alpha)}
|
||||
{\partial f_\theta(\mathbf{x}_i)}\right|_{y_i = 0}
|
||||
= \frac{\partial \hat{F}_1(\hat{F}_0^{-1}(\alpha))}{\partial \hat{F}_0^{-1}(\alpha)}
|
||||
\cdot \frac{\partial \hat{F}_0^{-1}(\alpha)}{\partial f_\theta(\mathbf{x}_i)}
|
||||
= \frac{\partial \hat{F}_1(\hat{F}_0^{-1}(1-\alpha))}{\partial \hat{F}_0^{-1}(1-\alpha)}
|
||||
\cdot \frac{\partial \hat{F}_0^{-1}(1-\alpha)}{\partial f_\theta(\mathbf{x}_i)}
|
||||
\end{equation}
|
||||
|
||||
Unifying both cases, the general gradient applicable to all ROLL instantiations is:
|
||||
@@ -269,10 +244,10 @@ Unifying both cases, the general gradient applicable to all ROLL instantiations
|
||||
\frac{\partial \mathcal{L}_{\text{ROLL-TPR@FPR}}(f_\theta(\mathbf{X}) ; \alpha)}
|
||||
{\partial f_\theta(\mathbf{x}_i)} =
|
||||
\begin{cases}
|
||||
\dfrac{\partial \hat{F}_1(\hat{F}_0^{-1}(\alpha))}{\partial f_\theta(\mathbf{x}_i)}
|
||||
\dfrac{\partial \hat{F}_1(\hat{F}_0^{-1}(1-\alpha))}{\partial f_\theta(\mathbf{x}_i)}
|
||||
& \text{if } y_i = 1 \\[10pt]
|
||||
\dfrac{\partial \hat{F}_1(\hat{F}_0^{-1}(\alpha))}{\partial \hat{F}_0^{-1}(\alpha)}
|
||||
\cdot \dfrac{\partial \hat{F}_0^{-1}(\alpha)}{\partial f_\theta(\mathbf{x}_i)}
|
||||
\dfrac{\partial \hat{F}_1(\hat{F}_0^{-1}(1-\alpha))}{\partial \hat{F}_0^{-1}(1-\alpha)}
|
||||
\cdot \dfrac{\partial \hat{F}_0^{-1}(1-\alpha)}{\partial f_\theta(\mathbf{x}_i)}
|
||||
& \text{if } y_i = 0
|
||||
\end{cases}
|
||||
\label{eq:roll-gradient}
|
||||
@@ -280,26 +255,49 @@ Unifying both cases, the general gradient applicable to all ROLL instantiations
|
||||
The concrete form of each partial derivative depends on the choice of distribution family
|
||||
$\mathcal{F}$, and is derived for each instantiation in \Cref{sec:roll-instantiations}.
|
||||
|
||||
\subsection{ROLL variant - direct AUC optimisation}
|
||||
|
||||
\subsection{Properties of the ROLL Formulation}
|
||||
\label{sec:roll-properties}
|
||||
In addition to optimising for a specific rate, ROLL provides us an opportunity to
|
||||
optimize the AUC of the ROC directly.
|
||||
|
||||
% State and prove (or sketch) key properties: consistency of the estimator as
|
||||
% the number of samples grows, monotonicity of the loss with respect to the
|
||||
% TPR/FPR trade-off, and behaviour at the constraint boundary. Discuss what
|
||||
% guarantees ROLL provides and under what assumptions they hold.
|
||||
% NOTE - these properties are auto-generated, consider suggestions.
|
||||
The AUC is defined as the area under the curve of the ROC. The ROC is a function that
|
||||
for every FPR gives us the performing TPR of the model we have fitted.
|
||||
|
||||
Formally, for the function $TPR(\alpha)$ as defined above, we may define AUC as:
|
||||
|
||||
\subsection{Gradient Computation and the Custom Backward Pass}
|
||||
\label{sec:roll-backward}
|
||||
\begin{equation}
|
||||
\operatorname{AUC} = \int_{0}^{1}\operatorname{TPR}(\alpha) d \alpha
|
||||
\end{equation}
|
||||
|
||||
% Because the distribution parameters are themselves functions of model scores
|
||||
% (not fixed), the backward pass must differentiate through the distribution
|
||||
% fitting step. Describe the custom autograd implementation: how gradients flow
|
||||
% from the ROLL loss back through the distribution parameters to the model
|
||||
% weights. Highlight where standard autograd would fail and what the manual
|
||||
% backward pass does differently.
|
||||
The AUC is something we would like to maximize. In order to do so, we would like to
|
||||
minimize the area over the curve, which is $1-AUC$.
|
||||
|
||||
We note that $1-\operatorname{TPR}(\alpha) = \mathcal{L}(TPR@FPR)$.
|
||||
|
||||
From here, we would like to minimize:
|
||||
|
||||
\begin{equation}
|
||||
\mathcal{L}_{AOC} = \int_0^1\mathcal{L}_{\text{ROLL-TPR@FPR}}(\alpha) d \alpha
|
||||
\end{equation}
|
||||
|
||||
A direct integration is not part of this work. However, integration can be approximated. Instead
|
||||
of integrating directly, it is possible to evaluate the loss for every FPR between 0 and 1, sum them, and re-normalize.
|
||||
|
||||
Therefor, subdividing the interval $(0, 1)$ to 99 steps (every alpha starting from $0.01$ and ending with $0.99$, since a value of $0$ or $1$ would be illegal), our loss for minimizing AOC becomes:
|
||||
|
||||
\begin{equation}
|
||||
\mathcal{L}_{ROLL-AOC} = \frac{1}{99}\sum_{i=1}^{99}\mathcal{L}_{ROLL-TPR@FPR}\left(\frac{i}{100}\right)
|
||||
\label{eq:roll-aoc}
|
||||
\end{equation}
|
||||
|
||||
The gradient per score then simply becomes the average gradient over every threshold.
|
||||
|
||||
\begin{equation}
|
||||
\frac{\partial \mathcal{L}_{ROLL-AOC}(f_\theta(\mathbf{X}))}
|
||||
{\partial f_\theta(\mathbf{x}_i)} = \frac{1}{99}\sum_{i=1}^{99}\left(\frac{\partial \mathcal{L}_{\text{ROLL-TPR@FPR}}\left(f_\theta(\mathbf{X}) ; \frac{i}{100}\right)}
|
||||
{\partial f_\theta(\mathbf{x}_i)} \right)
|
||||
\label{eq:roll-aoc-gradient}
|
||||
\end{equation}
|
||||
|
||||
|
||||
%------------------------------------------------
|
||||
@@ -341,15 +339,15 @@ Firstly, we must estimate the parameters to fit the gaussian distributions
|
||||
to the true and false populations of the score output.
|
||||
|
||||
\begin{equation}
|
||||
\mu_0 = \frac{\sum_{\mathbf{x}_0 \in \mathcal{B}_0}{f_{\theta}(\mathbf{x}_0)}}{|\mathcal{B}_0|}
|
||||
\mu_0 = \frac{1}{|\mathcal{B}_0|}\sum_{\mathbf{x}_j \in \mathcal{B}_0}{f_{\theta}(\mathbf{x}_j)}
|
||||
\qquad
|
||||
\mu_1 = \frac{\sum_{\mathbf{x}_0 \in \mathcal{B}_1}{f_{\theta}(\mathbf{x}_1)}}{|\mathcal{B}_1|}
|
||||
\mu_1 = \frac{1}{|\mathcal{B}_1|}\sum_{\mathbf{x}_j \in \mathcal{B}_1}{f_{\theta}(\mathbf{x}_j)}
|
||||
\end{equation}
|
||||
|
||||
\begin{equation}
|
||||
\sigma_0^2 = \frac{1}{|\mathcal{B}_0|}\sum_{\mathbf{x}_0 \in \mathcal{B}_0}(\mathbf{x}_0 - \mu_0)^2
|
||||
\sigma_0^2 = \frac{1}{|\mathcal{B}_0|}\sum_{\mathbf{x}_j \in \mathcal{B}_0}(f_\theta(\mathbf{x}_j) - \mu_0)^2
|
||||
\qquad
|
||||
\sigma_1^2 = \frac{1}{|\mathcal{B}_1|}\sum_{\mathbf{x}_1 \in \mathcal{B}_1}(\mathbf{x}_1 - \mu_1)^2
|
||||
\sigma_1^2 = \frac{1}{|\mathcal{B}_1|}\sum_{\mathbf{x}_j \in \mathcal{B}_1}(f_\theta(\mathbf{x}_j) - \mu_1)^2
|
||||
\end{equation}
|
||||
|
||||
Once these parameters are estimated, we can then define our ROLL objective as required.
|
||||
@@ -362,47 +360,103 @@ and its inverse (quantile function) is:
|
||||
\[
|
||||
F^{-1}(p) = \mu + \sigma\sqrt{2}\,\text{ierf}(2p - 1)
|
||||
\]
|
||||
where TODO EXPLAIN ERF AND IERF. Plugging into the general ROLL framework, the threshold
|
||||
where $\operatorname{erf}(x) = \frac{2}{\sqrt{\pi}}\int_0^x e^{-t^2}\,\mathrm{d}t$ is the Gauss
|
||||
error function and $\operatorname{ierf}$ denotes its functional inverse, satisfying
|
||||
$\operatorname{ierf}(\operatorname{erf}(x)) = x$. Neither has a closed form, but both are
|
||||
available in standard numerical libraries (e.g.\ \texttt{torch.erf} and
|
||||
\texttt{torch.erfinv}). Plugging into the general ROLL framework, the threshold
|
||||
$\tau$ achieving FPR $= \alpha$ and the resulting loss are:
|
||||
\begin{align}
|
||||
\tau &= \mu_0 + \sigma_0\sqrt{2}\,\text{ierf}(2\alpha - 1) \nonumber \\
|
||||
\mathcal{L}_{\text{ROLL-TPR@FPR}}^{\text{GAUSSIAN}}(f_{\theta}(\mathbf{X}) ; \alpha)
|
||||
\tau &= \mu_0 + \sigma_0\sqrt{2}\,\text{ierf}(1 - 2\alpha) \nonumber \\
|
||||
\mathcal{L}_{\text{ROLL-TPR@FPR}}^{\text{GAUSSIAN}}(f_{\theta}(\mathcal{B}) ; \alpha)
|
||||
&= \frac{1}{2}\left[1 + \text{erf}\!\left(\frac{\tau - \mu_1}{\sigma_1\sqrt{2}}\right)\right]
|
||||
\label{eq:roll-tpr-at-fpr-gaussian}
|
||||
\end{align}
|
||||
|
||||
|
||||
|
||||
|
||||
\subsubsection{Gradient Derivation}
|
||||
\label{sec:roll-gaussian-backward}
|
||||
|
||||
% Differentiate the Gaussian loss with respect to $f_\theta(\mathbf{x}_i)$.
|
||||
% Gradients flow through $\mu_k$ and $\sigma_k^2$ (which are differentiable
|
||||
% functions of the scores), and through $\Phi$ and $\Phi^{-1}$, which have
|
||||
% simple closed-form derivatives.
|
||||
The full derivation is given in Appendix~\ref{appendix:gauss-grad}.
|
||||
Letting $\tau = \hat{F}_0^{-1}(1-\alpha) = \mu_0 + \sigma_0\sqrt{2}\,\operatorname{ierf}(1-2\alpha)$, the gradient is:
|
||||
|
||||
\begin{equation}
|
||||
\frac{\partial \mathcal{L}_{\text{ROLL-TPR@FPR}}^{\text{GAUSSIAN}}}{\partial f_\theta(\mathbf{x}_i)} =
|
||||
\begin{cases}
|
||||
\dfrac{-1}{|\mathcal{B}_1|\,\sigma_1\sqrt{2\pi}}
|
||||
\exp\!\left(-\dfrac{(\tau-\mu_1)^2}{2\sigma_1^2}\right)
|
||||
\!\left(1 + \dfrac{(\tau-\mu_1)(f_\theta(\mathbf{x}_i)-\mu_1)}{\sigma_1^2}\right)
|
||||
& \text{if } y_i = 1 \\[18pt]
|
||||
\dfrac{1}{|\mathcal{B}_0|\,\sigma_1\sqrt{2\pi}}
|
||||
\exp\!\left(-\dfrac{(\tau-\mu_1)^2}{2\sigma_1^2}\right)
|
||||
\!\left(1 + \dfrac{(\tau-\mu_0)(f_\theta(\mathbf{x}_i)-\mu_0)}{\sigma_0^2}\right)
|
||||
& \text{if } y_i = 0
|
||||
\end{cases}
|
||||
\label{eq:gauss-grad-combined}
|
||||
\end{equation}
|
||||
|
||||
\subsection{Beta ROLL}
|
||||
\label{sec:roll-beta}
|
||||
|
||||
% When scores are passed through a sigmoid they live in $[0,1]$ and a Beta
|
||||
% distribution is a natural parametric fit. The integrals involve the
|
||||
% regularised incomplete beta function. Discuss when Beta ROLL is preferable
|
||||
% to Gaussian ROLL and what its failure modes are.
|
||||
Beta ROLL applies when model scores are constrained to the open interval $(0,1)$,
|
||||
as is the case when $f_\theta$ uses a sigmoid output activation. The Beta distribution
|
||||
is supported on $(0,1)$ and its two shape parameters allow it to capture a wide range
|
||||
of unimodal and skewed score distributions that a Gaussian cannot represent.
|
||||
|
||||
\subsubsection{Forward}
|
||||
\label{sec:roll-beta-forward}
|
||||
|
||||
% Instantiate \eqref{eq:roll-tpr-at-fpr} under Beta-distributed scores.
|
||||
% Parameters $a_k, b_k > 0$ are fitted by MLE (or method of moments).
|
||||
% The CDF is the regularised incomplete beta function $I_x(a, b)$.
|
||||
Given score vectors $\mathbf{s}^{(k)} \in (0,1)^{|\mathcal{B}_k|}$ for class $k$, let
|
||||
$\mathbf{t}^{(k)} = \mathbf{1} - \mathbf{s}^{(k)}$ denote the complementary scores.
|
||||
For any vector $\mathbf{v}$, write $\bar{v} = \frac{1}{|\mathbf{v}|}\sum_i v_i$.
|
||||
We estimate the shape parameters using the closed-form log-moment estimator
|
||||
of~\cite{chen2022beta}: define for each class $k$,
|
||||
|
||||
\begin{equation}
|
||||
D_k = \overline{s^{(k)}\!\ln s^{(k)}} - \bar{s}^{(k)}\cdot\overline{\ln s^{(k)}}
|
||||
+ \overline{t^{(k)}\!\ln t^{(k)}} - \bar{t}^{(k)}\cdot\overline{\ln t^{(k)}}
|
||||
\label{eq:beta-denom}
|
||||
\end{equation}
|
||||
|
||||
\begin{equation}
|
||||
\hat{a}_k = \frac{\bar{s}^{(k)}}{D_k}, \qquad \hat{b}_k = \frac{\bar{t}^{(k)}}{D_k}
|
||||
\label{eq:beta-params}
|
||||
\end{equation}
|
||||
|
||||
These estimators follow from two simultaneous moment conditions:
|
||||
$\mathbb{E}[X] = a/(a+b)$ and
|
||||
$\operatorname{Cov}(X,\ln X) + \operatorname{Cov}(Y,\ln Y) = 1/(a+b)$,
|
||||
where $Y=1-X\sim\operatorname{Beta}(b,a)$.
|
||||
Replacing expectations by sample means and solving in closed form yields~\eqref{eq:beta-params}.
|
||||
|
||||
The CDF of $\operatorname{Beta}(\hat{a}_k,\hat{b}_k)$ at $x\in(0,1)$ is the
|
||||
regularised incomplete beta function:
|
||||
\begin{equation}
|
||||
\hat{F}_k(x) = I_x(\hat{a}_k,\hat{b}_k)
|
||||
= \frac{B(x;\,\hat{a}_k,\hat{b}_k)}{B(\hat{a}_k,\hat{b}_k)},
|
||||
\qquad B(x;\,a,b) = \int_0^x u^{a-1}(1-u)^{b-1}\,\mathrm{d}u
|
||||
\label{eq:beta-cdf}
|
||||
\end{equation}
|
||||
|
||||
Substituting into the general ROLL framework \eqref{eq:roll-tpr-at-fpr}:
|
||||
\begin{equation}
|
||||
\mathcal{L}_{\text{ROLL-TPR@FPR}}^{\text{BETA}}(f_\theta(\mathcal{B});\,\alpha)
|
||||
= I_{\hat\tau}(\hat{a}_1,\hat{b}_1),
|
||||
\qquad
|
||||
\hat\tau = I^{-1}_{1-\alpha}(\hat{a}_0,\hat{b}_0)
|
||||
\label{eq:roll-tpr-at-fpr-beta}
|
||||
\end{equation}
|
||||
where $I^{-1}_{1-\alpha}(a,b)$ denotes the $(1-\alpha)$-quantile of $\operatorname{Beta}(a,b)$.
|
||||
|
||||
\subsubsection{Gradient Derivation}
|
||||
\label{sec:roll-beta-backward}
|
||||
|
||||
% Differentiate through $I_x$ and its inverse. Both have known derivatives;
|
||||
% note any numerical considerations when $a$ or $b$ are near zero.
|
||||
Unlike the Gaussian case, we do not derive closed-form gradients for Beta ROLL.
|
||||
The regularised incomplete beta function $I_x(a,b)$ and its inverse are smooth
|
||||
functions of all their arguments and are available as differentiable operations in
|
||||
PyTorch (\texttt{torch.special.betainc} and \texttt{torch.special.betaincinv}).
|
||||
Gradients of \eqref{eq:roll-tpr-at-fpr-beta} with respect to $f_\theta(\mathbf{x}_i)$
|
||||
are therefore propagated via automatic differentiation through the estimator
|
||||
\eqref{eq:beta-params}, the CDF evaluation, and the quantile inversion.
|
||||
|
||||
|
||||
\subsection{KDE ROLL}
|
||||
@@ -413,6 +467,10 @@ $\tau$ achieving FPR $= \alpha$ and the resulting loss are:
|
||||
% Discuss the trade-off between flexibility and computational cost, and how
|
||||
% the bandwidth (see \Cref{sec:kde-bandwidth}) controls bias-variance.
|
||||
|
||||
Unfortunately, the score output of trained models remains unpredictable. A solution for this is to use a Kernel Density Estimation (KDE) for probability
|
||||
estimation. This benefits from being far more adaptable to real-world distributions of outputs of our trained models. The tradeoff of course is computational
|
||||
complexity - computing the gradient derivation in such a case is non-trivial.
|
||||
|
||||
\subsubsection{Forward}
|
||||
\label{sec:roll-kde-forward}
|
||||
|
||||
@@ -420,6 +478,40 @@ $\tau$ achieving FPR $= \alpha$ and the resulting loss are:
|
||||
% kernel CDFs; the ICDF is computed numerically. Derive the loss expression
|
||||
% explicitly and discuss computational cost relative to the parametric cases.
|
||||
|
||||
As before, to calculate the loss, we calculate:
|
||||
|
||||
|
||||
\[
|
||||
\mathcal{L}_{\text{ROLL-TPR@FPR}}^{\text{KDE}}(f_{\theta}(\mathcal{B}) ; \alpha) = \hat{F}_1(\hat{F}_0^{-1}(1-\alpha))
|
||||
\]
|
||||
|
||||
To calculate this, we first define the kernel function as $\sigma '$, and the CDF of the kernel function as $\sigma$. The approach used here is to define $\sigma$ as a sigmoid function,.
|
||||
|
||||
\begin{equation}
|
||||
\sigma(x ; v) = \left( 1 + \exp(-vx)\right)^{-1} \qquad \sigma'(x ; v) = \frac{v \cdot \exp(-v \cdot x)}{(1 + \exp(-vx))^2}
|
||||
\end{equation}
|
||||
|
||||
Where $v$ is the bandwidth parameter. This affects the calculation. We calculate $v_0$ as the bandwidth for the false population and $v_1$ for the true population. Bandwidth calculation
|
||||
will be discussed later.
|
||||
|
||||
We then denote $\sigma_1(x) = \sigma(x ; v_1), \sigma_0(x) = \sigma(x ; v_0)$ to differentiate between the kernel functions for the true and false population, which only differentiate by bandwidth.
|
||||
|
||||
The CDF of the KDE function then becomes:
|
||||
|
||||
\[
|
||||
\hat{F}_{KDE}(\tau ; \mathbf{X}) = \frac{1}{|X|}\sum_i \sigma(\tau - x_i)
|
||||
\]
|
||||
|
||||
While the CDF has a closed form, its inverse has none and must be found numerically.
|
||||
We use bisection, which requires no derivative evaluation and is guaranteed to converge
|
||||
for any strictly monotone CDF; the bracket construction and iteration are detailed in
|
||||
Appendix~\ref{appendix:kde-bisection}.
|
||||
|
||||
We can now relatively easily calculate $\hat{F}_1(\tau ; \mathbf{X}_1)$ using the formulat for $\hat{F}_{KDE}(\tau ; \mathbf{X})$ above.
|
||||
|
||||
Importantly, we must save the calculation of $\tau$ for the backward derivation, as well as the calculated bandwidths $v_1, v_0$.
|
||||
|
||||
|
||||
\subsubsection{Gradient Derivation}
|
||||
\label{sec:roll-kde-backward}
|
||||
|
||||
@@ -428,22 +520,203 @@ $\tau$ achieving FPR $= \alpha$ and the resulting loss are:
|
||||
% Gradients flow through the kernel evaluations back to the model scores.
|
||||
% This is the section that connects to \Cref{sec:roll-backward} motivation.
|
||||
|
||||
The derivation is given in Appendix~\ref{appendix:kde-grad}; letting $\tau = \hat{F}_0^{-1}(1-\alpha ; \mathcal{B}_0)$, the gradient is:
|
||||
|
||||
\begin{equation}
|
||||
\frac{\partial \mathcal{L}_{\text{ROLL-TPR@FPR}}}{\partial f_\theta(\mathbf{x}_i)} =
|
||||
\begin{cases}
|
||||
-\dfrac{1}{|\mathcal{B}_1|}\,\sigma_1'(\tau - f_{\theta}(\mathbf{x}_i))
|
||||
& \text{if } y_i = 1 \\[14pt]
|
||||
+\dfrac{1}{|\mathcal{B}_1|}\!\displaystyle\sum_{\mathbf{x}_j \in \mathcal{B}_1}\!\sigma_1'(\tau - f_{\theta}(\mathbf{x}_j))
|
||||
\;\cdot\;
|
||||
\dfrac{\sigma_0'(\tau - f_{\theta}(\mathbf{x}_i))}{\displaystyle\sum_{\mathbf{x}_j \in \mathcal{B}_0}\sigma_0'(\tau - f_{\theta}(\mathbf{x}_j))}
|
||||
& \text{if } y_i = 0
|
||||
\end{cases}
|
||||
\label{eq:kde-grad-combined}
|
||||
\end{equation}
|
||||
|
||||
where $\tau = \hat{F}_0^{-1}(1-\alpha ; \mathcal{B}_0)$.
|
||||
|
||||
The explicit sigmoid-kernel form of \eqref{eq:kde-grad-combined} — forward pass CDF, efficient gradient, and implementation notes — is given in Appendix~\ref{appendix:kde-sigmoid}.
|
||||
Positive-class samples are weighted by their individual kernel
|
||||
PDF at $\tau$; negative-class samples are weighted by the mean positive-class kernel
|
||||
PDF scaled by each sample's share of the negative-class kernel mass.
|
||||
|
||||
%------------------------------------------------
|
||||
|
||||
\subsection{Properties of the ROLL Formulation}
|
||||
\label{sec:roll-properties}
|
||||
|
||||
% State and prove (or sketch) key properties: consistency of the estimator as
|
||||
% the number of samples grows, monotonicity of the loss with respect to the
|
||||
% TPR/FPR trade-off, and behaviour at the constraint boundary. Discuss what
|
||||
% guarantees ROLL provides and under what assumptions they hold.
|
||||
% NOTE - these properties are auto-generated, consider suggestions.
|
||||
|
||||
A structural consequence of the ROLL formulation is that gradient mass is automatically
|
||||
balanced between the two classes, regardless of their relative sizes in the batch.
|
||||
This stands in contrast to standard losses such as cross-entropy, where the total gradient
|
||||
contribution from each class is proportional to its count, making heavy class imbalance
|
||||
a direct source of biased updates. Therefor, instead of needing to compensate for this
|
||||
with loss-weighting per-class, fitting updates are made based equally off the results
|
||||
of both classes.
|
||||
|
||||
\begin{proposition}[Gradient balance]
|
||||
\label{prop:gradient-balance}
|
||||
Let $\mathcal{F}$ be any distribution family whose CDF estimator is translation-invariant,
|
||||
meaning $\hat{F}_k(t\,;\,\mathbf{s}+\delta) = \hat{F}_k(t-\delta\,;\,\mathbf{s})$
|
||||
for all $\delta \in \mathbb{R}$. Then for any batch $\mathcal{B} = \mathcal{B}_0 \cup \mathcal{B}_1$:
|
||||
\begin{equation}
|
||||
\sum_{\mathbf{x}_i \in \mathcal{B}_1}
|
||||
\frac{\partial \mathcal{L}_{\emph{\text{ROLL-TPR@FPR}}}}{\partial f_\theta(\mathbf{x}_i)}
|
||||
\;=\;
|
||||
-\!\sum_{\mathbf{x}_i \in \mathcal{B}_0}
|
||||
\frac{\partial \mathcal{L}_{\emph{\text{ROLL-TPR@FPR}}}}{\partial f_\theta(\mathbf{x}_i)}
|
||||
\label{eq:grad-balance}
|
||||
\end{equation}
|
||||
with both sides equal in magnitude to $\dfrac{\partial \hat{F}_1(\tau)}{\partial \tau}$,
|
||||
independently of $|\mathcal{B}_0|$ and $|\mathcal{B}_1|$.
|
||||
\end{proposition}
|
||||
|
||||
\begin{proof}
|
||||
We treat each class separately via a uniform-shift argument.
|
||||
|
||||
\medskip\noindent\textit{Sum over $\mathcal{B}_0$.}
|
||||
From \Cref{eq:roll-gradient}, for $y_i = 0$:
|
||||
\[
|
||||
\frac{\partial \mathcal{L}}{\partial f_\theta(\mathbf{x}_i)}
|
||||
= \frac{\partial \hat{F}_1(\tau)}{\partial \tau}
|
||||
\cdot \frac{\partial \tau}{\partial f_\theta(\mathbf{x}_i)}
|
||||
\]
|
||||
We claim $\sum_{\mathbf{x}_i \in \mathcal{B}_0}\frac{\partial \tau}{\partial f_\theta(\mathbf{x}_i)} = 1$.
|
||||
Shift all negative scores uniformly: $f_\theta(\mathbf{x}_i) \to f_\theta(\mathbf{x}_i) + \delta$ for
|
||||
$\mathbf{x}_i \in \mathcal{B}_0$. By translation invariance, $\hat{F}_0$ simply translates, so
|
||||
$\tau = \hat{F}_0^{-1}(1-\alpha)$ shifts by exactly $\delta$. Differentiating with respect to $\delta$ at
|
||||
$\delta = 0$:
|
||||
\[
|
||||
1 = \frac{\mathrm{d}\tau}{\mathrm{d}\delta}\bigg|_{\delta=0}
|
||||
= \sum_{\mathbf{x}_i \in \mathcal{B}_0} \frac{\partial \tau}{\partial f_\theta(\mathbf{x}_i)}
|
||||
\]
|
||||
Summing the $y_i = 0$ gradients therefore gives:
|
||||
\begin{equation}
|
||||
\sum_{\mathbf{x}_i \in \mathcal{B}_0}
|
||||
\frac{\partial \mathcal{L}}{\partial f_\theta(\mathbf{x}_i)}
|
||||
= \frac{\partial \hat{F}_1(\tau)}{\partial \tau} \cdot 1
|
||||
= \frac{\partial \hat{F}_1(\tau)}{\partial \tau}
|
||||
\label{eq:grad-balance-b0}
|
||||
\end{equation}
|
||||
|
||||
\medskip\noindent\textit{Sum over $\mathcal{B}_1$.}
|
||||
For $y_i = 1$, $\frac{\partial \mathcal{L}}{\partial f_\theta(\mathbf{x}_i)} = \frac{\partial \hat{F}_1(\tau)}{\partial f_\theta(\mathbf{x}_i)}$.
|
||||
Shifting all positive scores by $\delta$ and applying translation invariance of $\hat{F}_1$:
|
||||
$\hat{F}_1(\tau\,;\,\mathbf{s}+\delta\mathbf{1}) = \hat{F}_1(\tau-\delta\,;\,\mathbf{s})$.
|
||||
Differentiating at $\delta = 0$:
|
||||
\[
|
||||
\sum_{\mathbf{x}_i \in \mathcal{B}_1}
|
||||
\frac{\partial \hat{F}_1(\tau)}{\partial f_\theta(\mathbf{x}_i)}
|
||||
= -\frac{\partial \hat{F}_1(\tau)}{\partial \tau}
|
||||
\]
|
||||
Combining with \eqref{eq:grad-balance-b0} gives \eqref{eq:grad-balance}.
|
||||
\end{proof}
|
||||
|
||||
The translation-invariance assumption holds for both the Gaussian distribution,
|
||||
as well as the KDE-based distribution, but not for the Beta distribution.
|
||||
For Gaussian MLE, a uniform shift in scores moves $\mu_k$ by the same amount while
|
||||
leaving $\sigma_k$ unchanged, which translates $\hat{F}_k$; for KDE, each kernel term is
|
||||
evaluated at $\tau - f_\theta(\mathbf{x}_j)$, so a uniform shift in the $f_\theta(\mathbf{x}_j)$
|
||||
is equivalent to an equal shift in $\tau$. The condition is necessary: without it, a uniform
|
||||
score shift need not produce a rigid translation of $\hat{F}_k$, so the shift argument in the
|
||||
proof breaks down and the gradient sums need not be equal.
|
||||
|
||||
The Beta distribution only accepts values in the interval $\[0, 1\]$. Since it
|
||||
is mapped only over an interval, the translation-invariance assumption does not hold
|
||||
for this distribution.
|
||||
|
||||
\paragraph{Gradient locality.}
|
||||
\label{para:gradient-locality}
|
||||
|
||||
A second structural consequence of the ROLL-KDE formulation is that gradient mass is
|
||||
\emph{concentrated near the operating threshold $\tau$}. Samples whose scores lie
|
||||
close to $\tau$ receive larger gradient updates, while samples
|
||||
already far on the correct side contribute very little.
|
||||
|
||||
This follows directly from \Cref{eq:kde-grad-combined}. For the positive class ($y_i = 1$),
|
||||
the gradient magnitude is
|
||||
\[
|
||||
\left|\frac{\partial \mathcal{L}}{\partial f_\theta(\mathbf{x}_i)}\right|
|
||||
= \frac{1}{|\mathcal{B}_1|}\,\sigma_1'(f_\theta(\mathbf{x}_i) - \tau),
|
||||
\]
|
||||
which is a bell-shaped kernel PDF that peaks when $f_\theta(\mathbf{x}_i) = \tau$ and
|
||||
decays to zero as the score moves away from the threshold in either direction.
|
||||
For the negative class ($y_i = 0$), the gradient is proportional to
|
||||
$\sigma_0'(\tau - f_\theta(\mathbf{x}_i))$ (normalized over $\mathcal{B}_0$), which is
|
||||
again bell-shaped and centred on $\tau$.
|
||||
|
||||
The effect is that each gradient step focuses the model's attention on the region that
|
||||
directly determines the operating-point performance: samples that could plausibly be
|
||||
re-ranked relative to the threshold at the current step. Samples classified with strong
|
||||
confidence relative to the threshold, with scores far enough away from said threshold,
|
||||
receive small gradient updates, since a small change to the score of these samples will not
|
||||
bring any change to the placement of the threshold itself, or the performance at said threshold.
|
||||
|
||||
This stands in contrast to standard losses such as cross-entropy, where the per-sample
|
||||
gradient is determined by the predicted probability, with no direct dependence on an
|
||||
operating threshold.
|
||||
|
||||
\begin{observation}[Gradient locality]
|
||||
\label{obs:gradient-locality}
|
||||
For KDE-ROLL, the gradient magnitude of sample $\mathbf{x}_i$ is proportional to
|
||||
the kernel PDF evaluated at the distance between $f_\theta(\mathbf{x}_i)$ and the
|
||||
operating threshold $\tau$. Samples nearest to $\tau$ receive the largest updates;
|
||||
samples far from $\tau$ in either direction contribute negligible gradient.
|
||||
\end{observation}
|
||||
|
||||
\Cref{fig:threshold-locality} illustrates this for a simulated two-class setting. The
|
||||
upper panel shows the score distributions and the operating threshold; the lower panel
|
||||
shows the gradient magnitude of each sample. Both classes exhibit a clear bell-shaped
|
||||
concentration of gradient mass around $\tau$, with smooth decay on either side.
|
||||
|
||||
\begin{figure}[H]
|
||||
\centering
|
||||
\includegraphics[width=0.82\textwidth]{content/method/figures/threshold_weighting.pdf}
|
||||
\caption{Gradient locality in KDE-ROLL. \textit{Upper}: score distributions for the
|
||||
negative (blue, $y=0$) and positive (orange, $y=1$) classes; dashed line marks the
|
||||
operating threshold $\tau$ at FPR $= \alpha$. \textit{Lower}: gradient magnitude
|
||||
$|\partial\mathcal{L}/\partial f_\theta(\mathbf{x})|$ for each sample. Both classes
|
||||
show a bell-shaped concentration of gradient mass near $\tau$, with samples far from
|
||||
the threshold contributing negligible updates. Smooth curves are the kernel PDFs scaled
|
||||
for reference.}
|
||||
\label{fig:threshold-locality}
|
||||
\end{figure}
|
||||
|
||||
%------------------------------------------------
|
||||
|
||||
\section{Implementation Considerations}
|
||||
\label{sec:roll-implementation}
|
||||
|
||||
\subsection{Numerical Stability and the Scaling Trick}
|
||||
\subsection{Combatting Numerical Instability With Score Scaling}
|
||||
\label{sec:roll-numerical-stability}
|
||||
|
||||
% The KDE loss involves sums of kernel evaluations that can span many orders of
|
||||
% magnitude, leading to floating-point underflow or overflow during training.
|
||||
% Describe the scaling trick that normalizes these sums to a numerically stable
|
||||
% range without changing the gradient direction, and show that it is equivalent
|
||||
% to computing in log-space with a max-subtraction stabilizer (analogous to the
|
||||
% log-sum-exp trick).
|
||||
Before computing the KDE, all batch scores are divided by their overall standard
|
||||
deviation:
|
||||
\[
|
||||
\tilde{s}_i = \frac{f_\theta(\mathbf{x}_i)}{\hat\sigma_{\mathcal{B}}},
|
||||
\qquad
|
||||
\hat\sigma_{\mathcal{B}} = \sqrt{\operatorname{Var}(f_\theta(\mathbf{x}_i) : (\mathbf{x}_i, y_i) \in \mathcal{B})}.
|
||||
\]
|
||||
This keeps the values given to the sigmoid kernel ($v(\tau - \tilde{s}_j)$)
|
||||
near unit scale, preventing floating-point overflow and underflow in the exponential
|
||||
evaluations.
|
||||
|
||||
The scaling has no effect on the loss value itself or derived gradients.
|
||||
Silverman's rule derives the bandwidth proportionally to the data standard deviation,
|
||||
$h \propto \hat\sigma \cdot n^{-1/5}$, so dividing all scores by $\hat\sigma_\mathcal{B}$
|
||||
causes the bandwidth to scale by the same factor.
|
||||
The product $v \cdot (\tau - s_j)$ — the only quantity entering the sigmoid — is
|
||||
therefore invariant under the rescaling.
|
||||
In the backward pass, gradients with respect to the normalised scores are multiplied
|
||||
by $1/\hat\sigma_\mathcal{B}$ to apply the chain rule, providing gradients mathematically
|
||||
identical to those that would have been obtained without normalisation.
|
||||
|
||||
\subsection{Bandwidth Estimation for KDE}
|
||||
\label{sec:kde-bandwidth}
|
||||
@@ -456,12 +729,33 @@ $\tau$ achieving FPR $= \alpha$ and the resulting loss are:
|
||||
% NOTE - not silverman's rule, we do use scheduling, but we also use different
|
||||
% estimator - see impl.
|
||||
|
||||
When computing KDE, bandwidth estimation strongly affects the result. Large bandwidths
|
||||
produce an overly smooth CDF that fails to track the true score distribution; small
|
||||
bandwidths produce a jagged CDF, yielding negligible gradients for all but the
|
||||
nearest-threshold samples.
|
||||
|
||||
We estimate the bandwidth using the Improved Sheather-Jones (ISJ)
|
||||
method~\cite{botev2010kde}, which minimises the asymptotic mean integrated squared
|
||||
error via an iterative plug-in procedure. ISJ tracks the true score distribution more
|
||||
closely than closed-form rules, as illustrated in \Cref{fig:density-fit-comparison}.
|
||||
Because ISJ is iterative, it can fail to converge on small or degenerate batches early
|
||||
in training. In those cases we fall back to Silverman's rule of
|
||||
thumb~\cite{silverman1986density},
|
||||
\[
|
||||
h = 1.06\,\hat{\sigma}\,n^{-1/5},
|
||||
\]
|
||||
which is closed-form and always produces a valid bandwidth.
|
||||
The kernel parameter is then set to $v = 1/h$.
|
||||
|
||||
\subsubsection{Bandwidth Scaling for KDE}
|
||||
\label{sec:kde-bandwidth-scaling}
|
||||
|
||||
Although we may select a valid bandwidth at each step, it tends to be
|
||||
too small near the start of training, when scores are not yet
|
||||
spread out and $\hat{\sigma}$ is small. This causes the KDE to undersmooth the score
|
||||
distribution, producing very low gradient magnitudes and stalling convergence.
|
||||
|
||||
The solution was to use a bandwidth scheduler. Starting with large bandwidths resulted in gradients that spread out more evenly across all scores, and fewer non-convergence runs occurred. The bandwidth scheduler artificially scales up the bandwidth at the beginning to speed up initial training, and then tapers back to restore the intended bandwidth in order to maximise results.
|
||||
|
||||
|
||||
% TODO: algorithm box — full training loop (forward → ROLL loss → custom backward → weight update)
|
||||
|
||||
|
||||
%%% Local Variables:
|
||||
%%% mode: latex
|
||||
%%% TeX-master: "../../main"
|
||||
%%% TeX-engine: xetex
|
||||
%%% End:
|
||||
|
||||
@@ -1,119 +1,491 @@
|
||||
\section{topic a}
|
||||
\label{sec:related_work:jigsaw_puzzles}
|
||||
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{נושא א}}
|
||||
% ============================================================
|
||||
% PLACEMENT: near the START of Ch. 2, before KDE and AUC/lit
|
||||
% review sections. This is the reader's first encounter with
|
||||
% the problem setting and the TPR@FPR objective. The arc
|
||||
% should be: (1) standard accuracy under imbalance fails;
|
||||
% (2) class-weighting is the usual fix but gives no direct
|
||||
% operating-point control; (3) define TPR, FPR, ROC formally;
|
||||
% (4) state the target: maximise TPR at a fixed FPR budget α.
|
||||
% ============================================================
|
||||
\section{Imbalanced Classification and The TPR/FPR Objective}
|
||||
\label{sec:imbalanced-tpr-fpr}
|
||||
|
||||
To be continued.
|
||||
% IMPROVEMENT SUGGESTIONS (carried over from original draft):
|
||||
% - Narrative arc is currently backwards. Drive the section
|
||||
% with: accuracy is gameable under imbalance -> class
|
||||
% weighting is the standard fix -> it gives no direct
|
||||
% control over the operating point -> therefore we need
|
||||
% TPR@FPR directly.
|
||||
% - The inseparability tangent (cascade-classifier example)
|
||||
% dilutes the argument; trim to one sentence or cut.
|
||||
% - $\mathcal{D}_0$ and $\mathcal{D}_1$ must be defined
|
||||
% before first use.
|
||||
% - Land the section on a concrete, formal statement of the
|
||||
% TPR@FPR objective with proper notation.
|
||||
|
||||
When fitting a classifier, it is most common to try and maximize the probability
|
||||
of success. That is, given some dataset
|
||||
$\mathcal{D} = \{(\mathbf{x}_i, y_i)\}_{i=1}^{n}$
|
||||
with inputs $\mathbf{x}_i \in \mathcal{X}$ and binary labels $y_i \in \{0, 1\}$, we learn a scoring function $f_\theta : \mathcal{X} \to \mathbb{R}$ and apply a threshold $\tau$ to produce predictions $\hat{y} = \mathbf{1}[f_\theta(\mathbf{x}) > \tau]$. The standard objective is to minimize the probability of classification error:
|
||||
|
||||
$$\mathcal{L}(\theta) = \frac{1}{n} \sum_{i=1}^{n} \mathbf{1}[\hat{y}_i \neq y_i] = P(\hat{y} \neq y)$$
|
||||
|
||||
which is equivalent to maximizing classification accuracy $P(\hat{y} = y)$.
|
||||
|
||||
For most applications, this optimization is warranted. However, there are many real-world settings in which
|
||||
this type of fitting is ill-suited.
|
||||
|
||||
One such application is in the case of inseparable classes. In many cases, no meaningful
|
||||
separation boundary can be found between $\mathcal{D}_0$ and $\mathcal{D}_1$. In many cases,
|
||||
especially in tabular datasets, there can be two data elements $i,j$ for which $\mathbf{x}_i = \mathbf{x}_j$,
|
||||
however, $y_i \neq y_j$. Any deterministic classifier will have to decide whether to classify
|
||||
inseparable points as one class or the other, but it remains hard to control the behavior of
|
||||
a classifier while it is fitting to suggest one outcome or the other.
|
||||
|
||||
The issue of inseparability becomes more apparent in certain applications where runtime is a key
|
||||
and limiting factor. In many classification pipelines, a ``quick and dirty'' classifier may
|
||||
filter out many candidates before an ``in depth'' classifier gives the final answer. This may be done
|
||||
in events where running the full in-depth classification on all data points is not a cost-effective or time-efficient solution.
|
||||
The ``quick and dirty'' classifier can be either low in parameters, making it run quickly. It may also only
|
||||
make a quick decision based on part of the data point. For example, in the case of a classifier looking to flag
|
||||
images of the ocean, a ``quick and dirty'' classifier may look at a very-low resolution version of the image and simply check if it is relatively ``blue''. It filters out many certain false candidates this way, then passes on the remainder to an in-depth, higher accuracy classifier. Solutions like this allow to run at near-optimal performance with lower compute resources in limited-time applications. These situations are common in large-volume image applications on the internet, as well as in astronomy/microscopy.
|
||||
|
||||
Another such application is a case of highly imbalanced data. Even with separability, it is very
|
||||
easy to a classifier to simply ``ignore'' the minority class. This may very likely be a local minimum
|
||||
while fitting, which may be inescapable. As was once said, ``a classifier that always returns 0 is
|
||||
well fitted for flagging pictures of Michael Jordan bathing in green M\&Ms''. Combined with inseparability, we see that in extreme cases, classifiers may choose to ignore, or otherwise penalize, the minority class.
|
||||
|
||||
For every trained classifier, the output per model is a score (TODO rewrite and formalize).
|
||||
|
||||
TODO finish this section
|
||||
|
||||
\begin{itemize}
|
||||
\item What usually happens
|
||||
\begin{itemize}
|
||||
\item Fitting
|
||||
\item Graphing ROC and picking threshold based on tradeoff
|
||||
\end{itemize}
|
||||
\item Usually we compensate for poor performance with
|
||||
\begin{itemize}
|
||||
\item Weighting between classes
|
||||
\item Possible other methods
|
||||
\end{itemize}
|
||||
\item Problem with this method is lack of control.
|
||||
\end{itemize}
|
||||
|
||||
|
||||
\section{topic b}
|
||||
\label{sec:related_work:relaxation_labeling}
|
||||
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{נושא ב}}
|
||||
|
||||
To be continued.
|
||||
|
||||
\subsection{sub topic b.1}
|
||||
\label{subsec:formulation_as_rl:rationale:type_2}
|
||||
\addcontentsline{tocheb}{subsection}{\protect\numberline{\subsecnumforhebrewtoc}{תת נושא ב1}}
|
||||
|
||||
To be continued.
|
||||
|
||||
\begin{figure}[H]
|
||||
\centering
|
||||
|
||||
\begin{subfigure}[b]{0.3\textwidth}
|
||||
\begin{tikzpicture}
|
||||
\node[anchor=south west, inner sep=0] at (0,0) {\includegraphics[width=\textwidth]{content/related_work/images/2x2_puzzle_grid.png}};
|
||||
\draw[step=0.5\textwidth] (0,0) grid (\textwidth,\textwidth);
|
||||
\node[font=\large] at (0.25\textwidth,0.75\textwidth) {(1,1)};
|
||||
\node[font=\large] at (0.75\textwidth,0.75\textwidth) {(1,2)};
|
||||
\node[font=\large] at (0.25\textwidth,0.25\textwidth) {(2,1)};
|
||||
\node[font=\large] at (0.75\textwidth,0.25\textwidth) {(2,2)};
|
||||
\end{tikzpicture}
|
||||
\caption{}
|
||||
\label{fig:type_1_goal_and_labeling:dimensions}
|
||||
\end{subfigure}
|
||||
\hfill
|
||||
\begin{subfigure}[b]{0.3\textwidth}
|
||||
\begin{tikzpicture}
|
||||
\node[anchor=south west, inner sep=0] at (0,0) {\includegraphics[width=\textwidth]{content/related_work/images/2x2puzzle_type_1.png}};
|
||||
\draw[step=0.5\textwidth] (0,0) grid (\textwidth,\textwidth);
|
||||
\node[font=\large, color=red] at (0.25\textwidth,0.75\textwidth) {Piece 1};
|
||||
\node[font=\large, color=red] at (0.75\textwidth,0.75\textwidth) {Piece 2};
|
||||
\node[font=\large, color=red] at (0.25\textwidth,0.25\textwidth) {Piece 3};
|
||||
\node[font=\large, color=red] at (0.75\textwidth,0.25\textwidth) {Piece 4};
|
||||
\end{tikzpicture}
|
||||
\caption{}
|
||||
\label{fig:type_1_goal_and_labeling:pieces}
|
||||
\end{subfigure}
|
||||
\hfill
|
||||
\begin{subfigure}[b]{0.3\textwidth}
|
||||
\begin{tikzpicture}
|
||||
\node[anchor=south west, inner sep=0] at (0,0) {\includegraphics[width=\textwidth]{content/related_work/images/2x2puzzle_solution.png}};
|
||||
\draw[step=0.5\textwidth] (0,0) grid (\textwidth,\textwidth);
|
||||
|
||||
\node[font=\large, color=red] at (0.25\textwidth,0.75\textwidth) {Piece 3};
|
||||
|
||||
\node[font=\large, color=red] at (0.75\textwidth,0.75\textwidth) {Piece 2};
|
||||
|
||||
\node[font=\large, color=red] at (0.25\textwidth,0.25\textwidth) {Piece 4};
|
||||
|
||||
\node[font=\large, color=red] at (0.75\textwidth,0.25\textwidth) {Piece 1};
|
||||
\end{tikzpicture}
|
||||
\caption{}
|
||||
\label{fig:type_1_goal_and_labeling:solution}
|
||||
\end{subfigure}
|
||||
\vfill
|
||||
\begin{subfigure}[b]{1\textwidth}
|
||||
\centering
|
||||
|
||||
\begin{tikzpicture}
|
||||
\def\scaletitles{0.88}
|
||||
\def\minimumEntrySize{0.95cm}
|
||||
|
||||
\matrix[matrix of nodes,
|
||||
nodes={draw, align=center, minimum size=\minimumEntrySize},
|
||||
row 1/.style={nodes={draw=none, gray, font=\footnotesize, scale=\scaletitles}},
|
||||
column 1/.style={nodes={draw=none, gray, font=\footnotesize,
|
||||
scale=\scaletitles}}]
|
||||
{
|
||||
\node{}; &
|
||||
\node{(1,1)}; &
|
||||
\node{(1,2)}; &
|
||||
\node{(2,1)}; &
|
||||
\node{(2,2)};\\
|
||||
%----------%
|
||||
\node{Piece 1}; &
|
||||
\node{0}; & \node{0}; & \node{0}; & \node[text=blue]{1};\\
|
||||
%----------%
|
||||
\node{Piece 2}; &
|
||||
\node{0}; & \node[text=blue]{1}; & \node{0}; & \node{0};\\
|
||||
%----------%
|
||||
\node{Piece 3}; &
|
||||
\node[text=blue]{1}; & \node{0}; & \node{0}; & \node{0};\\
|
||||
%----------%
|
||||
\node{Piece 4}; &
|
||||
\node{0}; & \node{0}; & \node[text=blue]{1}; & \node{0};\\
|
||||
};
|
||||
\end{tikzpicture}
|
||||
\caption{}
|
||||
\label{fig:type_1_goal_and_labeling:labeling}
|
||||
\end{subfigure}
|
||||
\caption[fig A - Example for fig]{Some example}
|
||||
\label{fig:type_1_goal_and_labeling}
|
||||
\end{figure}
|
||||
% Motivate why standard loss functions (cross-entropy, MSE) break down under class
|
||||
% imbalance: they optimize average accuracy, which a model can game by predicting
|
||||
% the majority class. Introduce concrete examples (e.g. medical diagnosis, fraud
|
||||
% detection) where the cost of a false negative vastly outweighs a false positive,
|
||||
% and where the practitioner needs to operate at a specific FPR budget. Argue that
|
||||
% what is actually needed is direct control over TPR at a fixed FPR, not a
|
||||
% surrogate that only loosely correlates with it.
|
||||
|
||||
|
||||
\begin{table}[H]
|
||||
\centering
|
||||
\begin{tabular}{ |c|c|c|c|c|c| }
|
||||
\hline
|
||||
\multicolumn{1}{|c|}{Puzzle Type} & Direct & Neighbor & Perfect & Occupied & Feasible \\
|
||||
\hline
|
||||
Type 1 & {0\%} & {0.1\%} & 0 & {1\%} & 0 \\
|
||||
\hline
|
||||
Type 2 & {0.001\%} & {0.1\%} & 0 & {1.2\%} & 0 \\
|
||||
\hline
|
||||
\end{tabular}
|
||||
\caption[Table A - Example for table]{Some example}
|
||||
\label{table:plain_rl_results}
|
||||
\end{table}
|
||||
% ============================================================
|
||||
% PLACEMENT: Section 2.2, immediately after the imbalanced
|
||||
% classification section (sec:imbalanced-tpr-fpr). Establishes
|
||||
% AUC as the standard measure and surveys how the literature
|
||||
% optimizes it, before narrowing to partial AUC / single-point
|
||||
% objectives in the next section.
|
||||
% ============================================================
|
||||
\section{ROC Curves and AUC Optimization}
|
||||
\label{sec:roc-auc}
|
||||
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{עקומות ROC ואופטימיזציה של AUC}}
|
||||
|
||||
% \subsection{The ROC Curve}
|
||||
% Define the ROC curve as the plot of TPR vs.\ FPR as the threshold sweeps its full range.
|
||||
% Show that AUC equals $P(f(x^+) > f(x^-))$ for a random positive/negative pair (the
|
||||
% Wilcoxon--Mann--Whitney statistic): AUC is threshold-independent and measures ranking
|
||||
% quality, not calibration. Cite Hanley \& McNeil (1982).
|
||||
|
||||
% \subsection{Surrogate AUC Losses}
|
||||
% Explain why directly maximizing the empirical AUC is hard: the WMW indicator is pairwise
|
||||
% and discontinuous ($O(n^2)$ pairs). Introduce smooth surrogates --- sigmoid approximation
|
||||
% to the pairwise indicator, squared loss on score differences --- and their consistency
|
||||
% guarantees. Cite Cortes \& Mohri (2003/2004).
|
||||
|
||||
% \subsection{Stochastic AUC Maximization}
|
||||
% Describe mini-batch and compositional stochastic methods that make AUC optimization
|
||||
% practical at scale. Cover PESG (Yuan et al., NeurIPS 2020) and LibAUC (Yuan et al.,
|
||||
% ICML 2021), which is used as a baseline in \Cref{chapter:experiments}.
|
||||
% Key limitation shared by all: they optimize the \emph{whole-curve} AUC, not a specific
|
||||
% operating point on the ROC --- motivating the next section.
|
||||
|
||||
|
||||
\subsection{AUC and ROC definitions}
|
||||
|
||||
% TODO add formalizations
|
||||
% Formalize ROC and integration into AUC
|
||||
|
||||
A Receiver Operating Characteristic (ROC) is a graph that represents the performance of a classifier.
|
||||
The classifier in question is evaluated on binary-labeled data. The labeled data is ordered based off of the
|
||||
scores given by the classifier. Given any numerical threshold, it is then easy to see which labeled data points fall above and below.
|
||||
|
||||
For each threshold, the true positive rate and the false positive rate is given. The results for each threshold, and so for each possible TPR and FPR, are put on a graph. The resulting curve is the ROC curve.
|
||||
|
||||
The AUC is a number in the range $[0, 1]$ representing the total area under the curve. It is calculated by integrating over all TPRs and taking the resulting average FPRs. A result of $1$ represents a perfect classifier,
|
||||
while a result of $0$ represents a classifier with a perfect opposite result. A result of $0.5$ represents a classifier that found no separation whatsoever.
|
||||
|
||||
\subsection{AUC as a metric of model performance}
|
||||
|
||||
The AUC is a direct representation of the performance of a classifier, as opposed to a loss. There are instances where the loss may be minimized, yet seperation might be minimal. These two metrics might correlate, but do not necessarily correspond.
|
||||
|
||||
An example of this is a result for which all true samples receive a score of $0.51$ and all false samples receie a score of $0.49$. Using many losses, including BCE, this would result in a relatively high loss value. On this score however, the AUC is $1$. The model perfectly seperates the samples, so further training is not necessary.
|
||||
|
||||
Direct AUC optimization can be ideal with optimizing for imbalanced datasets. With most losses such as BCE or focal, the loss is calculated per-sample and then summed (or averaged). In order to properly train for both true and false populations, the scores are weighted, so that equal weight falls on both the true and the false populations. With direct AUC optimization, the imbalance does not affect the calculation, and the reweighting can be forgone.
|
||||
|
||||
Additionally, direct AUC methods are more resilient to label noise, as opposed to classical methods. Due to inseperability, correctly classifying a sample might require a model to incorrectly classify similar or even identical samples that were mislabeled. This is only worth it for the model in training if the total loss of the correctly labeled samples outweighs the total loss of the incorrectly labeled samples. For most methods, the weight per class is given by the class imbalance in the dataset, but this doesn't take into account the label noise in any way. Optimizing for AUC specifically would eliminate this risk.
|
||||
|
||||
\subsection{AUC optimization in literature}
|
||||
|
||||
While AUC as a metric for model performance has existed for a while, methods to optimize for it directly are relatively recent.
|
||||
%TODO add references
|
||||
|
||||
% ============================================================
|
||||
% PLACEMENT: Section 2.3, after ROC/AUC section above and
|
||||
% before the KDE background section. Narrows from whole-curve
|
||||
% AUC to the single operating-point objective that ROLL targets.
|
||||
% ============================================================
|
||||
\section{Partial AUC and TPR at Fixed FPR}
|
||||
\label{sec:partial-auc}
|
||||
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{AUC חלקי ו-TPR בנקודת FPR קבועה}}
|
||||
|
||||
% \subsection{Partial AUC}
|
||||
% Define partial AUC as the area under the ROC curve restricted to $\mathrm{FPR} \in [0, \alpha]$.
|
||||
% Motivate: in high-stakes settings (fraud, medical triage) only the low-FPR region matters.
|
||||
% Survey recent pAUC optimizers that extend the PESG/compositional framework
|
||||
% (Lin et al., 2022; Yang \& Ying, 2022).
|
||||
|
||||
% \subsection{TPR at a Fixed FPR: The Single-Point Objective}
|
||||
% Distinguish pAUC (integral over an interval) from the exact single-point objective:
|
||||
% maximize $\mathrm{TPR}(f, \alpha)$ for a fixed budget $\alpha$.
|
||||
% Argue that post-hoc threshold search and pAUC integration both fail to provide a
|
||||
% differentiable end-to-end training signal for this precise target --- this is the gap
|
||||
% that ROLL fills.
|
||||
|
||||
% \subsection{Neyman-Pearson Classification (Brief)}
|
||||
% Introduce NP classification (Cannon et al., 2002; Tong, 2013) as the statistical
|
||||
% framing: minimize type-II error subject to $\mathrm{FPR} \le \alpha$.
|
||||
% Note that classical NP methods are not gradient-based and cannot be applied to DNNs.
|
||||
% Full treatment deferred to \Cref{sec:neyman-pearson} at the end of this chapter.
|
||||
|
||||
TODO write this section.
|
||||
|
||||
|
||||
% ============================================================
|
||||
% PLACEMENT: Section 2.4, after partial AUC and before the
|
||||
% noisy labels section. Provides the technical KDE background
|
||||
% that Ch.\ 3 (sec:roll-kde) builds on directly. End with a
|
||||
% forward pointer to sec:roll-kde.
|
||||
% ============================================================
|
||||
\section{Kernel Density Estimation}
|
||||
\label{sec:kde-background}
|
||||
\addcontentsline{tocheb}{section}{\protect\numberline{\secnumforhebrewtoc}{אמידת צפיפות גרעין (KDE)}}
|
||||
|
||||
% \subsection{The Parzen Window Estimator}
|
||||
% Define KDE: $\hat{p}(x) = \frac{1}{nh}\sum_i K\!\bigl(\frac{x-x_i}{h}\bigr)$,
|
||||
% where $K$ is a kernel and $h > 0$ is the bandwidth.
|
||||
% Present common kernels: Gaussian, Epanechnikov, sigmoid. Discuss how kernel
|
||||
% smoothness affects gradient properties. Cite Parzen (1962), Silverman (1986).
|
||||
|
||||
% \subsection{Bandwidth Selection}
|
||||
% Describe Silverman's rule-of-thumb ($h = 1.06\,\hat\sigma\,n^{-1/5}$) as the default.
|
||||
% Mention cross-validation as the principled alternative, expensive at training time.
|
||||
% In ROLL, bandwidth is fixed (Silverman) or treated as a learnable parameter;
|
||||
% details deferred to Ch.\ 3.
|
||||
|
||||
% \subsection{Differentiability and Use as a Loss Ingredient}
|
||||
% Explain why KDE with a smooth kernel is differentiable w.r.t.\ the support points $x_i$:
|
||||
% the density at any query is a smooth weighted sum over all support points, so gradients
|
||||
% flow back to network parameters. This property is why KDE is suitable as a building
|
||||
% block inside a loss function. Forward pointer: \Cref{sec:roll-kde} exploits this to
|
||||
% construct ROLL's differentiable threshold estimate from the estimated score distributions.
|
||||
|
||||
When describing the source distribution for samples, rarely do they conform to some parametric distribution.
|
||||
Some data, such as adult heights, follows a Gaussian-like distribution; the occurrence of radioactive decay
|
||||
events or car arrivals at an intersection follows a Poisson distribution. For arbitrary real-world data,
|
||||
however, no such assumption can be made safely.
|
||||
|
||||
This motivates Kernel Density Estimation (KDE). Rather than assuming a parametric form, KDE uses the
|
||||
empirical sample itself as the model: each observed point $x_i$ contributes a smooth bump --- a
|
||||
\emph{kernel} --- to the estimated density. The assumption is that nearby points are plausibly likely,
|
||||
so probability mass is spread locally around each sample. Summing all kernels and normalizing yields a
|
||||
smooth PDF that mirrors the shape of the underlying distribution without committing to a parametric family.
|
||||
|
||||
Formally, let $\{x_i\}_{i=1}^n$ be the observed samples, $K$ a kernel function satisfying
|
||||
$\int_{-\infty}^{\infty} K(u)\,du = 1$, and $h > 0$ the \emph{bandwidth} controlling the spread of
|
||||
each kernel. Setting the correct bandwidth is important - too low a bandwidth parameter and the KDE becomes too jagged, and the probability anywhere there was no candidate approaches zero, avoiding generalization. Too high a bandwidth parameter, on the other hand, smooths out the distribution entirely, and the entire distribution starts to approach that of the kernel itself, losing the specificity that the candidates bring.
|
||||
|
||||
The KDE density estimator is:
|
||||
\[
|
||||
\hat{p}(x) = \frac{1}{nh} \sum_{i=1}^{n} K\!\left(\frac{x - x_i}{h}\right)
|
||||
\]
|
||||
Each term $\frac{1}{h}K\!\bigl(\frac{x-x_i}{h}\bigr)$ is a kernel centered at $x_i$; the
|
||||
$1/h$ factor compensates for the change of variables and ensures the full sum integrates to 1.
|
||||
|
||||
Valid kernels must be non-negative, symmetric around zero, and normalized. These differ in smoothness, tail behavior, and computational cost. A particularly useful choice is
|
||||
the \emph{logistic kernel}:
|
||||
\[
|
||||
K(u) = \frac{e^{-u}}{(1 + e^{-u})^2}
|
||||
\]
|
||||
which is smooth, bell-shaped, and centered at zero. Its key property is that its CDF has a closed form,
|
||||
which is the sigmoid function:
|
||||
\[
|
||||
\int_{-\infty}^{x} K(u)\,du \;=\; \frac{1}{1+e^{-x}} \;=:\; \sigma(x)
|
||||
\]
|
||||
This gives the KDE cumulative distribution function a closed form as well:
|
||||
\[
|
||||
\hat{F}(\tau) = \frac{1}{n}\sum_{i=1}^{n} \sigma\!\left(\frac{\tau - x_i}{h}\right)
|
||||
\]
|
||||
|
||||
|
||||
% ============================================================
|
||||
% PLACEMENT: Section 2.5, AFTER the KDE background section
|
||||
% (sec:kde-background) and BEFORE the Neyman-Pearson bridge
|
||||
% (sec:neyman-pearson, which becomes 2.6).
|
||||
%
|
||||
% Purpose: establish that existing pointwise losses are
|
||||
% fragile under label corruption, motivating both the
|
||||
% practical robustness of ROLL and the poisoning experiment
|
||||
% results in Ch. 4.
|
||||
%
|
||||
% Narrative arc:
|
||||
% 1. Define label noise types: symmetric (uniform flip),
|
||||
% asymmetric / class-conditional (only certain class
|
||||
% transitions occur), instance-dependent. Our poisoning
|
||||
% experiment is asymmetric: P(\tilde{y}=0 \mid y=1)>0,
|
||||
% P(\tilde{y}=1 \mid y=0)=0.
|
||||
% 2. Why BCE is fragile: the symmetry condition (Ghosh
|
||||
% et al., AAAI 2017) — a loss \ell is noise-tolerant
|
||||
% iff \sum_k \ell(f(x),k) = C (constant). CE fails
|
||||
% this; MAE satisfies it but converges slowly. Cite
|
||||
% Feng et al. (IJCAI 2020) for "Can CE be robust?"
|
||||
% (only under very strict conditions). Implication:
|
||||
% mislabeled samples produce large unmitigated
|
||||
% gradients; DNNs memorize noise late in training
|
||||
% (memorization effect).
|
||||
% 3. Family of existing fixes — keep brief; these are what
|
||||
% ROLL sidesteps structurally:
|
||||
% (a) Robust loss functions: MAE (noise-tolerant,
|
||||
% slow), GCE (Zhang & Sabuncu, NeurIPS 2018;
|
||||
% interpolates MAE↔CE via q), asymmetric losses
|
||||
% (Zhou et al., ICML 2021), symmetrization
|
||||
% (arXiv:2605.20347).
|
||||
% (b) Label correction via noise transition matrix
|
||||
% (Patrini et al., CVPR 2017): estimate
|
||||
% T[i,j]=P(\tilde{y}=j|y=i), correct logits by
|
||||
% T^{-1}. Hard for asymmetric noise without a
|
||||
% clean reference set.
|
||||
% (c) Sample selection / co-teaching (Han et al.,
|
||||
% NeurIPS 2018): two networks select small-loss
|
||||
% examples for each other, exploiting the
|
||||
% memorization effect. Requires dual-model
|
||||
% training.
|
||||
% 4. Ranking / distributional objectives and noise
|
||||
% robustness: pairwise losses depend on score
|
||||
% differences, not absolute labels — one mislabeled
|
||||
% sample affects O(n) pairs, each with a small gradient
|
||||
% weight. Wang et al. (arXiv:2305.14258) formally show
|
||||
% noise robustness of partial-AUC objectives.
|
||||
% 5. Bridge (observation, not oversell): ROLL's KDE-based
|
||||
% distributional objective shares this structural
|
||||
% property. Fake negatives (copies of positives with
|
||||
% flipped labels) shift the estimated negative density
|
||||
% but do not erase distributional separation when true
|
||||
% positive scores remain clustered. ROLL achieves this
|
||||
% without any explicit noise-handling mechanism; the
|
||||
% poisoning experiment in Ch. 4 provides empirical
|
||||
% evidence.
|
||||
% ============================================================
|
||||
\section{Learning with Noisy Labels}
|
||||
\label{sec:noisy-labels}
|
||||
|
||||
% SUBSECTION STRUCTURE:
|
||||
%
|
||||
% \subsection{Types of Label Noise}
|
||||
% - Symmetric / uniform: each label flipped with
|
||||
% probability \varepsilon across all classes.
|
||||
% - Asymmetric / class-conditional: only certain class
|
||||
% transitions occur. Our poisoning experiment is of
|
||||
% this type: P(\tilde{y}=0 \mid y=1)>0,
|
||||
% P(\tilde{y}=1 \mid y=0)=0.
|
||||
% - Instance-dependent: flip probability varies per
|
||||
% sample (hardest to handle; mention only briefly).
|
||||
% - Note: even moderate noise rates significantly hurt
|
||||
% pointwise losses in the imbalanced setting, because
|
||||
% mislabeled positives are rare to begin with.
|
||||
%
|
||||
% \subsection{Why Cross-Entropy Is Fragile Under Label Noise}
|
||||
% - Introduce symmetry condition \cite{ghosh2017robust}:
|
||||
% a loss is noise-tolerant iff
|
||||
% \sum_k \ell(f(\mathbf{x}), k) = C (constant).
|
||||
% - CE does NOT satisfy this; MAE does (but is slow).
|
||||
% - Memorization effect: DNNs learn clean patterns early,
|
||||
% then memorize noisy labels late in training.
|
||||
% - Cite: \cite{ghosh2017robust}, \cite{feng2020can}.
|
||||
%
|
||||
% \subsection{Approaches to Robust Learning}
|
||||
% - Paragraph 1 — Robust loss functions:
|
||||
% MAE \cite{ghosh2017robust},
|
||||
% GCE \cite{zhang2018generalized} (L_q interpolation),
|
||||
% asymmetric losses \cite{zhou2021asymmetric}.
|
||||
% Key tradeoff: noise-tolerance vs.\ convergence speed.
|
||||
% - Paragraph 2 — Transition matrix correction
|
||||
% \cite{patrini2017making}: forward correction
|
||||
% multiplies logits by T^{-1}. Canonical method but
|
||||
% requires estimating T, which is hard for asymmetric
|
||||
% noise without a clean anchor set.
|
||||
% - Paragraph 3 — Sample selection / co-teaching
|
||||
% \cite{han2018coteaching}: two nets select small-loss
|
||||
% samples for each other each epoch. Bypasses T
|
||||
% estimation but requires dual-model training.
|
||||
%
|
||||
% \subsection{Ranking Objectives and Noise Robustness}
|
||||
% - Pairwise ranking losses are structurally more
|
||||
% noise-robust: a single mislabeled sample affects O(n)
|
||||
% pairs, each with small gradient weight, vs.\ one large
|
||||
% pointwise gradient in BCE.
|
||||
% - Cite Wang et al.\ \cite{wang2023weakly} for formal
|
||||
% treatment in the partial-AUC setting.
|
||||
% - Close with bridge sentence: ROLL's distributional
|
||||
% threshold objective operates in the same structural
|
||||
% regime — not pairwise, but also not pointwise —
|
||||
% conferring noise robustness without explicit correction.
|
||||
% Empirical evidence deferred to the Experiments chapter.
|
||||
|
||||
While collecting data with correct labels should be a priority, in some cases it is impossible.
|
||||
Many data collection methods lead to imperfect labeling, resulting in a muddy ground truth.
|
||||
This could be due to a number of factors. The labels might be based on human labeling, which is error prone.
|
||||
The labeling might be true, yet the signal might be noisy or wrong, creating a mismatch between signal and label.
|
||||
The labeling could be a resource-intensive process, and so it might only be done for a portion of the data.
|
||||
And in some cases, the labeling is done based off a different learned model, which too could be faulty.
|
||||
|
||||
\subsection{Label noise formalization}
|
||||
|
||||
Given a dataset $ \mathcal{D} = \{(x_i, y_i)\} $, we would define the noisy version as $ \tilde{\mathcal{D}} = \{(x_i, \tilde{y_i})\} $. The noise $N_i$ is defined as:
|
||||
|
||||
\[
|
||||
N_i =
|
||||
\begin{cases}
|
||||
0 & \text{if } y_i = \tilde{y_i} \\
|
||||
1 & \text{otherwise}
|
||||
\end{cases}
|
||||
\]
|
||||
|
||||
Label noise is usually categorized as being symmetric, asymmetric or instance dependent.
|
||||
|
||||
For instance dependent noise, $\mathbb{P}(N_i = 1)$ depends on $x_i$. That is, the probability that a label is flipped depends on $x_i$. For the other two types, the probability of the label being flipped depends only on the original label itself.
|
||||
|
||||
For symmetric noise, $\mathbb{P}(N_i = 1 | y_i = 1) = \mathbb{P}(N_i = 1 | y_i = 0)$. The probability of the label being flipped does not depend on the original label. This is unlike asymmetric noise, for which the probability of the label flipping does depend on the original label.
|
||||
|
||||
\subsection{Existing approaches}
|
||||
|
||||
Prior work has been done in order to make noise-robust learning processes. These can be categorized into three main approaches.
|
||||
|
||||
\subsubsection{Label meta-training and adjustment}
|
||||
Under this approach, the training process is adjusted by modelling label noise explicitly.
|
||||
\citet{patrini2017making} introduced forward and backward loss correction: by estimating a
|
||||
noise transition matrix $T$, where $T_{ij} = P(\tilde{y}=j \mid y=i)$, the observed noisy
|
||||
loss can be corrected to recover the clean-label risk.
|
||||
Estimating $T$ requires identifying high-confidence \emph{anchor points} for each class from
|
||||
a pre-trained noisy model --- a two-phase procedure that may fail under high or asymmetric
|
||||
noise rates.
|
||||
\citet{ren2018learning} sidestep transition matrix estimation by using a small clean
|
||||
validation set and meta-gradient descent to assign per-sample loss weights at each training
|
||||
step, adapting dynamically to whatever noise is present without assuming a fixed noise
|
||||
structure.
|
||||
Both approaches require either prior knowledge of the noise model or access to a clean
|
||||
held-out set, which is not always available in practice.
|
||||
|
||||
\subsubsection{Sample selection}
|
||||
Rather than modifying the loss, sample selection methods filter likely-noisy examples during
|
||||
training.
|
||||
The approach is grounded in the \emph{memorization effect} documented by
|
||||
\citet{arpit2017memorization}: deep networks learn clean-label patterns first and memorize
|
||||
noisy labels only later in training.
|
||||
This means that at early training stages, samples with small loss are more likely to be
|
||||
correctly labelled and can be preferentially retained.
|
||||
\citet{han2018coteaching} exploit this with Co-teaching: two networks train in parallel,
|
||||
each selecting its own small-loss samples and passing them to the other network for the
|
||||
parameter update, preventing both from converging to the same memorized errors.
|
||||
\citet{li2020dividemix} extend the idea by fitting a per-class Gaussian mixture model to
|
||||
the loss distribution at each epoch to separate clean from noisy samples, then treating the
|
||||
noisy partition as unlabeled data within a semi-supervised learning framework.
|
||||
These methods can achieve strong performance but substantially increase training complexity,
|
||||
requiring multiple models or multi-phase procedures.
|
||||
|
||||
\subsubsection{Robust loss functions}
|
||||
A third line of work designs loss functions that are intrinsically resistant to label noise,
|
||||
requiring no changes to the training procedure beyond swapping the loss term.
|
||||
\citet{ghosh2017robust} established a sufficient condition for noise tolerance: a loss
|
||||
$\ell$ tolerates uniform symmetric noise if and only if
|
||||
|
||||
\[
|
||||
\sum_{k=1}^{K} \ell(f(\mathbf{x}), k) = C \quad \text{for all } \mathbf{x},\, f
|
||||
\]
|
||||
|
||||
for some constant $C$ independent of $f(\mathbf{x})$.
|
||||
Intuitively, if the total loss over all class assignments is constant, randomly flipping
|
||||
labels cannot shift the expected risk minimizer.
|
||||
Mean Absolute Error (MAE) satisfies this condition --- one can verify that
|
||||
$\sum_k (1 - f_k(\mathbf{x})) = K - 1$ for a $K$-class softmax output ---
|
||||
whereas cross-entropy does not.
|
||||
However, MAE converges slowly in practice because its gradient has constant magnitude
|
||||
regardless of prediction confidence, making it insensitive to easy examples that should
|
||||
receive small updates.
|
||||
|
||||
\citet{zhang2018generalized} proposed the Generalised Cross Entropy (GCE) loss, defined via
|
||||
the $L_q$ family:
|
||||
|
||||
\[
|
||||
\ell_q(f(\mathbf{x}), y) = \frac{1 - f_y(\mathbf{x})^q}{q}, \quad q \in (0,\,1]
|
||||
\]
|
||||
|
||||
where $f_y(\mathbf{x})$ is the model's predicted probability for the true class $y$.
|
||||
As $q \to 0$ the loss recovers cross-entropy (via L'H\^{o}pital's rule); at $q = 1$ it
|
||||
reduces to MAE, which satisfies the symmetry condition.
|
||||
Intermediate values therefore trade noise tolerance for faster convergence.
|
||||
We evaluate GCE at $q = 0.7$ as a baseline in the Experiments chapter.
|
||||
\citet{zhou2021asymmetric} extend the theoretical framework to asymmetric noise, introducing
|
||||
the \emph{asymmetry ratio} to quantify how noise tolerance degrades when flip rates differ
|
||||
between classes.
|
||||
\citet{feng2020can} confirm that standard cross-entropy cannot be made noise-tolerant
|
||||
through simple modification, establishing robust losses as a genuinely distinct family
|
||||
rather than a patch on existing objectives.
|
||||
|
||||
All three families above treat noise as a quantity to be explicitly corrected --- by
|
||||
adjusting labels, filtering samples, or redesigning the loss.
|
||||
ROLL takes a different route: its distributional threshold objective aggregates over the
|
||||
full score distribution of each class, so a small fraction of mislabelled samples shifts
|
||||
the estimated density slightly but cannot dominate the threshold.
|
||||
This structural robustness arises from the distributional formulation itself, without any
|
||||
explicit noise-handling mechanism; empirical evidence is provided in the Experiments
|
||||
chapter.
|
||||
|
||||
% ============================================================
|
||||
% PLACEMENT: Section 2.6 (was 2.5), near the END of Ch. 2,
|
||||
% after KDE and Noisy Labels sections, just before the
|
||||
% chapter summary. This section bridges from prior work to
|
||||
% the method chapter by framing ROLL as a learnable,
|
||||
% gradient-based NP test.
|
||||
% NOTE: the label sec:neyman-pearson is unchanged; existing
|
||||
% \Cref references in Ch. 3 (sec:problem-formulation) still
|
||||
% work without modification.
|
||||
% ============================================================
|
||||
\section{Connection to Neyman-Pearson}
|
||||
\label{sec:neyman-pearson}
|
||||
|
||||
% Introduce the Neyman-Pearson lemma: among all tests at a
|
||||
% given false-positive rate, the likelihood-ratio test
|
||||
% maximizes TPR. Frame ROLL as a learnable, gradient-based
|
||||
% realization of this principle — rather than assuming known
|
||||
% class-conditional distributions (as classical NP does),
|
||||
% ROLL estimates them from model scores during training.
|
||||
% Briefly note that existing NP-inspired methods do not
|
||||
% support deep learning / gradient-based optimization; this
|
||||
% is the gap ROLL fills.
|
||||
|
||||
TODO write this section.
|
||||
|
||||
|
||||
@@ -58,7 +58,13 @@
|
||||
in
|
||||
{
|
||||
devShells.${system}.default = pkgs.mkShell {
|
||||
buildInputs = [ (pkgs.texlive.combine texPkgs) pkgs.culmus pkgs.fontconfig ];
|
||||
buildInputs = [
|
||||
(pkgs.texlive.combine texPkgs)
|
||||
pkgs.culmus
|
||||
pkgs.fontconfig
|
||||
pkgs.poppler-utils
|
||||
(pkgs.python3.withPackages (ps: [ ps.sympy ps.numpy ps.matplotlib ps.scipy ]))
|
||||
];
|
||||
shellHook = ''
|
||||
export FONTCONFIG_FILE="${fontsConf}"
|
||||
'';
|
||||
|
||||
@@ -124,6 +124,7 @@
|
||||
|
||||
\usepackage{graphicx}
|
||||
\graphicspath{ {content/images/} }
|
||||
\usepackage{booktabs}
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
\usepackage{titling}
|
||||
@@ -236,10 +237,35 @@
|
||||
\begin{appendices}
|
||||
|
||||
\chapter{Some title for apendix A}
|
||||
\label[appendix]{appendix:numerically_distinguished_comp}
|
||||
\label{appendix:numerically_distinguished_comp}
|
||||
\addtocontents{tocheb}{\contentsline{chapter}{\protect\numberline{נספח א}{\quad\qquadדרישת תאימות כלשהי}}{{\beginL\thepage\endL}}}
|
||||
\input{content/appendixs/appendix_a}
|
||||
|
||||
\chapter{Gaussian ROLL: Gradient Derivation}
|
||||
\label{appendix:gauss-grad}
|
||||
\addtocontents{tocheb}{\contentsline{chapter}{\protect\numberline{נספח ב}{\quad\qquadנגזרת גאוסיאנית של ROLL}}{{\beginL\thepage\endL}}}
|
||||
\input{content/appendixs/appendix_gauss_grad}
|
||||
|
||||
\chapter{KDE ROLL: Gradient Derivation}
|
||||
\label{appendix:kde-grad}
|
||||
\addtocontents{tocheb}{\contentsline{chapter}{\protect\numberline{נספח ג}{\quad\qquadנגזרת KDE של ROLL}}{{\beginL\thepage\endL}}}
|
||||
\input{content/appendixs/appendix_kde_grad}
|
||||
|
||||
\chapter{KDE ROLL: Sigmoid Kernel — Explicit Equations}
|
||||
\label{appendix:kde-sigmoid}
|
||||
\addtocontents{tocheb}{\contentsline{chapter}{\protect\numberline{נספח ד}{\quad\qquadKDE ROLL: ניסוח מפורש עם גרעין סיגמואיד}}{{\beginL\thepage\endL}}}
|
||||
\input{content/appendixs/appendix_kde_sigmoid}
|
||||
|
||||
\chapter{KDE ROLL: Bisection for CDF Inversion}
|
||||
\label{appendix:kde-bisection}
|
||||
\addtocontents{tocheb}{\contentsline{chapter}{\protect\numberline{נספח ה}{\quad\qquadKDE ROLL: היפוך CDF בשיטת חצייה}}{{\beginL\thepage\endL}}}
|
||||
\input{content/appendixs/appendix_kde_bisection}
|
||||
|
||||
\chapter{KDE ROLL: Newton--Raphson Inversion (Superseded)}
|
||||
\label{appendix:kde-nr}
|
||||
\addtocontents{tocheb}{\contentsline{chapter}{\protect\numberline{נספח ו}{\quad\qquadKDE ROLL: היפוך ניוטון-רפסון}}{{\beginL\thepage\endL}}}
|
||||
\input{content/appendixs/appendix_kde_nr}
|
||||
|
||||
\end{appendices}
|
||||
|
||||
%~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~%
|
||||
|
||||
Reference in New Issue
Block a user