diff --git a/.gitignore b/.gitignore index feb61c3..87870a7 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ _region_.prv/ # Auto-downloaded LaTeX package files *.sty +*.pdf +.#* +*.xdv diff --git a/content/method/figures/threshold_weighting.pdf b/content/method/figures/threshold_weighting.pdf new file mode 100644 index 0000000..844e075 Binary files /dev/null and b/content/method/figures/threshold_weighting.pdf differ diff --git a/content/method/figures/threshold_weighting.png b/content/method/figures/threshold_weighting.png new file mode 100644 index 0000000..513996b Binary files /dev/null and b/content/method/figures/threshold_weighting.png differ diff --git a/content/method/figures/threshold_weighting.py b/content/method/figures/threshold_weighting.py new file mode 100644 index 0000000..f100309 --- /dev/null +++ b/content/method/figures/threshold_weighting.py @@ -0,0 +1,108 @@ +""" +Illustrates the gradient locality property of KDE-ROLL. + +Upper panel: score distributions for both classes with operating threshold. +Lower panel: gradient magnitude per sample, showing bell-shaped decay from threshold. + +Run from the thesis root: + nix develop --command python3 content/method/figures/threshold_weighting.py +""" + +import numpy as np +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +# --- Add these configuration rules right here --- +plt.rcParams.update({ + "text.usetex": False, # Rely on Matplotlib's math parser instead of the system compiler + "mathtext.fontset": "cm", # Emulate standard LaTeX Computer Modern styles flawlessly + "pdf.fonttype": 42 # Ensure fonts are embedded as TrueType vector curves instead of Type 3 paths +}) + +OUT_DIR = "content/method/figures" + +np.random.seed(7) + +# --- Data ------------------------------------------------------------------ +n = 120 +alpha = 0.25 # target FPR + +neg_scores = np.sort(np.random.normal(-0.6, 0.75, n)) +pos_scores = np.sort(np.random.normal(1.1, 0.80, n)) + +# Threshold: alpha-quantile of the negative-class scores +tau = np.percentile(neg_scores, 100 * (1 - alpha)) + +# --- Bandwidth (Silverman's rule per class) -------------------------------- +h_neg = 1.06 * neg_scores.std() * n ** (-1 / 5) +h_pos = 1.06 * pos_scores.std() * n ** (-1 / 5) + + +def sigma_prime(x, h): + """Logistic kernel PDF: derivative of the sigmoid-smoothed CDF.""" + e = np.exp(-np.abs(x) / h) + return e / (h * (1 + e) ** 2) + + +# --- KDE-ROLL gradients (eq:kde-grad-combined) ---------------------------- +# y=1: (1/n) * sigma_prime(score - tau) +grad_pos = (1.0 / n) * sigma_prime(pos_scores - tau, h_pos) + +# y=0: -(scale) * sigma_prime(tau - score_i) / sum_j[sigma_prime(tau - score_j)] +# where scale = (1/n) * sum_j[sigma_prime(pos_j - tau)] +neg_kernel = sigma_prime(tau - neg_scores, h_neg) +scale = (1.0 / n) * np.sum(sigma_prime(pos_scores - tau, h_pos)) +grad_neg_signed = -scale * neg_kernel / neg_kernel.sum() +grad_neg_abs = np.abs(grad_neg_signed) + +# --- Smooth reference bell curves ----------------------------------------- +x_ref = np.linspace(neg_scores.min() - 0.3, pos_scores.max() + 0.3, 400) +bell_pos = sigma_prime(x_ref - tau, h_pos) +bell_pos *= (grad_pos.max() / bell_pos.max()) +bell_neg = sigma_prime(tau - x_ref, h_neg) +bell_neg *= (grad_neg_abs.max() / bell_neg.max()) + +# --- Plot ----------------------------------------------------------------- +C_NEG = "#4477AA" +C_POS = "#CC6633" +C_TAU = "#333333" + +fig, (ax1, ax2) = plt.subplots( + 2, 1, figsize=(5.5, 4.2), sharex=True, + gridspec_kw={"height_ratios": [1.3, 1.0]} +) +fig.subplots_adjust(hspace=0.06) + +# — Top: score distributions — +ax1.hist(neg_scores, bins=20, density=True, alpha=0.45, color=C_NEG, + label=r"Negative class ($y=0$)") +ax1.hist(pos_scores, bins=20, density=True, alpha=0.45, color=C_POS, + label=r"Positive class ($y=1$)") +ax1.axvline(tau, color=C_TAU, linestyle="--", linewidth=1.4, + label=rf"Threshold $\tau$ (FPR $=\alpha={alpha}$)") +ax1.set_ylabel("Density", fontsize=9) +ax1.set_yticks([]) +ax1.legend(fontsize=7.5, loc="upper right", framealpha=0.85) + +# — Bottom: gradient magnitudes — +ax2.scatter(neg_scores, grad_neg_abs, color=C_NEG, s=12, zorder=3, + alpha=0.75, label=r"$|\partial\mathcal{L}/\partial f_\theta(\mathbf{x})|$, $y=0$") +ax2.scatter(pos_scores, grad_pos, color=C_POS, s=12, zorder=3, + alpha=0.75, label=r"$\partial\mathcal{L}/\partial f_\theta(\mathbf{x})$, $y=1$") +ax2.plot(x_ref, bell_neg, color=C_NEG, linewidth=1.2, alpha=0.5) +ax2.plot(x_ref, bell_pos, color=C_POS, linewidth=1.2, alpha=0.5) +ax2.axvline(tau, color=C_TAU, linestyle="--", linewidth=1.4) +ax2.set_ylabel("Gradient magnitude", fontsize=9) +ax2.set_xlabel(r"Score $f_\theta(\mathbf{x})$", fontsize=9) +ax2.set_yticks([]) +ax2.legend(fontsize=7.5, loc="upper right", framealpha=0.85) + +for ax in (ax1, ax2): + ax.tick_params(labelsize=8) + ax.spines[["top", "right"]].set_visible(False) + +fig.savefig(f"{OUT_DIR}/threshold_weighting.pdf", bbox_inches="tight") +fig.savefig(f"{OUT_DIR}/threshold_weighting.png", bbox_inches="tight", dpi=180) +print(f"Saved to {OUT_DIR}/threshold_weighting.{{pdf,png}}") +print(f"Sum neg: {np.sum(grad_neg_signed)}, pos: {np.sum(grad_pos)}") diff --git a/content/method/method.tex b/content/method/method.tex index 89b2081..04e35d8 100644 --- a/content/method/method.tex +++ b/content/method/method.tex @@ -763,6 +763,64 @@ additive shift of the raw scores then produces a non-uniform shift of the transf changing the shape of the fitted distribution rather than merely translating it, and the balance property no longer holds. +\paragraph{Gradient locality.} +\label{para:gradient-locality} + +A second structural consequence of the ROLL formulation is that gradient mass is +\emph{concentrated near the operating threshold $\tau$}. Samples whose scores lie +close to $\tau$ receive disproportionately large gradient updates, while samples +already far on the correct side contribute very little. + +This follows directly from \Cref{eq:kde-grad-combined}. For the positive class ($y_i = 1$), +the gradient magnitude is +\[ + \left|\frac{\partial \mathcal{L}}{\partial f_\theta(\mathbf{x}_i)}\right| + = \frac{1}{|\mathcal{B}_1|}\,\sigma_1'(f_\theta(\mathbf{x}_i) - \tau), +\] +which is a bell-shaped kernel PDF that peaks when $f_\theta(\mathbf{x}_i) = \tau$ and +decays to zero as the score moves away from the threshold in either direction. +For the negative class ($y_i = 0$), the gradient is proportional to +$\sigma_0'(\tau - f_\theta(\mathbf{x}_i))$ (normalized over $\mathcal{B}_0$), which is +again bell-shaped and centred on $\tau$. + +The effect is that each gradient step focuses the model's attention on the region that +directly determines the operating-point performance: samples that could plausibly be +re-ranked relative to the threshold at the current step. Samples comfortably classified +on the correct side contribute negligible signal, not because they are deemed unimportant, +but because their contribution to the CDF estimate at $\tau$ is already saturated. + +This stands in contrast to standard losses such as cross-entropy, where the per-sample +gradient is determined by the predicted probability, with no direct dependence on an +operating threshold. Under heavy class imbalance this can cause the dominant class to +``push'' the threshold away from the intended operating point, a problem that gradient +locality avoids by construction. + +\begin{observation}[Gradient locality] +\label{obs:gradient-locality} +For KDE-ROLL, the gradient magnitude of sample $\mathbf{x}_i$ is proportional to +the kernel PDF evaluated at the distance between $f_\theta(\mathbf{x}_i)$ and the +operating threshold $\tau$. Samples nearest to $\tau$ receive the largest updates; +samples far from $\tau$ in either direction contribute negligible gradient. +\end{observation} + +\Cref{fig:threshold-locality} illustrates this for a simulated two-class setting. The +upper panel shows the score distributions and the operating threshold; the lower panel +shows the gradient magnitude of each sample. Both classes exhibit a clear bell-shaped +concentration of gradient mass around $\tau$, with smooth decay on either side. + +\begin{figure}[H] + \centering + \includegraphics[width=0.82\textwidth]{content/method/figures/threshold_weighting.pdf} + \caption{Gradient locality in KDE-ROLL. \textit{Upper}: score distributions for the + negative (blue, $y=0$) and positive (orange, $y=1$) classes; dashed line marks the + operating threshold $\tau$ at FPR $= \alpha$. \textit{Lower}: gradient magnitude + $|\partial\mathcal{L}/\partial f_\theta(\mathbf{x})|$ for each sample. Both classes + show a bell-shaped concentration of gradient mass near $\tau$, with samples far from + the threshold contributing negligible updates. Smooth curves are the kernel PDFs scaled + for reference.} + \label{fig:threshold-locality} +\end{figure} + \subsection{Gradient Computation and the Custom Backward Pass} \label{sec:roll-backward} diff --git a/flake.nix b/flake.nix index 05031b8..b1a8122 100644 --- a/flake.nix +++ b/flake.nix @@ -62,7 +62,7 @@ (pkgs.texlive.combine texPkgs) pkgs.culmus pkgs.fontconfig - (pkgs.python3.withPackages (ps: [ ps.sympy ])) + (pkgs.python3.withPackages (ps: [ ps.sympy ps.numpy ps.matplotlib ])) ]; shellHook = '' export FONTCONFIG_FILE="${fontsConf}"