From 33d55169efa14f1a7243e6ad6f45537e90a3a954 Mon Sep 17 00:00:00 2001 From: Aner Zakobar Date: Wed, 9 Sep 2026 12:07:01 +0300 Subject: [PATCH] Bandwidth freeze --- .../appendixs/appendix_bandwidth_freeze.tex | 219 +++++++++++++++ content/method/method.tex | 54 +++- flake.nix | 83 +++--- main.tex | 8 + mentor.tex | 258 ++++++++++++++++++ 5 files changed, 580 insertions(+), 42 deletions(-) create mode 100644 content/appendixs/appendix_bandwidth_freeze.tex create mode 100644 mentor.tex diff --git a/content/appendixs/appendix_bandwidth_freeze.tex b/content/appendixs/appendix_bandwidth_freeze.tex new file mode 100644 index 0000000..de9271a --- /dev/null +++ b/content/appendixs/appendix_bandwidth_freeze.tex @@ -0,0 +1,219 @@ +\Cref{sec:roll-kde-backward} derives \eqref{eq:kde-grad-combined} by treating the +bandwidths $v_0, v_1$ as constants during differentiation. This appendix makes explicit +what is given up by that choice: we derive the gradient that results from +differentiating through the bandwidths as well (\Cref{sec:bandwidth-freeze-general}), +then give two counterexamples (\Cref{sec:bandwidth-freeze-cex1,sec:bandwidth-freeze-cex2}) +showing that the resulting correction is not a vanishing refinement of +\eqref{eq:kde-grad-combined} but can cancel it exactly or reverse its sign entirely. + +\section{The Fully Adaptive Gradient} +\label{sec:bandwidth-freeze-general} + +In order to derive the gradient w.r.t.\ the bandwidth parameter, we now write +$v_k = v_k(\mathbf{s}^{(k)})$ for the bandwidth parameter of class $k$, now a +differentiable function of that class's scores $\mathbf{s}^{(k)} = +\bigl(f_\theta(\mathbf{x}_j)\bigr)_{\mathbf{x}_j \in \mathcal{B}_k}$, as +\Cref{sec:kde-bandwidth} in fact computes it (via ISJ or the Silverman fallback) on every +forward pass. Differentiating the sigmoid kernel~\eqref{eq:sigmoid-kernel} with respect +to $v_k$ and applying the identity~\eqref{eq:sigmoid-identity} gives the +bandwidth-sensitivity term + +\begin{equation} + \rho_k \;:=\; \frac{\partial \hat{F}_k(\tau)}{\partial v_k} + \;=\; \frac{1}{v_k\,|\mathcal{B}_k|}\sum_{\mathbf{x}_j \in \mathcal{B}_k} + \bigl(\tau - f_\theta(\mathbf{x}_j)\bigr)\,\sigma_k'\bigl(\tau - f_\theta(\mathbf{x}_j)\bigr). + \label{eq:bandwidth-sensitivity} +\end{equation} + +\paragraph{Case $y_i = 1$.} +$\tau$ does not depend on $\mathcal{B}_1$, so the chain rule adds a single term to the +direct contribution already derived in~\eqref{eq:kde-grad-y1}: +\begin{equation} + \frac{dL}{df_\theta(\mathbf{x}_i)}\bigg|_{y_i=1} + = \underbrace{-\frac{1}{|\mathcal{B}_1|}\,\sigma_1'\bigl(\tau - f_\theta(\mathbf{x}_i)\bigr)} + _{\text{\eqref{eq:kde-grad-y1}}} + \;+\; \rho_1 \cdot \frac{dv_1}{df_\theta(\mathbf{x}_i)}. + \label{eq:bandwidth-freeze-y1} +\end{equation} + +\paragraph{Case $y_i = 0$.} +Here $\tau$ is defined implicitly by $\hat{F}_0(\tau) = 1-\alpha$, and $v_0$ is itself a +function of $\mathbf{x}_i$'s class. Totally differentiating the defining equation and +comparing to~\eqref{eq:kde-grad-tau} shows the fixed-bandwidth result picks up the same +kind of correction: +\begin{equation} + \frac{d\tau}{df_\theta(\mathbf{x}_i)} + = \underbrace{\frac{\sigma_0'\bigl(\tau-f_\theta(\mathbf{x}_i)\bigr)} + {\sum_{\mathbf{x}_j \in \mathcal{B}_0}\sigma_0'\bigl(\tau-f_\theta(\mathbf{x}_j)\bigr)}} + _{\text{\eqref{eq:kde-grad-tau}}} + \;-\; \frac{|\mathcal{B}_0|\,\rho_0} + {\sum_{\mathbf{x}_j \in \mathcal{B}_0}\sigma_0'\bigl(\tau-f_\theta(\mathbf{x}_j)\bigr)} + \cdot \frac{dv_0}{df_\theta(\mathbf{x}_i)}, + \label{eq:bandwidth-freeze-tau} +\end{equation} +and chaining through $\partial \hat{F}_1(\tau)/\partial\tau$ from~\eqref{eq:kde-dF1-dtau}: +\begin{equation} + \frac{dL}{df_\theta(\mathbf{x}_i)}\bigg|_{y_i=0} + = \underbrace{\eqref{eq:kde-grad-combined}\big|_{y_i=0}}_{\text{unchanged}} + \;-\; \left(\frac{1}{|\mathcal{B}_1|}\sum_{\mathbf{x}_j\in\mathcal{B}_1} + \sigma_1'\bigl(\tau-f_\theta(\mathbf{x}_j)\bigr)\right) + \frac{|\mathcal{B}_0|\,\rho_0} + {\sum_{\mathbf{x}_j \in \mathcal{B}_0}\sigma_0'\bigl(\tau-f_\theta(\mathbf{x}_j)\bigr)} + \cdot \frac{dv_0}{df_\theta(\mathbf{x}_i)}. + \label{eq:bandwidth-freeze-y0} +\end{equation} + +In both cases, the fully adaptive gradient equals~\eqref{eq:kde-grad-combined} plus +a correction proportional to $dv_k/df_\theta(\mathbf{x}_i)$. For the Silverman fallback of +\Cref{sec:kde-bandwidth}, with class mean $\bar{f}_k$ and class variance +$\hat\sigma_k^2$, +\begin{equation} + \frac{dv_k}{df_\theta(\mathbf{x}_i)} + = -\frac{v_k\,\bigl(f_\theta(\mathbf{x}_i) - \bar{f}_k\bigr)}{|\mathcal{B}_k|\,\hat\sigma_k^2}, + \qquad \mathbf{x}_i \in \mathcal{B}_k, + \label{eq:silverman-dv-df} +\end{equation} +which the two counterexamples below instantiate directly. + +Notably, ISJ has no closed form, and so would require re-derivation through its numerical computation. +The Silverman instance already suffices to establish our claim. + +\section{Counterexample 1: Exact Cancellation} +\label{sec:bandwidth-freeze-cex1} + +Take $\mathcal{B}_0 = \{-1, 1\}$ with any bandwidth $v_0$, and +$\mathcal{B}_1 = \{0, b\}$ for a free scalar $b > 0$, with $\alpha = 1/2$. + +By symmetry of $\mathcal{B}_0$ about $0$ and evenness of the sigmoid +kernel~\eqref{eq:sigmoid-kernel}, $\hat{F}_0(0) = \tfrac{1}{2}\bigl[\sigma(1;v_0) + +\sigma(-1;v_0)\bigr] = \tfrac{1}{2}$ for \emph{every} $v_0$, so $\tau = 0$ exactly, +independent of the bandwidth. + +Silverman's rule on $\mathcal{B}_1 = \{0,b\}$ ($|\mathcal{B}_1|=2$, +$\hat\sigma_1 = b/2$) gives $h_1 = \kappa b$ with $\kappa := 1.06 \cdot 2^{-1/5}/2$, i.e.\ +$v_1 = 1/(\kappa b)$ — the bandwidth scales linearly with $b$. + +\paragraph{True adaptive loss.} Substituting $v_1 = 1/(\kappa b)$, +\[ + L(b) = \hat{F}_1(0) = \tfrac{1}{2}\bigl[\sigma(0;v_1) + \sigma(-b;v_1)\bigr] + = \tfrac{1}{2}\Bigl[\tfrac{1}{2} + \sigma(-1/\kappa)\Bigr] + = \tfrac{1}{4} + \tfrac{1}{2}\sigma(-1/\kappa), +\] +using $v_1 b = 1/\kappa$, a constant. The loss does not depend on $b$ at all: +\[ + \frac{dL}{db} = 0 \quad\text{exactly, for every } b > 0. +\] +Rescaling $b$ rescales its own bandwidth identically, so the standardized KDE shape at +$\tau = 0$ never moves. + +A loss that derives to a parameter gradient of zero is a serious failure mode for +gradient-based learning: it gives optimization nothing to act on, regardless of where +$b$ is initialized. In this example, once $b > 1$, this batch's negative-class scores +$\{-1,1\}$ are confined to $[-1,1]$, so scores above $1$ are a regime reserved solely +for class $1$, a more separated region of score space. +Under the fully adaptive gradient, $b$ would never move toward it: $dL/db = 0$ +identically, so gradient descent stalls wherever $b$ starts, however far from that +regime. + +\paragraph{What \eqref{eq:kde-grad-combined} gives instead.} Plugging directly into the +frozen-bandwidth term of~\eqref{eq:bandwidth-freeze-y1} (equivalently +\eqref{eq:kde-grad-y1}), +\[ + \frac{dL}{db}\bigg|_{\text{\eqref{eq:kde-grad-combined}}} + = -\tfrac{1}{2}\,\sigma_1'(-b; v_1), +\] +which is strictly negative for every finite $b, v_1 > 0$, since $\sigma_1' > 0$ +everywhere. \Cref{eq:kde-grad-combined} therefore reports a nonzero, consistently signed +gradient. +We see that using the frozen-bandwidth term, eventually, $b$ would be pushed +higher, approaching $b>1$, and achieving better separability. + +\section{Counterexample 2: Persistent Sign Reversal} +\label{sec:bandwidth-freeze-cex2} + +Take the same $\mathcal{B}_0 = \{-1,1\}$ (so again $\tau = 0$ for any $v_0$), and +$\mathcal{B}_1 = \{1+\varepsilon,\ b\}$ for a small fixed $\varepsilon > 0$ and a free +scalar $b > 1+\varepsilon$. We define $\delta := b - (1+\varepsilon) > 0$. +Silverman's rule again gives $h_1 = \kappa\delta$, $v_1 = 1/(\kappa\delta)$, with $\kappa$ as above. + +\paragraph{What \eqref{eq:kde-grad-combined} gives.} Plugging into +\eqref{eq:kde-grad-y1} and simplifying with the sigmoid-identity~\eqref{eq:sigmoid-identity}, +\begin{equation} + \frac{dL}{db}\bigg|_{\text{\eqref{eq:kde-grad-combined}}} + = -\tfrac{1}{2}\,\sigma_1'(-b;v_1) + = -\frac{1}{8\kappa\delta}\operatorname{sech}^2\!\left(\frac{b}{2\kappa\delta}\right) + \;<\; 0 \qquad \text{for every } \delta > 0. + \label{eq:cex2-frozen} +\end{equation} +This is the intuitively correct sign. $\mathbf{x}_i$ already scores above $\tau$, so +increasing $b$ further should only increase separability, therefore decreasing the loss. + +\paragraph{The fully adaptive gradient.} Substituting the same configuration into +\eqref{eq:bandwidth-freeze-y1} requires $\rho_1$ and $dv_1/db$. Since +$v_1 = 1/(\kappa\delta)$ depends on $b$ only through $\delta = b-(1+\varepsilon)$, +\begin{equation} + \frac{dv_1}{db} = -\frac{1}{\kappa\delta^2} = -\frac{v_1}{\delta}. + \label{eq:cex2-dv1-db} +\end{equation} +Expanding \eqref{eq:bandwidth-sensitivity} for $\mathcal{B}_1 = \{1+\varepsilon, b\}$ +and again applying the sigmoid-identity~\eqref{eq:sigmoid-identity}, +\begin{equation} + \rho_1 = -\frac{1}{8}\left[(1+\varepsilon)\operatorname{sech}^2\!\left(\frac{(1+\varepsilon)v_1}{2}\right) + + b\operatorname{sech}^2\!\left(\frac{bv_1}{2}\right)\right]. + \label{eq:cex2-rho1} +\end{equation} +Substituting \eqref{eq:cex2-dv1-db} and \eqref{eq:cex2-rho1} into \eqref{eq:bandwidth-freeze-y1} +alongside \eqref{eq:cex2-frozen}, the $\operatorname{sech}^2(b/(2\kappa\delta))$ terms +combine and $b - \delta = 1+\varepsilon$ cancels the remainder, leaving: +\begin{equation} + \frac{dL}{db}\bigg|_{\text{full}} + = \frac{1+\varepsilon}{8\kappa\delta^2}\left[ + \operatorname{sech}^2\!\left(\frac{b}{2\kappa\delta}\right) + + \operatorname{sech}^2\!\left(\frac{1+\varepsilon}{2\kappa\delta}\right) + \right]. + \label{eq:cex2-full} +\end{equation} +Since $\operatorname{sech}^2(x) > 0$ for every finite $x$ and $1+\varepsilon,\, \kappa,\, \delta^2 > 0$, \eqref{eq:cex2-full} is +strictly positive for every $\delta > 0$, the opposite sign to +\eqref{eq:cex2-frozen}, established here for the entire domain $b > 1+\varepsilon$. + +\Cref{tab:cex2-values} confirms \eqref{eq:cex2-full} numerically (cross-checked against +an independent finite-difference derivative of $L(b)$) across an exponential range of +$\delta$, from just off the degenerate boundary $\delta \to 0^+$ to well-separated, for +$\varepsilon = 0.05$ ($\kappa \approx 0.4614$): + +\begin{center} +\begin{tabular}{rrrr} +\toprule +$\delta$ & $b$ & \eqref{eq:kde-grad-combined} (frozen) & fully adaptive, \eqref{eq:cex2-full} \\ +\midrule +$10^{-2}$ & 1.06 & $-1.82\times 10^{-98}$ & $+1.86\times 10^{-95}$ \\ +$10^{-1}$ & 1.15 & $-1.62\times 10^{-10}$ & $+1.66\times 10^{-8}$ \\ +$10^{0}$ & 2.05 & $-1.24\times 10^{-2}$ & $+1.09\times 10^{-1}$ \\ +$10^{1}$ & 11.05 & $-8.30\times 10^{-3}$ & $+3.68\times 10^{-3}$ \\ +$10^{2}$ & 101.05 & $-9.81\times 10^{-4}$ & $+3.87\times 10^{-5}$ \\ +\bottomrule +\end{tabular} +\label{tab:cex2-values} +\end{center} + +The fully adaptive gradient is positive at every finite +$\delta > 0$, never matching \eqref{eq:cex2-frozen}'s sign. Gradient +descent on the fully adaptive loss would push an already well-separated positive sample +back toward the decision threshold, throughout its entire domain, weakening separability +against what we are trying to achieve. + +\section{Discussion} +\label{sec:bandwidth-freeze-discussion} + +The two counterexamples isolate the same correction term, +$\rho_k \cdot dv_k/df_\theta(\mathbf{x}_i)$, in two regimes: in +\Cref{sec:bandwidth-freeze-cex1} it exactly cancels the gradient we need to continue learning, +and in \Cref{sec:bandwidth-freeze-cex2} it inverts it entirely. In neither case is it +simply a small correction to \eqref{eq:kde-grad-combined}. It is not bounded by, or shrinking +relative to, the term it is added to. With this term added, we could no longer guarantee +gradient sign correctness (\Cref{obs:gradient-sign-correctness}): \Cref{sec:bandwidth-freeze-cex2} +gives a sample with $y_i=1$ whose gradient is positive throughout its entire domain, in +direct violation of the guarantee. \Cref{sec:roll-kde-backward} therefore adopts +\eqref{eq:kde-grad-combined}, with $v_0, v_1$ frozen at their forward-pass values, as the +definition of the ROLL gradient. diff --git a/content/method/method.tex b/content/method/method.tex index 7cae3ec..bed6042 100644 --- a/content/method/method.tex +++ b/content/method/method.tex @@ -22,13 +22,21 @@ The \emph{true positive rate} (TPR) and \emph{false positive rate} (FPR) at thre \operatorname{TPR}(\tau) &= P\!\bigl(f_\theta(\mathbf{x}) > \tau \mid y = 1\bigr), \\ \operatorname{FPR}(\tau) &= P\!\bigl(f_\theta(\mathbf{x}) > \tau \mid y = 0\bigr). \end{align} -For $\alpha \in (0,1)$, let $\tau^*(\alpha)$ denote the threshold achieving -$\operatorname{FPR}(\tau^*(\alpha)) = \alpha$, and write -$\operatorname{TPR}(\alpha) = \operatorname{TPR}(\tau^*(\alpha))$ for the true positive rate -at that operating point. The curve $\alpha \mapsto \operatorname{TPR}(\alpha)$ is the ROC curve. + +Conversely, we may denote the inverse operations $\operatorname{FPR}^{-1}$ and $\operatorname{TPR}^{-1}$ to be: + +\begin{align} + \operatorname{TPR^{-1}}\!\bigl(P\!\bigl(f_\theta(\mathbf{x}) > \tau \mid y = 1\bigr)\bigr) &= \tau, \\ + \operatorname{FPR^{-1}}\!\bigl(P\!\bigl(f_\theta(\mathbf{x}) > \tau \mid y = 0\bigr)\bigr) &= \tau . +\end{align} + + +For $\alpha \in (0,1)$, let $\tau = \operatorname{FPR}^{-1}(\alpha)$ denote the threshold achieving +$\operatorname{FPR}(\tau) = \alpha$. +The curve $\alpha \mapsto \operatorname{TPR}(\operatorname{FPR^{-1}}(\alpha))$ is the ROC curve. The primary learning objective is to maximize its area: \begin{equation} - \max_\theta\;\operatorname{AUC}(f_\theta) = \int_0^1 \operatorname{TPR}(\alpha)\,d\alpha. + \max_\theta\;\operatorname{AUC}(f_\theta) = \int_0^1 \operatorname{TPR}(\operatorname{FPR}^{-1}(\alpha))\,d\alpha. \label{eq:objective-auc} \end{equation} Maximizing AUC is the natural objective under class imbalance and label noise, where no @@ -410,6 +418,14 @@ Positive-class samples are weighted by their individual kernel PDF at $\tau$; negative-class samples are weighted by the mean positive-class kernel PDF scaled by each sample's share of the negative-class kernel mass. +Equation~\eqref{eq:kde-grad-combined} is derived by treating the bandwidths $v_0, v_1$ +as fixed constants, even though \Cref{sec:kde-bandwidth} re-estimates them from the +batch scores on every forward pass. We adopt this as a deliberate stop-gradient +convention: as \Cref{obs:gradient-sign-correctness} shows, it is exactly what +guarantees every per-sample gradient points in the direction that improves separation. +Differentiating fully through the bandwidth estimator can break that guarantee, as +Appendix~\ref{appendix:bandwidth-freeze} shows by counterexample. + %------------------------------------------------ \subsection{Properties of the ROLL Formulation} @@ -470,6 +486,34 @@ average of 99 per-threshold objectives each satisfying \eqref{eq:grad-balance}, is preserved by linearity of summation. \end{corollary} +\paragraph{Gradient sign correctness.} +\label{para:gradient-sign-correctness} + +A third structural consequence of \eqref{eq:kde-grad-combined} is that every per-sample +gradient points in the direction that improves separation, never against it. + +\begin{observation}[Gradient sign correctness] +\label{obs:gradient-sign-correctness} +For every $\mathbf{x}_i \in \mathcal{B}$, +\[ + y_i = 1 \;\Longrightarrow\; \frac{\partial \mathcal{L}_{\text{ROLL-TPR@FPR}}}{\partial f_\theta(\mathbf{x}_i)} \leq 0, + \qquad + y_i = 0 \;\Longrightarrow\; \frac{\partial \mathcal{L}_{\text{ROLL-TPR@FPR}}}{\partial f_\theta(\mathbf{x}_i)} \geq 0. +\] +\end{observation} + +This follows directly from \eqref{eq:kde-grad-combined}: for $y_i = 1$ the gradient is +$-\sigma_1'(\tau-f_\theta(\mathbf{x}_i))/|\mathcal{B}_1|$, and $\sigma_1' \geq 0$ +everywhere; for $y_i = 0$ the gradient is a product of two nonnegative quantities (a +mean of $\sigma_1'$ values and a normalized $\sigma_0'$ ratio). Gradient descent +therefore never pushes a positive-class score down, nor a negative-class score up, a +minimal requirement for any loss intended to drive separation. + +This guarantee is specific to the frozen-bandwidth gradient~\eqref{eq:kde-grad-combined}. +Differentiating fully through the adaptive bandwidth of \Cref{sec:kde-bandwidth} can +break it: Appendix~\ref{appendix:bandwidth-freeze} gives an exact counterexample where +the fully adaptive gradient violates this sign guarantee. + \paragraph{Gradient locality.} \label{para:gradient-locality} diff --git a/flake.nix b/flake.nix index 045549e..299a752 100644 --- a/flake.nix +++ b/flake.nix @@ -55,6 +55,49 @@ fontsConf = pkgs.makeFontsConf { fontDirectories = [ pkgs.culmus ]; }; + + thesisBuildInputs = [ + pkgs.coreutils + pkgs.fontconfig + (pkgs.texlive.combine texPkgs) + pkgs.culmus + ]; + + mkThesisDoc = { name, texFile }: + let pdfFile = builtins.replaceStrings [ ".tex" ] [ ".pdf" ] texFile; + in pkgs.stdenvNoCC.mkDerivation { + inherit name; + src = self; + buildInputs = thesisBuildInputs; + phases = [ "unpackPhase" "buildPhase" "installPhase" ]; + buildPhase = '' + export PATH="${pkgs.lib.makeBinPath thesisBuildInputs}"; + mkdir -p .cache/texmf-var .cache/fontconfig + + # Build a fontconfig pointing at culmus with a writable cache + FCACHE="$(pwd)/.cache/fontconfig" + cat > .cache/fonts.conf < + + + ${pkgs.culmus}/share/fonts/truetype + $FCACHE + +EOF + export FONTCONFIG_FILE="$(pwd)/.cache/fonts.conf" + export FONTCONFIG_CACHE="$FCACHE" + fc-cache -f 2>/dev/null || true + + env TEXMFHOME=.cache TEXMFVAR=.cache/texmf-var \ + latexmk -xelatex -f --interaction=nonstopmode \ + ${texFile} || true + test -f ${pdfFile} + ''; + installPhase = '' + mkdir -p $out + cp ${pdfFile} $out/ + ''; + }; in { devShells.${system}.default = pkgs.mkShell { @@ -70,43 +113,9 @@ ''; }; - packages.${system}.default = pkgs.stdenvNoCC.mkDerivation rec { - name = "roll-thesis"; - src = self; - buildInputs = [ - pkgs.coreutils - pkgs.fontconfig - (pkgs.texlive.combine texPkgs) - pkgs.culmus - ]; - phases = [ "unpackPhase" "buildPhase" "installPhase" ]; - buildPhase = '' - export PATH="${pkgs.lib.makeBinPath buildInputs}"; - mkdir -p .cache/texmf-var .cache/fontconfig - - # Build a fontconfig pointing at culmus with a writable cache - FCACHE="$(pwd)/.cache/fontconfig" - cat > .cache/fonts.conf < - - - ${pkgs.culmus}/share/fonts/truetype - $FCACHE - -EOF - export FONTCONFIG_FILE="$(pwd)/.cache/fonts.conf" - export FONTCONFIG_CACHE="$FCACHE" - fc-cache -f 2>/dev/null || true - - env TEXMFHOME=.cache TEXMFVAR=.cache/texmf-var \ - latexmk -xelatex -f --interaction=nonstopmode \ - main.tex || true - test -f main.pdf - ''; - installPhase = '' - mkdir -p $out - cp main.pdf $out/ - ''; + packages.${system} = { + default = mkThesisDoc { name = "roll-thesis"; texFile = "main.tex"; }; + mentor = mkThesisDoc { name = "roll-thesis-mentor"; texFile = "mentor.tex"; }; }; }; } diff --git a/main.tex b/main.tex index 51320cc..80093fa 100644 --- a/main.tex +++ b/main.tex @@ -167,6 +167,8 @@ \Crefname{observation}{Observation}{Observations} \Crefname{assumption}{Assumption}{Assumptions} \Crefname{remark}{Remark}{Remarks} +\crefname{appendix}{appendix}{appendices} +\Crefname{appendix}{Appendix}{Appendices} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %For Hebrew table of contents: (\addcontentsline is used to add section and subsections to the Hebrew toc. For some reason, the page number is reversed only for chapter, thus we use \addtocontents for them) \makeatletter @@ -256,6 +258,7 @@ \addtocontents{tocheb}{\contentsline{chapter}{\protect\numberline{\chapnumforhebrewtoc}{דיון ומסקנות}}{{\beginL\thepage\endL}}} \input{content/discussion_and_conclusions/discussion_and_conclusions} +\appendix \begin{appendices} \chapter{Gaussian ROLL: Forward Pass and Gradient Derivation} @@ -303,6 +306,11 @@ \addtocontents{tocheb}{\contentsline{chapter}{\protect\numberline{נספח ט}{\quad\qquadהוכחת טענת איזון הגרדיאנט}}{{\beginL\thepage\endL}}} \input{content/appendixs/appendix_grad_balance} +\chapter{KDE ROLL: Adaptive-Bandwidth Gradients and the Freezing Convention} +\label{appendix:bandwidth-freeze} +\addtocontents{tocheb}{\contentsline{chapter}{\protect\numberline{נספח י}{\quad\qquadKDE ROLL: גרדיאנט רוחב פס אדפטיבי ומוסכמת ההקפאה}}{{\beginL\thepage\endL}}} +\input{content/appendixs/appendix_bandwidth_freeze} + \end{appendices} %~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~% diff --git a/mentor.tex b/mentor.tex new file mode 100644 index 0000000..c0f4b35 --- /dev/null +++ b/mentor.tex @@ -0,0 +1,258 @@ +\documentclass[12pt,a4paper,oneside,onecolumn]{book} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% remove unnecessary warnings +\usepackage{silence} +\WarningFilter{caption}{Unknown document class} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% Hebrew + English via XeLaTeX: +\usepackage{fontspec} +\usepackage{polyglossia} +\setmainlanguage{english} +\setotherlanguage{hebrew} +% Culmus OpenType fonts for Hebrew (exact fc family names) +\newfontfamily\hebrewfont[Script=Hebrew]{Frank Ruehl CLM} +\newfontfamily\hebrewfontsf[Script=Hebrew]{Miriam CLM} +\newfontfamily\hebrewfonttt[Script=Hebrew]{Miriam Mono CLM} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% Set page numbering with no headers (according to guidelines): +\usepackage{fancyhdr} +\pagestyle{fancy} +\lhead{} +\chead{\thepage} +\rhead{} +\lfoot{} +\cfoot{} +\rfoot{} +\renewcommand{\headrulewidth}{0pt} +\fancypagestyle{plain}{\pagestyle{fancy}} +\setlength{\headheight}{15pt} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% Sub-figures/tables parenthesis: + +% For parenthesis in sub-figure reference +\usepackage[labelformat=simple]{subcaption} +\renewcommand\thesubfigure{(\alph{subfigure})} + +% For parenthesis before sub-table caption +\renewcommand\thesubtable{(\alph{subtable})} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% Misc packages: + +\usepackage{textcomp} %loaded before "gensymb" to avoid annoying warnings +\usepackage{gensymb} %Degree symbol +\usepackage{mathtools} %cases* +\usepackage{amssymb} %Math letters +\usepackage[shortlabels]{enumitem} +\usepackage[multiple]{footmisc} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%theorem, corollary and lemma definitions: + +\usepackage[amsmath, thmmarks]{ntheorem} + +\newtheorem{theorem}{Theorem} +\newtheorem{corollary}[theorem]{Corollary} +\newtheorem{lemma}[theorem]{Lemma} +\newtheorem{proposition}[theorem]{Proposition} +\newtheorem{observation}[theorem]{Observation} +\newtheorem{assumption}[theorem]{Assumption} +\newtheorem{remark}[theorem]{Remark} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%proof definition: + +\theoremstyle{nonumberplain} +\theoremheaderfont{\itshape} +\theorembodyfont{\normalfont} +\theoremseparator{.} +\theoremsymbol{\ensuremath{\square}} +\newtheorem{proof}{Proof} + +% Change back to default values - needed to repeat assumptions +\theoremstyle{plain} +\theoremheaderfont{\normalfont\bfseries} +\theorembodyfont{\itshape} +\theoremseparator{} +\theoremsymbol{} + +\usepackage{chngcntr} +\counterwithin{theorem}{chapter} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% Misc: +\DeclareMathOperator*{\argmax}{arg\,max} +\usepackage[ruled,vlined,algochapter]{algorithm2e} +\usepackage{csquotes} + +% Show numbers also for subsubsection +\setcounter{secnumdepth}{4} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% bibliography: + +\usepackage[numbers,sort,compress,square]{natbib} +\bibliographystyle{bibliography/unsrtnat_adjusted_for_bgu_thesis} + +%add bib to toc +\usepackage[nottoc]{tocbibind} + +%add command for citation with possessive apostrophe +\newcommand\citepossessive [1]{\citeauthor{#1}'s\ \cite{#1}} + +%Natbib cites cannot be broken to two lines +\makeatletter +\def\NAT@spacechar{~}% NEW +\makeatother + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% paragraph settings: +\setlength{\parindent}{0pt} +\setlength{\parskip} {5pt} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% drawing and graphics: +\usepackage{tikz} +\usetikzlibrary{matrix,positioning} + +\usepackage{float} %Not sure it will be needed + +\usepackage{graphicx} +\graphicspath{ {content/images/} } +\usepackage{booktabs} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\usepackage{titling} +\usepackage[titletoc]{appendix} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% Code listings: +\usepackage{listings} +\lstset{ + basicstyle=\ttfamily\footnotesize, + breaklines=true, + breakatwhitespace=false, + showstringspaces=false, + frame=single, + numbers=left, + numberstyle=\tiny\ttfamily, + numbersep=6pt, + xleftmargin=18pt, + framexleftmargin=18pt, + tabsize=4, + captionpos=b, + keepspaces=true, + columns=flexible, +} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%Not included since 'hyperref' and 'bookmark' are too problematic with Hebrew +% \usepackage[hidelinks]{hyperref} +% \usepackage{bookmark} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% cleveref: +\usepackage[capitalise]{cleveref} +%Workaround for babel and cleveref (for citing few equations) +\makeatletter +\def\@@number#1{#1} +\makeatother + +\Crefname{observation}{Observation}{Observations} +\Crefname{assumption}{Assumption}{Assumptions} +\Crefname{remark}{Remark}{Remarks} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%set spacing between lines +\usepackage{setspace} +\doublespacing + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%Title and names for the article +\newcommand{\thesistitle}{\textbf{Noise-Robust Gradient-Based Learning Utilizing Rate Optimized Likelihood Loss}} +\newcommand{\thesistitlehe}{\textbf{למידה עמידה לרעש מבוססת גרדיאנט תוך שימוש בפונקציית הפסד קירוב-הסתברות}} + +\newcommand{\thesisauthorname}{\textbf{Aner Zakobar}} +\newcommand{\thesisauthornamehe}{\textbf{ענר זקובר}} + +\newcommand{\thesissupervisername}{\textbf{Prof. Lior Rokach, Prof. Aryeh Kontorovich}} +\newcommand{\thesissupervisernamehe}{\textbf{פרופ. ליאור רוקח, פרופ. אריה קונטורוביץ}} + +\newcommand{\thesismonth}{\textbf{September}} +\newcommand{\thesismonthhe}{\textbf{ספטמבר}} +\newcommand{\thesisyear}{\textbf{2026}} +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +\begin{document} + +\begin{titlepage} + \centering + \vspace*{3cm} + {\LARGE\bfseries Noise-Robust Gradient-Based Learning\\[0.5em] + Utilizing Rate Optimized Likelihood Loss}\\[2.5em] + {\large Aner Zakobar}\\[0.5em] + {\normalsize Supervised by: Prof.\ Lior Rokach, Prof.\ Aryeh Kontorovich}\\[3em] + \rule{0.5\textwidth}{0.4pt}\\[1em] + {\large\itshape Excerpt: Chapter~3 \& Appendices A, C--F, I}\\[0.5em] + {\normalsize Note: cross-references to omitted chapters appear as \textbf{??}}\\[1em] + \rule{0.5\textwidth}{0.4pt} + \vfill + {\large September 2026} +\end{titlepage} + +\tableofcontents +\clearpage + +\mainmatter + +%~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~% + +\setcounter{chapter}{2} +\chapter{Method} +\label{chapter:method} +\input{content/method/method} + +%~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~% + +\begin{appendices} + +\setcounter{chapter}{0} +\chapter{Gaussian ROLL: Forward Pass and Gradient Derivation} +\label{appendix:gauss-grad} +\input{content/appendixs/appendix_gauss_grad} + +\setcounter{chapter}{2} +\chapter{KDE ROLL: Gradient Derivation} +\label{appendix:kde-grad} +\input{content/appendixs/appendix_kde_grad} + +\setcounter{chapter}{3} +\chapter{KDE ROLL: Sigmoid Kernel -- Explicit Equations} +\label{appendix:kde-sigmoid} +\input{content/appendixs/appendix_kde_sigmoid} + +\setcounter{chapter}{4} +\chapter{KDE ROLL: Bisection for CDF Inversion} +\label{appendix:kde-bisection} +\input{content/appendixs/appendix_kde_bisection} + +\setcounter{chapter}{5} +\chapter{KDE ROLL: Newton--Raphson Inversion (Superseded)} +\label{appendix:kde-nr} +\input{content/appendixs/appendix_kde_nr} + +\setcounter{chapter}{8} +\chapter{Proof of the Gradient Balance Proposition} +\label{appendix:grad-balance} +\input{content/appendixs/appendix_grad_balance} + +\end{appendices} + +%~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~% +\bibliography{bibliography/thesis} + +\end{document}