backup: 2026-07-25 14:39

2026-07-25 14:39:02 +03:00
parent 3a2b6bfde8
commit 3e65bf2dab
19 changed files with 884 additions and 32 deletions
@@ -67,6 +67,215 @@ Lower priority (useful if sweeping many baselines):
 5. Satellite, Segment, Pen-local — common in full KEEL sweeps
 6. KC1/PC1 — software metrics datasets; different domain from biology/finance

+** Noisy Labels Literature
+
+Sources found 2026-07-13 for thesis sec:noisy-labels (section 2.5 in related work chapter).
+Poisoning finding: label-poisoning (true positives duplicated with flipped label) hurts
+bce-weighted more than roll. ROLL's distributional/threshold objective is structurally
+more robust to class-conditional label noise without any explicit noise-handling mechanism.
+
+*** Competing Method Concepts
+
+Four families per Li & Zhu survey (arXiv:2404.04159). All require knowing or estimating
+something about the noise; ROLL requires none of it.
+
+*1. Robust Loss Functions* — change the loss so mislabeled samples cannot dominate the gradient.
+- Key insight: symmetry condition (Ghosh 2017): sum_k l(f(x),k)=C => noise-tolerant.
+  CE fails this; MAE satisfies it.
+- MAE: provably tolerant, very slow to converge.
+- GCE / L_q (Zhang & Sabuncu 2018): interpolates MAE<->CE via q. Tune for tradeoff.
+- Asymmetric losses (Zhou 2021): saturate on suspected noisy samples; different margins
+  per class direction. Suited to class-conditional noise.
+- Label smoothing: a soft regularization variant; does not fully satisfy symmetry condition
+  but reduces overconfidence on noisy labels.
+- Requires: approximate noise rate to choose hyperparameters.
+
+*2. Noise Transition Matrix / Label Correction* — model the corruption explicitly.
+- Estimate T where T[i,j] = P(observed label j | true label i).
+- Forward correction (Patrini 2017): multiply model output by T^{-1} before the loss.
+- Requires: a clean anchor set or structural assumptions about T. Hard for one-sided
+  asymmetric noise (our poisoning: only positives corrupted) without an anchor.
+- Representative methods: GLC, RoG, CAN, Noise Adaptation Layer.
+
+*3. Sample Selection* — identify and train only on likely-clean samples.
+- Memorization effect: DNNs fit clean patterns first, noisy labels late in training.
+  Small-loss samples early on are likely clean.
+- Co-teaching (Han 2018): two nets cross-select small-loss samples each epoch.
+- MentorNet, JoCoR, TopoFilter, MORPH are variants.
+- Requires: two models; breaks down at high noise or with imbalance (minority samples
+  often have high loss even when clean, so they get incorrectly filtered out).
+
+*4. Semi-Supervised Learning (SSL)-Based* — treat noisy samples as unlabeled, not wrong.
+- Divide training into clean (labeled) and noisy (unlabeled) subsets, then run SSL.
+- DivideMix: fits a mixture model to per-sample loss to decide clean/noisy split,
+  then applies MixMatch. The dominant method on standard benchmarks as of 2024.
+- Other representatives: SELFIE, Jo-SRC, UNICON, TCL, SoftMatch.
+- Requires: reliable clean/noisy split; the mixture model can fail under imbalance
+  because minority-class loss distributions overlap with noisy majority distributions.
+
+*Common thread*: all four families are corrective — they assume or estimate noise
+structure. ROLL's robustness is structural (distributional objective), not corrective.
+
+*Benchmark gap (confirmed by Li & Zhu 2024)*: the entire field evaluates on multi-class
+image datasets with synthetic symmetric/asymmetric noise or a handful of real web-scraped
+image datasets (Clothing1M ~38.5% noise, Food-101N ~18.4%, ANIMAL-10N ~8%,
+WebVision ~20%, CIFAR-10N/100N human-reannotated). No tabular data, no binary
+imbalanced setting, no one-sided minority-class corruption anywhere in the standard
+benchmark suite.
+
+*** Noisy Label Datasets (real-world, from the literature)
+
+| Dataset | Size | Noise rate | Domain |
+|---------|------|------------|--------|
+| Clothing1M | ~1M | ~38.5% | clothing images (web) |
+| Food-101N | ~310K | ~18.4% | food images (web) |
+| ANIMAL-10N | 55K | ~8% | animal images |
+| WebVision | 2.5M | ~20% | general web images |
+| CIFAR-10N / CIFAR-100N | 50K | varies | CIFAR re-annotated by humans |
+
+All are image datasets. No standard noisy-label tabular benchmark exists.
+
+*** Surveys
+
+| Key | Venue | arXiv / DOI | Notes |
+|-----|-------|-------------|-------|
+| song2022survey | IEEE TNNLS 2022/23 | 10.1109/TNNLS.2022.3152527 | Main survey; 62 methods in 5 categories |
+| survey2025label | Systems Sci & Control Eng 2025 | 10.1080/21642583.2025.2488120 | Deep learning focus; 5-type taxonomy |
+| li2024noisy | arXiv:2404.04159 (Apr 2024) | — | Li & Zhu; 4-category taxonomy; confirmed no binary/imbalanced coverage |
+
+*** Robust Loss Functions
+
+| Key | Venue | Notes |
+|-----|-------|-------|
+| ghosh2017robust | AAAI 2017, arXiv:1712.09482 | Symmetry condition: sum_k l(f(x),k)=C => noise-tolerant; CE fails; MAE passes |
+| zhang2018generalized | NeurIPS 2018 | GCE: L_q interpolates MAE<->CE via q; q->0 = MAE, q->1 = CE |
+| zhou2021asymmetric | ICML 2021, PMLR v139 | Asymmetric loss; different margins per class; suited to class-conditional noise |
+| feng2020can | IJCAI 2020 | Can CE be robust? — only under very restricted conditions |
+| symmetrization2025 | arXiv:2605.20347 | Symmetrize any loss via additive complement (2025) |
+
+*** Label Correction / Transition Matrix
+
+| Key | Venue | Notes |
+|-----|-------|-------|
+| patrini2017making | CVPR 2017 | Estimate T[i,j]=P(y_tilde=j|y=i); correct logits by T^{-1}. Canonical forward-correction |
+| cannings2020classification | Biometrika 2020, arXiv:1805.11505 | kNN/SVM robust to imperfect labels; LDA not unless class priors equal |
+
+*** Sample Selection
+
+| Key | Venue | Notes |
+|-----|-------|-------|
+| han2018coteaching | NeurIPS 2018 | Co-teaching: two nets select small-loss samples for each other per epoch |
+
+*** Semi-Supervised / DivideMix Family
+
+| Key | Venue | Notes |
+|-----|-------|-------|
+| dividemix | ICLR 2020 | Dominant SSL-based method; GMM splits clean/noisy, then MixMatch |
+
+*** AUC / Ranking Objectives and Noise Robustness
+
+| Key | Venue | Notes |
+|-----|-------|-------|
+| xie2024wsauc | IEEE TPAMI 2024, arXiv:2305.14258 | WSAUC/rpAUC; unifies weak supervision for AUC; rpAUC removes high-loss pairs — more robust under noisy labels. ADDED to thesis.bib |
+| auc2025retrieval | arXiv:2510.00137 | AUC-driven learning for neural retrieval; robustness framing |
+
+*** BibTeX Entries (staged here; move to thesis.bib when writing sec:noisy-labels)
+
+#+begin_src bibtex
+@article{song2022survey,
+  title={Learning From Noisy Labels With Deep Neural Networks: A Survey},
+  author={Song, Hwanjun and Kim, Minseok and Park, Dongkwan and Shin, Yooju and Lee, Jae-Gil},
+  journal={IEEE Transactions on Neural Networks and Learning Systems},
+  volume={34},
+  number={11},
+  pages={8135--8153},
+  year={2023},
+  doi={10.1109/TNNLS.2022.3152527}
+}
+
+@article{li2024noisy,
+  title={Noisy Label Processing for Classification: A Survey},
+  author={Li, Mengting and Zhu, Chuang},
+  journal={arXiv preprint arXiv:2404.04159},
+  year={2024}
+}
+
+@inproceedings{ghosh2017robust,
+  title={Robust Loss Functions under Label Noise for Deep Neural Networks},
+  author={Ghosh, Aritra and Kumar, Himanshu and Sastry, P. S.},
+  booktitle={Proceedings of the Thirty-First AAAI Conference on Artificial Intelligence},
+  pages={1919--1925},
+  year={2017}
+}
+
+@inproceedings{patrini2017making,
+  title={Making Deep Neural Networks Robust to Label Noise: A Loss Correction Approach},
+  author={Patrini, Giorgio and Rozza, Alessandro and Krishna Menon, Aditya and Nock, Richard and Qu, Lizhen},
+  booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition},
+  pages={1944--1952},
+  year={2017}
+}
+
+@inproceedings{zhang2018generalized,
+  title={Generalized Cross Entropy Loss for Training Deep Neural Networks with Noisy Labels},
+  author={Zhang, Zhilu and Sabuncu, Mert R.},
+  booktitle={Advances in Neural Information Processing Systems},
+  volume={31},
+  year={2018}
+}
+
+@inproceedings{han2018coteaching,
+  title={Co-teaching: Robust Training of Deep Neural Networks with Extremely Noisy Labels},
+  author={Han, Bo and Yao, Quanming and Yu, Xingrui and Niu, Gang and Xu, Miao and Hu, Weihua and Tsang, Ivor W. and Sugiyama, Masashi},
+  booktitle={Advances in Neural Information Processing Systems},
+  volume={31},
+  year={2018}
+}
+
+@inproceedings{zhou2021asymmetric,
+  title={Asymmetric Loss Functions for Learning with Noisy Labels},
+  author={Zhou, Xiong and Liu, Xianming and Jiang, Junjun and Gao, Xin and Ji, Xiangyang},
+  booktitle={Proceedings of the 38th International Conference on Machine Learning},
+  pages={12846--12856},
+  year={2021},
+  volume={139},
+  series={Proceedings of Machine Learning Research},
+  publisher={PMLR}
+}
+
+@inproceedings{feng2020can,
+  title={Can Cross Entropy Loss Be Robust to Label Noise?},
+  author={Feng, Lei and Shu, Senlin and Lin, Zhuoyi and Lv, Fengmei and Li, Li and An, Bo},
+  booktitle={Proceedings of the Twenty-Ninth International Joint Conference on Artificial Intelligence},
+  pages={2206--2212},
+  year={2020}
+}
+
+@article{cannings2020classification,
+  title={Classification with imperfect training labels},
+  author={Cannings, Timothy I. and Fan, Yingying and Samworth, Richard J.},
+  journal={Biometrika},
+  volume={107},
+  number={2},
+  pages={311--330},
+  year={2020},
+  doi={10.1093/biomet/asaa011}
+}
+
+@article{symmetrization2025,
+  title={Symmetrization of Loss Functions for Robust Training of Neural Networks
+         in the Presence of Noisy Labels},
+  journal={arXiv preprint arXiv:2605.20347},
+  year={2025}
+}
+
+@article{auc2025retrieval,
+  title={Optimizing What Matters: {AUC}-Driven Learning for Robust Neural Retrieval},
+  journal={arXiv preprint arXiv:2510.00137},
+  year={2025}
+}
+#+end_src
+
 ** Paper Subnodes

 - [[id:bf0fc08a-e806-48df-b188-7a2c4c41c693][impl/paper-tabpfn]] — TabPFN: in-context learning for small tabular classification (ICLR 2023)