| { | |
| "analysis_kind": "exploratory", | |
| "question": "Does AdamW weight decay improve held-out accuracy of a small MLP on the bundled digits data?", | |
| "objective": "Replicate a known fixed study to evaluate engineering completion; do not claim new scientific confirmation.", | |
| "hypotheses": [ | |
| { | |
| "id": "h1", | |
| "prediction": "Mean paired accuracy difference for decay 0.01 versus 0 is positive.", | |
| "alternatives": [ | |
| "No effect", | |
| "Regularization reduces accuracy", | |
| "An apparent gain is caused by training randomness or an implementation error" | |
| ], | |
| "distinguishing_predictions": [ | |
| "Paired seeds and identical data/order isolate weight decay", | |
| "The shuffled-label control should lose useful accuracy", | |
| "Recomputing metrics from held-out predictions detects arithmetic mistakes" | |
| ] | |
| } | |
| ], | |
| "primary_metric": "Mean over registered seeds of test accuracy(weight_decay=0.01) minus test accuracy(weight_decay=0)", | |
| "falsification": "A paired conditional 95% bootstrap interval wholly below zero contradicts the directional hypothesis; an interval spanning zero is inconclusive, not proof of equivalence.", | |
| "analysis_plan": "Use 10000 paired bootstrap resamples of test example indices with RNG seed 2026, preserving pairing across methods and all trained seeds. This interval conditions on the trained models and does not estimate training-population uncertainty. Separately show each seed effect and its sample standard deviation. Treat other comparisons as descriptive, with no post-hoc primary selection.", | |
| "data_boundaries": "Frozen stratified 60/20/20 train/validation/test partition. Split seed 42. Divide pixels by the documented constant 16. Pilot reports validation only. No learned preprocessing or selection uses test data. Generalization is limited to this random example split, not new writers or datasets. Prior results on this evaluation set are known. All replication observations and conclusions are exploratory; no untouched-data claim is made.", | |
| "budget": "180.0 active minutes and 60 CLI invocations; explicit future changes are recorded separately, never silently extended.", | |
| "methods": [ | |
| "regularized", | |
| "unregularized", | |
| "sensitivity", | |
| "shuffled_labels", | |
| "logistic", | |
| "class_prior" | |
| ], | |
| "seeds": [ | |
| 0, | |
| 1, | |
| 2 | |
| ], | |
| "architecture": [ | |
| 64, | |
| 128, | |
| 10 | |
| ], | |
| "activation": "ReLU", | |
| "optimizer": "AdamW", | |
| "learning_rate": 0.001, | |
| "batch_size": 64, | |
| "epochs": 50, | |
| "weight_decay": { | |
| "regularized": 0.01, | |
| "unregularized": 0.0, | |
| "sensitivity": 0.001, | |
| "shuffled_labels": 0.01 | |
| }, | |
| "split_seed": 42, | |
| "bootstrap_seed": 2026, | |
| "bootstrap_resamples": 10000, | |
| "logistic_C": 1.0, | |
| "logistic_max_iter": 1000, | |
| "evaluation_id": "digits-stratified-42-test-v1", | |
| "hyperparameter_rationale": "Small prespecified network and fixed training budget provide fast feedback on an established task. The three decay values are a descriptive local sensitivity check, not a test-selected sweep. Three seeds expose limited initialization sensitivity; they do not support broad population claims.", | |
| "pilot_order": "Publish the regularized seed-0 validation-only pilot and checkpoint before fitting comparative arms.", | |
| "sources": [ | |
| "https://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html", | |
| "https://docs.pytorch.org/docs/stable/generated/torch.optim.AdamW.html", | |
| "https://www.alignmentforum.org/s/5GT3yoYM9gRmMEKqL" | |
| ], | |
| "prior_results_known": true, | |
| "measurements": [ | |
| "verified phase completion", | |
| "saved predictive accuracy", | |
| "descriptive paired accuracy difference" | |
| ], | |
| "prior_evidence": [ | |
| "https://github.com/BurnyCoder/eureka-digits-haiku", | |
| "https://github.com/BurnyCoder/eureka-digits-luna" | |
| ] | |
| } | |