| { |
| "description": "Controlled scaling experiment: 8M vs 46.5M parameter Diffusion-LM riddle solvers", |
| "dataset": { |
| "name": "Riddles \u2014 A Synthetic Riddle Dataset for NLP", |
| "source": "Kaggle", |
| "url": "https://www.kaggle.com/datasets/prajwaldongre/riddles-a-synthetic-riddle-dataset-for-nlp", |
| "license": "CC0", |
| "train_size": 232, |
| "heldout_size": 66, |
| "unique_total": 298, |
| "duplicates_removed": 702, |
| "split_method": "SHA-256 of normalized riddle text", |
| "answer_max_len": 4, |
| "vocab_source": "training set only" |
| }, |
| "checkpoints": { |
| "phase3": { |
| "label": "Phase 3 Reconstructed", |
| "architecture": { |
| "params": null, |
| "T": 200, |
| "d_model": 256, |
| "n_layers": 4 |
| }, |
| "training": { |
| "steps": 500, |
| "lr": 0.0003, |
| "schedule": "constant", |
| "batch": 64, |
| "seed": 42, |
| "T": 200 |
| }, |
| "eval_heldout": { |
| "exact_match_K1_pct": 46.96969696969697, |
| "exact_match_K10_pct": 46.96969696969697, |
| "token_f1_K1": 0.7124098124098122, |
| "n": 66 |
| }, |
| "eval_train": { |
| "exact_match_K1_pct": 87.5, |
| "exact_match_K10_pct": 86.45833333333333, |
| "token_f1_K1": 0.9597222222222221, |
| "n": 192 |
| }, |
| "decoding_modes": { |
| "A_full_chain_em_pct": 0.5, |
| "B_pure_noise_1shot_em_pct": 0.494949494949495, |
| "C_gold_corrupted_1shot_em_pct": 0.5303030303030303 |
| }, |
| "signal_degradation": { |
| "raw_xt_tok_acc_at_t2_pct": 100.0, |
| "model_tok_acc_at_t2_pct": 76.6, |
| "degradation_ppt": 23.4 |
| }, |
| "state_drift": { |
| "t2_oracle_tok_acc_pct": 75.9, |
| "t2_free_tok_acc_pct": 72.0, |
| "drift_ppt": 3.91, |
| "oracle_flat_across_noise": false |
| } |
| }, |
| "phase4_original": { |
| "label": "Phase 4 Original", |
| "architecture": { |
| "params": null, |
| "T": 500, |
| "d_model": 512, |
| "n_layers": 6 |
| }, |
| "training": { |
| "steps": 20000, |
| "lr": 0.0003, |
| "schedule": "constant", |
| "batch": 64, |
| "seed": 42, |
| "T": 500 |
| }, |
| "eval_heldout": { |
| "exact_match_K1_pct": 15.151515151515152, |
| "exact_match_K10_pct": 15.151515151515152, |
| "token_f1_K1": 0.5692279942279943, |
| "n": 66 |
| }, |
| "eval_train": { |
| "exact_match_K1_pct": 51.5625, |
| "exact_match_K10_pct": 50.520833333333336, |
| "token_f1_K1": 0.9034226190476198, |
| "n": 192 |
| }, |
| "decoding_modes": { |
| "A_full_chain_em_pct": 0.15656565656565657, |
| "B_pure_noise_1shot_em_pct": 0.11616161616161616, |
| "C_gold_corrupted_1shot_em_pct": 0.13636363636363635 |
| }, |
| "signal_degradation": { |
| "raw_xt_tok_acc_at_t2_pct": 100.0, |
| "model_tok_acc_at_t2_pct": 68.3, |
| "degradation_ppt": 31.7 |
| }, |
| "state_drift": { |
| "t2_oracle_tok_acc_pct": 68.3, |
| "t2_free_tok_acc_pct": 68.3, |
| "drift_ppt": 0.0, |
| "oracle_flat_across_noise": true |
| } |
| }, |
| "phase4_scheduled": { |
| "label": "Phase 4 Scheduled", |
| "architecture": { |
| "params": null, |
| "T": 500, |
| "d_model": 512, |
| "n_layers": 6 |
| }, |
| "training": { |
| "steps": 20000, |
| "lr": 0.0001, |
| "schedule": "warmup+cosine", |
| "batch": 64, |
| "seed": 42, |
| "T": 500 |
| }, |
| "eval_heldout": { |
| "exact_match_K1_pct": 13.636363636363637, |
| "exact_match_K10_pct": 13.636363636363637, |
| "token_f1_K1": 0.6034992784992784, |
| "n": 66 |
| }, |
| "eval_train": { |
| "exact_match_K1_pct": 40.625, |
| "exact_match_K10_pct": 39.583333333333336, |
| "token_f1_K1": 0.8827380952380958, |
| "n": 192 |
| }, |
| "decoding_modes": { |
| "A_full_chain_em_pct": 0.13636363636363635, |
| "B_pure_noise_1shot_em_pct": 0.19696969696969696, |
| "C_gold_corrupted_1shot_em_pct": 0.21717171717171718 |
| }, |
| "signal_degradation": { |
| "raw_xt_tok_acc_at_t2_pct": 100.0, |
| "model_tok_acc_at_t2_pct": 76.6, |
| "degradation_ppt": 23.4 |
| }, |
| "state_drift": { |
| "t2_oracle_tok_acc_pct": 75.1, |
| "t2_free_tok_acc_pct": 74.6, |
| "drift_ppt": 0.51, |
| "oracle_flat_across_noise": false |
| } |
| } |
| }, |
| "key_findings": [ |
| "Diffusion corruption is NOT the primary bottleneck at near-clean inputs. Raw-x_t achieves 100% token accuracy at t=2; the learned denoiser degrades this to 68-77%.", |
| "Scaling from 8M to 46.5M parameters reduces held-out exact match from 47% to 15%, despite 40x more training steps.", |
| "Phase 4 Original has 0.0% state drift (oracle = free-running) and is timestep-agnostic (flat 68.3% accuracy at every noise level).", |
| "Phase 4 Scheduled has ~0.5 ppt drift and one-shot decoding (B/C = 20-22%) outperforms full chain (A = 14%).", |
| "K=10 best-of-K never improves EM: only 1.08-1.26 unique candidates out of 10 (pairwise Jaccard > 0.97).", |
| "The failure is denoiser/model-behavior-limited, not diffusion-corruption-limited." |
| ] |
| } |
