Upload diagnostic_summary.json with huggingface_hub

diagnostic_summary.json

@@ -0,0 +1,161 @@
+{
+  "description": "Controlled scaling experiment: 8M vs 46.5M parameter Diffusion-LM riddle solvers",
+  "dataset": {
+    "name": "Riddles \u2014 A Synthetic Riddle Dataset for NLP",
+    "source": "Kaggle",
+    "url": "https://www.kaggle.com/datasets/prajwaldongre/riddles-a-synthetic-riddle-dataset-for-nlp",
+    "license": "CC0",
+    "train_size": 232,
+    "heldout_size": 66,
+    "unique_total": 298,
+    "duplicates_removed": 702,
+    "split_method": "SHA-256 of normalized riddle text",
+    "answer_max_len": 4,
+    "vocab_source": "training set only"
+  },
+  "checkpoints": {
+    "phase3": {
+      "label": "Phase 3 Reconstructed",
+      "architecture": {
+        "params": null,
+        "T": 200,
+        "d_model": 256,
+        "n_layers": 4
+      },
+      "training": {
+        "steps": 500,
+        "lr": 0.0003,
+        "schedule": "constant",
+        "batch": 64,
+        "seed": 42,
+        "T": 200
+      },
+      "eval_heldout": {
+        "exact_match_K1_pct": 46.96969696969697,
+        "exact_match_K10_pct": 46.96969696969697,
+        "token_f1_K1": 0.7124098124098122,
+        "n": 66
+      },
+      "eval_train": {
+        "exact_match_K1_pct": 87.5,
+        "exact_match_K10_pct": 86.45833333333333,
+        "token_f1_K1": 0.9597222222222221,
+        "n": 192
+      },
+      "decoding_modes": {
+        "A_full_chain_em_pct": 0.5,
+        "B_pure_noise_1shot_em_pct": 0.494949494949495,
+        "C_gold_corrupted_1shot_em_pct": 0.5303030303030303
+      },
+      "signal_degradation": {
+        "raw_xt_tok_acc_at_t2_pct": 100.0,
+        "model_tok_acc_at_t2_pct": 76.6,
+        "degradation_ppt": 23.4
+      },
+      "state_drift": {
+        "t2_oracle_tok_acc_pct": 75.9,
+        "t2_free_tok_acc_pct": 72.0,
+        "drift_ppt": 3.91,
+        "oracle_flat_across_noise": false
+      }
+    },
+    "phase4_original": {
+      "label": "Phase 4 Original",
+      "architecture": {
+        "params": null,
+        "T": 500,
+        "d_model": 512,
+        "n_layers": 6
+      },
+      "training": {
+        "steps": 20000,
+        "lr": 0.0003,
+        "schedule": "constant",
+        "batch": 64,
+        "seed": 42,
+        "T": 500
+      },
+      "eval_heldout": {
+        "exact_match_K1_pct": 15.151515151515152,
+        "exact_match_K10_pct": 15.151515151515152,
+        "token_f1_K1": 0.5692279942279943,
+        "n": 66
+      },
+      "eval_train": {
+        "exact_match_K1_pct": 51.5625,
+        "exact_match_K10_pct": 50.520833333333336,
+        "token_f1_K1": 0.9034226190476198,
+        "n": 192
+      },
+      "decoding_modes": {
+        "A_full_chain_em_pct": 0.15656565656565657,
+        "B_pure_noise_1shot_em_pct": 0.11616161616161616,
+        "C_gold_corrupted_1shot_em_pct": 0.13636363636363635
+      },
+      "signal_degradation": {
+        "raw_xt_tok_acc_at_t2_pct": 100.0,
+        "model_tok_acc_at_t2_pct": 68.3,
+        "degradation_ppt": 31.7
+      },
+      "state_drift": {
+        "t2_oracle_tok_acc_pct": 68.3,
+        "t2_free_tok_acc_pct": 68.3,
+        "drift_ppt": 0.0,
+        "oracle_flat_across_noise": true
+      }
+    },
+    "phase4_scheduled": {
+      "label": "Phase 4 Scheduled",
+      "architecture": {
+        "params": null,
+        "T": 500,
+        "d_model": 512,
+        "n_layers": 6
+      },
+      "training": {
+        "steps": 20000,
+        "lr": 0.0001,
+        "schedule": "warmup+cosine",
+        "batch": 64,
+        "seed": 42,
+        "T": 500
+      },
+      "eval_heldout": {
+        "exact_match_K1_pct": 13.636363636363637,
+        "exact_match_K10_pct": 13.636363636363637,
+        "token_f1_K1": 0.6034992784992784,
+        "n": 66
+      },
+      "eval_train": {
+        "exact_match_K1_pct": 40.625,
+        "exact_match_K10_pct": 39.583333333333336,
+        "token_f1_K1": 0.8827380952380958,
+        "n": 192
+      },
+      "decoding_modes": {
+        "A_full_chain_em_pct": 0.13636363636363635,
+        "B_pure_noise_1shot_em_pct": 0.19696969696969696,
+        "C_gold_corrupted_1shot_em_pct": 0.21717171717171718
+      },
+      "signal_degradation": {
+        "raw_xt_tok_acc_at_t2_pct": 100.0,
+        "model_tok_acc_at_t2_pct": 76.6,
+        "degradation_ppt": 23.4
+      },
+      "state_drift": {
+        "t2_oracle_tok_acc_pct": 75.1,
+        "t2_free_tok_acc_pct": 74.6,
+        "drift_ppt": 0.51,
+        "oracle_flat_across_noise": false
+      }
+    }
+  },
+  "key_findings": [
+    "Diffusion corruption is NOT the primary bottleneck at near-clean inputs. Raw-x_t achieves 100% token accuracy at t=2; the learned denoiser degrades this to 68-77%.",
+    "Scaling from 8M to 46.5M parameters reduces held-out exact match from 47% to 15%, despite 40x more training steps.",
+    "Phase 4 Original has 0.0% state drift (oracle = free-running) and is timestep-agnostic (flat 68.3% accuracy at every noise level).",
+    "Phase 4 Scheduled has ~0.5 ppt drift and one-shot decoding (B/C = 20-22%) outperforms full chain (A = 14%).",
+    "K=10 best-of-K never improves EM: only 1.08-1.26 unique candidates out of 10 (pairwise Jaccard > 0.97).",
+    "The failure is denoiser/model-behavior-limited, not diffusion-corruption-limited."
+  ]
+}