{
  "description": "Controlled scaling experiment: 8M vs 46.5M parameter Diffusion-LM riddle solvers",
  "dataset": {
    "name": "Riddles \u2014 A Synthetic Riddle Dataset for NLP",
    "source": "Kaggle",
    "url": "https://www.kaggle.com/datasets/prajwaldongre/riddles-a-synthetic-riddle-dataset-for-nlp",
    "license": "CC0",
    "train_size": 232,
    "heldout_size": 66,
    "unique_total": 298,
    "duplicates_removed": 702,
    "split_method": "SHA-256 of normalized riddle text",
    "answer_max_len": 4,
    "vocab_source": "training set only"
  },
  "checkpoints": {
    "phase3": {
      "label": "Phase 3 Reconstructed",
      "architecture": {
        "params": null,
        "T": 200,
        "d_model": 256,
        "n_layers": 4
      },
      "training": {
        "steps": 500,
        "lr": 0.0003,
        "schedule": "constant",
        "batch": 64,
        "seed": 42,
        "T": 200
      },
      "eval_heldout": {
        "exact_match_K1_pct": 46.96969696969697,
        "exact_match_K10_pct": 46.96969696969697,
        "token_f1_K1": 0.7124098124098122,
        "n": 66
      },
      "eval_train": {
        "exact_match_K1_pct": 87.5,
        "exact_match_K10_pct": 86.45833333333333,
        "token_f1_K1": 0.9597222222222221,
        "n": 192
      },
      "decoding_modes": {
        "A_full_chain_em_pct": 0.5,
        "B_pure_noise_1shot_em_pct": 0.494949494949495,
        "C_gold_corrupted_1shot_em_pct": 0.5303030303030303
      },
      "signal_degradation": {
        "raw_xt_tok_acc_at_t2_pct": 100.0,
        "model_tok_acc_at_t2_pct": 76.6,
        "degradation_ppt": 23.4
      },
      "state_drift": {
        "t2_oracle_tok_acc_pct": 75.9,
        "t2_free_tok_acc_pct": 72.0,
        "drift_ppt": 3.91,
        "oracle_flat_across_noise": false
      }
    },
    "phase4_original": {
      "label": "Phase 4 Original",
      "architecture": {
        "params": null,
        "T": 500,
        "d_model": 512,
        "n_layers": 6
      },
      "training": {
        "steps": 20000,
        "lr": 0.0003,
        "schedule": "constant",
        "batch": 64,
        "seed": 42,
        "T": 500
      },
      "eval_heldout": {
        "exact_match_K1_pct": 15.151515151515152,
        "exact_match_K10_pct": 15.151515151515152,
        "token_f1_K1": 0.5692279942279943,
        "n": 66
      },
      "eval_train": {
        "exact_match_K1_pct": 51.5625,
        "exact_match_K10_pct": 50.520833333333336,
        "token_f1_K1": 0.9034226190476198,
        "n": 192
      },
      "decoding_modes": {
        "A_full_chain_em_pct": 0.15656565656565657,
        "B_pure_noise_1shot_em_pct": 0.11616161616161616,
        "C_gold_corrupted_1shot_em_pct": 0.13636363636363635
      },
      "signal_degradation": {
        "raw_xt_tok_acc_at_t2_pct": 100.0,
        "model_tok_acc_at_t2_pct": 68.3,
        "degradation_ppt": 31.7
      },
      "state_drift": {
        "t2_oracle_tok_acc_pct": 68.3,
        "t2_free_tok_acc_pct": 68.3,
        "drift_ppt": 0.0,
        "oracle_flat_across_noise": true
      }
    },
    "phase4_scheduled": {
      "label": "Phase 4 Scheduled",
      "architecture": {
        "params": null,
        "T": 500,
        "d_model": 512,
        "n_layers": 6
      },
      "training": {
        "steps": 20000,
        "lr": 0.0001,
        "schedule": "warmup+cosine",
        "batch": 64,
        "seed": 42,
        "T": 500
      },
      "eval_heldout": {
        "exact_match_K1_pct": 13.636363636363637,
        "exact_match_K10_pct": 13.636363636363637,
        "token_f1_K1": 0.6034992784992784,
        "n": 66
      },
      "eval_train": {
        "exact_match_K1_pct": 40.625,
        "exact_match_K10_pct": 39.583333333333336,
        "token_f1_K1": 0.8827380952380958,
        "n": 192
      },
      "decoding_modes": {
        "A_full_chain_em_pct": 0.13636363636363635,
        "B_pure_noise_1shot_em_pct": 0.19696969696969696,
        "C_gold_corrupted_1shot_em_pct": 0.21717171717171718
      },
      "signal_degradation": {
        "raw_xt_tok_acc_at_t2_pct": 100.0,
        "model_tok_acc_at_t2_pct": 76.6,
        "degradation_ppt": 23.4
      },
      "state_drift": {
        "t2_oracle_tok_acc_pct": 75.1,
        "t2_free_tok_acc_pct": 74.6,
        "drift_ppt": 0.51,
        "oracle_flat_across_noise": false
      }
    }
  },
  "key_findings": [
    "Diffusion corruption is NOT the primary bottleneck at near-clean inputs. Raw-x_t achieves 100% token accuracy at t=2; the learned denoiser degrades this to 68-77%.",
    "Scaling from 8M to 46.5M parameters reduces held-out exact match from 47% to 15%, despite 40x more training steps.",
    "Phase 4 Original has 0.0% state drift (oracle = free-running) and is timestep-agnostic (flat 68.3% accuracy at every noise level).",
    "Phase 4 Scheduled has ~0.5 ppt drift and one-shot decoding (B/C = 20-22%) outperforms full chain (A = 14%).",
    "K=10 best-of-K never improves EM: only 1.08-1.26 unique candidates out of 10 (pairwise Jaccard > 0.97).",
    "The failure is denoiser/model-behavior-limited, not diffusion-corruption-limited."
  ]
}