| window.DASH = { |
| "generated_utc": "2026-07-20 21:49 UTC", |
| "provenance": "suite numbers recomputed from data/**.jsonl at build time; NESTFUL IBM-scorer metrics + frontier context transcribed from run logs (EXPERIMENT_LOG.md)", |
| "countdown": { |
| "qwen": { |
| "baseline": { |
| "file": "data/countdown/qwen_unconstrained_n256.jsonl", |
| "acc": 0.8047, |
| "n": 256 |
| }, |
| "mask": { |
| "file": "data/countdown/qwen_mask_n256.jsonl", |
| "acc": 0.832, |
| "n": 256, |
| "flips": { |
| "lost": 0, |
| "gained": 7, |
| "n": 256 |
| } |
| }, |
| "mask_dfa": { |
| "file": "data/countdown/qwen_maskdfa_n256.jsonl", |
| "acc": 0.832, |
| "n": 256, |
| "flips": { |
| "lost": 0, |
| "gained": 7, |
| "n": 256 |
| } |
| }, |
| "hmm_w02": { |
| "file": "data/countdown/qwen_hmm_w02_n256.jsonl", |
| "acc": 0.8164, |
| "n": 256, |
| "flips": { |
| "lost": 4, |
| "gained": 7, |
| "n": 256 |
| } |
| }, |
| "mask_bon": { |
| "file": "data/countdown/qwen_maskbon_n256.jsonl", |
| "acc": 0.8359, |
| "n": 256, |
| "flips": { |
| "lost": 0, |
| "gained": 8, |
| "n": 256 |
| } |
| }, |
| "free_bon4": { |
| "file": "data/countdown/qwen_bon4_n256.jsonl", |
| "acc": 0.832, |
| "n": 256, |
| "flips": { |
| "lost": 1, |
| "gained": 8, |
| "n": 256 |
| } |
| } |
| }, |
| "llama": { |
| "baseline": { |
| "file": "data/countdown/llama_unconstrained_n256.jsonl", |
| "acc": 0.1328, |
| "n": 256 |
| }, |
| "mask": { |
| "file": "data/countdown/llama_mask_n256.jsonl", |
| "acc": 0.1758, |
| "n": 256, |
| "flips": { |
| "lost": 0, |
| "gained": 11, |
| "n": 256 |
| } |
| } |
| }, |
| "frontier": { |
| "claude-opus-4-7": { |
| "acc": 0.9688, |
| "n": 256 |
| }, |
| "claude-opus-4-8": { |
| "acc": 0.9219, |
| "n": 256 |
| }, |
| "claude-sonnet-5": { |
| "acc": 0.9453, |
| "n": 256 |
| }, |
| "gpt-5_6-luna": { |
| "acc": 0.9414, |
| "n": 256 |
| }, |
| "gpt-5_6-sol": { |
| "acc": 0.9805, |
| "n": 256 |
| }, |
| "gpt-5_6-terra": { |
| "acc": 0.9609, |
| "n": 256 |
| } |
| } |
| }, |
| "nested_arith": { |
| "qwen": { |
| "baseline": { |
| "file": "data/nested_arith/qwen_unconstrained_n384.jsonl", |
| "acc": 0.3958, |
| "n": 384, |
| "chain": { |
| "acc": 0.3958, |
| "n": 384, |
| "chain_valid": 0.3073, |
| "faithful": 0.2995 |
| } |
| }, |
| "mask": { |
| "file": "data/nested_arith/qwen_mask_n384.jsonl", |
| "acc": 0.3828, |
| "n": 384, |
| "flips": { |
| "lost": 7, |
| "gained": 2, |
| "n": 384 |
| }, |
| "chain": { |
| "acc": 0.3828, |
| "n": 384, |
| "chain_valid": 0.4323, |
| "faithful": 0.3802 |
| } |
| }, |
| "mask_dfa": { |
| "file": "data/nested_arith/qwen_mask_dfa_n384.jsonl", |
| "acc": 0.3177, |
| "n": 384, |
| "flips": { |
| "lost": 52, |
| "gained": 22, |
| "n": 384 |
| } |
| }, |
| "hmm_w05": { |
| "file": "data/nested_arith/qwen_hmm_h256_w05_n384.jsonl", |
| "acc": 0.151, |
| "n": 384, |
| "flips": { |
| "lost": 108, |
| "gained": 14, |
| "n": 384 |
| }, |
| "chain": { |
| "acc": 0.151, |
| "n": 384, |
| "chain_valid": 0.9844, |
| "faithful": 0.151 |
| } |
| } |
| }, |
| "llama": { |
| "baseline": { |
| "file": "data/nested_arith/llama_unconstrained_n384.jsonl", |
| "acc": 0.1953, |
| "n": 384, |
| "chain": { |
| "acc": 0.1953, |
| "n": 384, |
| "chain_valid": 0.5365, |
| "faithful": 0.1901 |
| } |
| }, |
| "mask": { |
| "file": "data/nested_arith/llama_mask_n384.jsonl", |
| "acc": 0.2005, |
| "n": 384, |
| "flips": { |
| "lost": 1, |
| "gained": 3, |
| "n": 384 |
| }, |
| "chain": { |
| "acc": 0.2005, |
| "n": 384, |
| "chain_valid": 0.7448, |
| "faithful": 0.2005 |
| } |
| } |
| }, |
| "lambda_grid_train": { |
| "02": { |
| "acc": 0.3203, |
| "n": 128, |
| "chain_valid": 0.9297, |
| "faithful": 0.3203 |
| }, |
| "05": { |
| "acc": 0.3359, |
| "n": 128, |
| "chain_valid": 0.9844, |
| "faithful": 0.3359 |
| }, |
| "10": { |
| "acc": 0.0, |
| "n": 128, |
| "chain_valid": 1.0, |
| "faithful": 0.0 |
| } |
| }, |
| "frontier": { |
| "claude-opus-4-7": { |
| "acc": 0.7448, |
| "n": 384 |
| }, |
| "claude-opus-4-8": { |
| "acc": 0.7578, |
| "n": 384 |
| }, |
| "claude-sonnet-5": { |
| "acc": 0.5885, |
| "n": 384 |
| }, |
| "gpt-5_6-luna": { |
| "acc": 0.7786, |
| "n": 384 |
| }, |
| "gpt-5_6-sol": { |
| "acc": 0.9219, |
| "n": 384 |
| }, |
| "gpt-5_6-terra": { |
| "acc": 0.8724, |
| "n": 384 |
| } |
| } |
| }, |
| "planetarium": { |
| "qwen": { |
| "baseline": { |
| "file": "data/planetarium/qwen_unconstrained_n256.jsonl", |
| "acc": 0.3346, |
| "n": 254 |
| }, |
| "mask": { |
| "file": "data/planetarium/qwen_mask_n256.jsonl", |
| "acc": 0.332, |
| "n": 256, |
| "flips": { |
| "lost": 0, |
| "gained": 0, |
| "n": 254 |
| } |
| }, |
| "mask_dfa": { |
| "file": "data/planetarium/qwen_mask_dfa_n256.jsonl", |
| "acc": 0.332, |
| "n": 256, |
| "flips": { |
| "lost": 0, |
| "gained": 0, |
| "n": 254 |
| } |
| } |
| }, |
| "llama": { |
| "baseline": { |
| "file": "data/planetarium/llama_unconstrained_n256.jsonl", |
| "acc": 0.0117, |
| "n": 256 |
| }, |
| "mask": { |
| "file": "data/planetarium/llama_mask_n256.jsonl", |
| "acc": 0.0156, |
| "n": 256, |
| "flips": { |
| "lost": 0, |
| "gained": 1, |
| "n": 256 |
| } |
| } |
| }, |
| "frontier": { |
| "claude-opus-4-7": { |
| "acc": 0.6914, |
| "n": 256 |
| }, |
| "claude-opus-4-8": { |
| "acc": 0.707, |
| "n": 256 |
| }, |
| "claude-sonnet-5": { |
| "acc": 0.7617, |
| "n": 256 |
| }, |
| "gpt-5_6-luna": { |
| "acc": 0.6523, |
| "n": 256 |
| }, |
| "gpt-5_6-sol": { |
| "acc": 0.7305, |
| "n": 256 |
| }, |
| "gpt-5_6-terra": { |
| "acc": 0.6523, |
| "n": 256 |
| } |
| } |
| }, |
| "racket": { |
| "qwen": { |
| "baseline": { |
| "file": "data/racket/qwen_unconstrained_n161.jsonl", |
| "acc": 0.2857, |
| "n": 161 |
| }, |
| "mask": { |
| "file": "data/racket/qwen_mask_n161.jsonl", |
| "acc": 0.3602, |
| "n": 161, |
| "flips": { |
| "lost": 0, |
| "gained": 12, |
| "n": 161 |
| } |
| } |
| }, |
| "llama": { |
| "baseline": { |
| "file": "data/racket/llama_unconstrained_n161.jsonl", |
| "acc": 0.1925, |
| "n": 161 |
| }, |
| "mask": { |
| "file": "data/racket/llama_mask_n161.jsonl", |
| "acc": 0.1988, |
| "n": 161, |
| "flips": { |
| "lost": 2, |
| "gained": 3, |
| "n": 161 |
| } |
| } |
| }, |
| "frontier": { |
| "claude-opus-4-7": { |
| "acc": 0.913, |
| "n": 161 |
| }, |
| "claude-opus-4-8": { |
| "acc": 0.9379, |
| "n": 161 |
| }, |
| "claude-sonnet-5": { |
| "acc": 0.8447, |
| "n": 161 |
| }, |
| "gpt-5_6-luna": { |
| "acc": 0.5901, |
| "n": 161 |
| }, |
| "gpt-5_6-sol": { |
| "acc": 0.913, |
| "n": 161 |
| }, |
| "gpt-5_6-terra": { |
| "acc": 0.4348, |
| "n": 161 |
| } |
| } |
| }, |
| "bbeh_dyck": { |
| "qwen": { |
| "baseline": { |
| "file": "data/bbeh_dyck/qwen_unconstrained_n200.jsonl", |
| "acc": 0.205, |
| "n": 200 |
| }, |
| "mask": { |
| "file": "data/bbeh_dyck/qwen_mask_n200.jsonl", |
| "acc": 0.3, |
| "n": 200, |
| "flips": { |
| "lost": 4, |
| "gained": 23, |
| "n": 200 |
| } |
| } |
| }, |
| "llama": { |
| "baseline": { |
| "file": "data/bbeh_dyck/llama_unconstrained_n200.jsonl", |
| "acc": 0.025, |
| "n": 200 |
| }, |
| "mask": { |
| "file": "data/bbeh_dyck/llama_mask_n200.jsonl", |
| "acc": 0.005, |
| "n": 200, |
| "flips": { |
| "lost": 4, |
| "gained": 0, |
| "n": 200 |
| } |
| } |
| }, |
| "frontier": { |
| "claude-opus-4-7": { |
| "acc": 0.7, |
| "n": 200 |
| }, |
| "claude-opus-4-8": { |
| "acc": 0.345, |
| "n": 200 |
| }, |
| "claude-sonnet-5": { |
| "acc": 0.225, |
| "n": 200 |
| }, |
| "gpt-5_6-luna": { |
| "acc": 0.555, |
| "n": 200 |
| }, |
| "gpt-5_6-sol": { |
| "acc": 0.84, |
| "n": 200 |
| }, |
| "gpt-5_6-terra": { |
| "acc": 0.725, |
| "n": 200 |
| } |
| } |
| }, |
| "structural": { |
| "countdown/qwen": { |
| "n": 256, |
| "fails": 50, |
| "parse_reject": 17, |
| "incomplete": 19, |
| "semantic": 14 |
| }, |
| "countdown/llama": { |
| "n": 256, |
| "fails": 222, |
| "parse_reject": 64, |
| "incomplete": 148, |
| "semantic": 10 |
| }, |
| "nested_arith/qwen": { |
| "n": 384, |
| "fails": 232, |
| "parse_reject": 167, |
| "incomplete": 62, |
| "semantic": 3 |
| }, |
| "nested_arith/llama": { |
| "n": 384, |
| "fails": 309, |
| "parse_reject": 178, |
| "incomplete": 40, |
| "semantic": 91 |
| }, |
| "planetarium/qwen": { |
| "n": 254, |
| "fails": 169, |
| "parse_reject": 2, |
| "incomplete": 2, |
| "semantic": 165 |
| }, |
| "planetarium/llama": { |
| "n": 256, |
| "fails": 253, |
| "parse_reject": 2, |
| "incomplete": 69, |
| "semantic": 182 |
| } |
| }, |
| "dyck_kl": { |
| "config": { |
| "quick": false, |
| "hs": [ |
| 2, |
| 4, |
| 8, |
| 16, |
| 32 |
| ], |
| "n_samples": 100000, |
| "em_iters": 100, |
| "restarts": 2, |
| "smooth": 0.001, |
| "lm_seed": 7, |
| "sample_seed": 1234, |
| "em_seed": 1000 |
| }, |
| "language": { |
| "depth": 3, |
| "max_len_incl_eos": 12, |
| "n_accepted": 1347, |
| "nwa_states": 48, |
| "lm_mass_on_accepted": 0.023153314044920457 |
| }, |
| "distillation": { |
| "n_samples": 100000, |
| "n_unique": 33198, |
| "n_forced_eos": 17791, |
| "per_h": { |
| "2": { |
| "avg_loglik": -13.994782996264572, |
| "em_iters_used": 100, |
| "distill_kl_mc": 4.078227244013312 |
| }, |
| "4": { |
| "avg_loglik": -13.15911380651663, |
| "em_iters_used": 100, |
| "distill_kl_mc": 3.2405201988037593 |
| }, |
| "8": { |
| "avg_loglik": -12.366249485576574, |
| "em_iters_used": 100, |
| "distill_kl_mc": 2.4476052910339057 |
| }, |
| "16": { |
| "avg_loglik": -11.92665970131934, |
| "em_iters_used": 100, |
| "distill_kl_mc": 2.0088123023190465 |
| }, |
| "32": { |
| "avg_loglik": -11.578717748637416, |
| "em_iters_used": 100, |
| "distill_kl_mc": 1.6594528339427308 |
| } |
| } |
| }, |
| "table": [ |
| { |
| "policy": "mask-only", |
| "h": null, |
| "kl_fwd": 0.5310182411550299, |
| "kl_rev": 0.5516616319620444, |
| "sum_p": 1.0, |
| "guide_mc_kl": null |
| }, |
| { |
| "policy": "ctrlg", |
| "h": 2, |
| "kl_fwd": 1.9992174110894245, |
| "kl_rev": 1.0392534771829336, |
| "sum_p": 1.0000000000000004, |
| "guide_mc_kl": 4.078227244013312 |
| }, |
| { |
| "policy": "ctrlg", |
| "h": 4, |
| "kl_fwd": 1.7284096916719445, |
| "kl_rev": 1.0090787228214624, |
| "sum_p": 0.9999999999999998, |
| "guide_mc_kl": 3.2405201988037593 |
| }, |
| { |
| "policy": "ctrlg", |
| "h": 8, |
| "kl_fwd": 1.1733715958351294, |
| "kl_rev": 0.6541745569739358, |
| "sum_p": 0.9999999999999996, |
| "guide_mc_kl": 2.4476052910339057 |
| }, |
| { |
| "policy": "ctrlg", |
| "h": 16, |
| "kl_fwd": 0.8072083795865748, |
| "kl_rev": 0.47696071923941025, |
| "sum_p": 0.9999999999999997, |
| "guide_mc_kl": 2.0088123023190465 |
| }, |
| { |
| "policy": "ctrlg", |
| "h": 32, |
| "kl_fwd": 0.4811768698605567, |
| "kl_rev": 0.33703712896819793, |
| "sum_p": 1.0000000000000004, |
| "guide_mc_kl": 1.6594528339427308 |
| }, |
| { |
| "policy": "oracle", |
| "h": null, |
| "kl_fwd": -4.595798209410908e-16, |
| "kl_rev": 4.595798209410923e-16, |
| "sum_p": 1.0000000000000007, |
| "guide_mc_kl": null |
| } |
| ], |
| "runtime_s": 652.6919736862183 |
| }, |
| "nestful": { |
| "provenance": "IBM vendored scorer output, test-200 (train-100 for the grid); job logs 2026-07-16..19, see EXPERIMENT_LOG.md", |
| "test200": { |
| "unconstrained": { |
| "fsm": 0.145, |
| "partial": 0.183, |
| "f1_func": 0.703, |
| "parse_err": 20, |
| "valid": null |
| }, |
| "mask_only": { |
| "fsm": 0.235, |
| "partial": 0.251, |
| "f1_func": null, |
| "parse_err": 1, |
| "valid": "~178-196/200" |
| }, |
| "ctrlg_old_guide": { |
| "fsm": 0.21, |
| "partial": 0.271, |
| "f1_func": 0.931, |
| "parse_err": 6, |
| "valid": "194/200" |
| }, |
| "ctrlg_walker_l1": { |
| "fsm": 0.125, |
| "partial": 0.211, |
| "f1_func": 0.745, |
| "parse_err": 0, |
| "valid": "200/200" |
| }, |
| "ctrlg_walker_l01": { |
| "fsm": 0.23, |
| "partial": 0.286, |
| "f1_func": null, |
| "parse_err": 0, |
| "valid": "200/200" |
| } |
| }, |
| "lambda_grid_train100": { |
| "mask_only": { |
| "fsm": 0.22, |
| "partial": 0.271, |
| "valid": 99 |
| }, |
| "0.1": { |
| "fsm": 0.2, |
| "partial": 0.266, |
| "valid": 99 |
| }, |
| "0.2": { |
| "fsm": 0.2, |
| "partial": 0.264, |
| "valid": 99 |
| }, |
| "0.5": { |
| "fsm": 0.17, |
| "partial": 0.24, |
| "valid": 100 |
| }, |
| "1.0": { |
| "fsm": 0.125, |
| "partial": 0.211, |
| "valid": 100 |
| } |
| }, |
| "frontier_no_reasoning": { |
| "claude-opus-4-7": 0.36, |
| "claude-sonnet-5": 0.35, |
| "claude-opus-4-8": 0.33, |
| "gpt-5.6-sol": 0.315, |
| "gpt-5.6-terra": 0.3, |
| "gpt-5.6-luna": 0.255 |
| } |
| }, |
| "live": { |
| "name": "nested_arith test-384 \u00b7 full Ctrl-G h256 \u03bb=0.5", |
| "done": 384, |
| "total": 384, |
| "acc": 0.151 |
| } |
| }; |
|
|
