window.DASH = {
 "generated_utc": "2026-07-20 21:49 UTC",
 "provenance": "suite numbers recomputed from data/**.jsonl at build time; NESTFUL IBM-scorer metrics + frontier context transcribed from run logs (EXPERIMENT_LOG.md)",
 "countdown": {
  "qwen": {
   "baseline": {
    "file": "data/countdown/qwen_unconstrained_n256.jsonl",
    "acc": 0.8047,
    "n": 256
   },
   "mask": {
    "file": "data/countdown/qwen_mask_n256.jsonl",
    "acc": 0.832,
    "n": 256,
    "flips": {
     "lost": 0,
     "gained": 7,
     "n": 256
    }
   },
   "mask_dfa": {
    "file": "data/countdown/qwen_maskdfa_n256.jsonl",
    "acc": 0.832,
    "n": 256,
    "flips": {
     "lost": 0,
     "gained": 7,
     "n": 256
    }
   },
   "hmm_w02": {
    "file": "data/countdown/qwen_hmm_w02_n256.jsonl",
    "acc": 0.8164,
    "n": 256,
    "flips": {
     "lost": 4,
     "gained": 7,
     "n": 256
    }
   },
   "mask_bon": {
    "file": "data/countdown/qwen_maskbon_n256.jsonl",
    "acc": 0.8359,
    "n": 256,
    "flips": {
     "lost": 0,
     "gained": 8,
     "n": 256
    }
   },
   "free_bon4": {
    "file": "data/countdown/qwen_bon4_n256.jsonl",
    "acc": 0.832,
    "n": 256,
    "flips": {
     "lost": 1,
     "gained": 8,
     "n": 256
    }
   }
  },
  "llama": {
   "baseline": {
    "file": "data/countdown/llama_unconstrained_n256.jsonl",
    "acc": 0.1328,
    "n": 256
   },
   "mask": {
    "file": "data/countdown/llama_mask_n256.jsonl",
    "acc": 0.1758,
    "n": 256,
    "flips": {
     "lost": 0,
     "gained": 11,
     "n": 256
    }
   }
  },
  "frontier": {
   "claude-opus-4-7": {
    "acc": 0.9688,
    "n": 256
   },
   "claude-opus-4-8": {
    "acc": 0.9219,
    "n": 256
   },
   "claude-sonnet-5": {
    "acc": 0.9453,
    "n": 256
   },
   "gpt-5_6-luna": {
    "acc": 0.9414,
    "n": 256
   },
   "gpt-5_6-sol": {
    "acc": 0.9805,
    "n": 256
   },
   "gpt-5_6-terra": {
    "acc": 0.9609,
    "n": 256
   }
  }
 },
 "nested_arith": {
  "qwen": {
   "baseline": {
    "file": "data/nested_arith/qwen_unconstrained_n384.jsonl",
    "acc": 0.3958,
    "n": 384,
    "chain": {
     "acc": 0.3958,
     "n": 384,
     "chain_valid": 0.3073,
     "faithful": 0.2995
    }
   },
   "mask": {
    "file": "data/nested_arith/qwen_mask_n384.jsonl",
    "acc": 0.3828,
    "n": 384,
    "flips": {
     "lost": 7,
     "gained": 2,
     "n": 384
    },
    "chain": {
     "acc": 0.3828,
     "n": 384,
     "chain_valid": 0.4323,
     "faithful": 0.3802
    }
   },
   "mask_dfa": {
    "file": "data/nested_arith/qwen_mask_dfa_n384.jsonl",
    "acc": 0.3177,
    "n": 384,
    "flips": {
     "lost": 52,
     "gained": 22,
     "n": 384
    }
   },
   "hmm_w05": {
    "file": "data/nested_arith/qwen_hmm_h256_w05_n384.jsonl",
    "acc": 0.151,
    "n": 384,
    "flips": {
     "lost": 108,
     "gained": 14,
     "n": 384
    },
    "chain": {
     "acc": 0.151,
     "n": 384,
     "chain_valid": 0.9844,
     "faithful": 0.151
    }
   }
  },
  "llama": {
   "baseline": {
    "file": "data/nested_arith/llama_unconstrained_n384.jsonl",
    "acc": 0.1953,
    "n": 384,
    "chain": {
     "acc": 0.1953,
     "n": 384,
     "chain_valid": 0.5365,
     "faithful": 0.1901
    }
   },
   "mask": {
    "file": "data/nested_arith/llama_mask_n384.jsonl",
    "acc": 0.2005,
    "n": 384,
    "flips": {
     "lost": 1,
     "gained": 3,
     "n": 384
    },
    "chain": {
     "acc": 0.2005,
     "n": 384,
     "chain_valid": 0.7448,
     "faithful": 0.2005
    }
   }
  },
  "lambda_grid_train": {
   "02": {
    "acc": 0.3203,
    "n": 128,
    "chain_valid": 0.9297,
    "faithful": 0.3203
   },
   "05": {
    "acc": 0.3359,
    "n": 128,
    "chain_valid": 0.9844,
    "faithful": 0.3359
   },
   "10": {
    "acc": 0.0,
    "n": 128,
    "chain_valid": 1.0,
    "faithful": 0.0
   }
  },
  "frontier": {
   "claude-opus-4-7": {
    "acc": 0.7448,
    "n": 384
   },
   "claude-opus-4-8": {
    "acc": 0.7578,
    "n": 384
   },
   "claude-sonnet-5": {
    "acc": 0.5885,
    "n": 384
   },
   "gpt-5_6-luna": {
    "acc": 0.7786,
    "n": 384
   },
   "gpt-5_6-sol": {
    "acc": 0.9219,
    "n": 384
   },
   "gpt-5_6-terra": {
    "acc": 0.8724,
    "n": 384
   }
  }
 },
 "planetarium": {
  "qwen": {
   "baseline": {
    "file": "data/planetarium/qwen_unconstrained_n256.jsonl",
    "acc": 0.3346,
    "n": 254
   },
   "mask": {
    "file": "data/planetarium/qwen_mask_n256.jsonl",
    "acc": 0.332,
    "n": 256,
    "flips": {
     "lost": 0,
     "gained": 0,
     "n": 254
    }
   },
   "mask_dfa": {
    "file": "data/planetarium/qwen_mask_dfa_n256.jsonl",
    "acc": 0.332,
    "n": 256,
    "flips": {
     "lost": 0,
     "gained": 0,
     "n": 254
    }
   }
  },
  "llama": {
   "baseline": {
    "file": "data/planetarium/llama_unconstrained_n256.jsonl",
    "acc": 0.0117,
    "n": 256
   },
   "mask": {
    "file": "data/planetarium/llama_mask_n256.jsonl",
    "acc": 0.0156,
    "n": 256,
    "flips": {
     "lost": 0,
     "gained": 1,
     "n": 256
    }
   }
  },
  "frontier": {
   "claude-opus-4-7": {
    "acc": 0.6914,
    "n": 256
   },
   "claude-opus-4-8": {
    "acc": 0.707,
    "n": 256
   },
   "claude-sonnet-5": {
    "acc": 0.7617,
    "n": 256
   },
   "gpt-5_6-luna": {
    "acc": 0.6523,
    "n": 256
   },
   "gpt-5_6-sol": {
    "acc": 0.7305,
    "n": 256
   },
   "gpt-5_6-terra": {
    "acc": 0.6523,
    "n": 256
   }
  }
 },
 "racket": {
  "qwen": {
   "baseline": {
    "file": "data/racket/qwen_unconstrained_n161.jsonl",
    "acc": 0.2857,
    "n": 161
   },
   "mask": {
    "file": "data/racket/qwen_mask_n161.jsonl",
    "acc": 0.3602,
    "n": 161,
    "flips": {
     "lost": 0,
     "gained": 12,
     "n": 161
    }
   }
  },
  "llama": {
   "baseline": {
    "file": "data/racket/llama_unconstrained_n161.jsonl",
    "acc": 0.1925,
    "n": 161
   },
   "mask": {
    "file": "data/racket/llama_mask_n161.jsonl",
    "acc": 0.1988,
    "n": 161,
    "flips": {
     "lost": 2,
     "gained": 3,
     "n": 161
    }
   }
  },
  "frontier": {
   "claude-opus-4-7": {
    "acc": 0.913,
    "n": 161
   },
   "claude-opus-4-8": {
    "acc": 0.9379,
    "n": 161
   },
   "claude-sonnet-5": {
    "acc": 0.8447,
    "n": 161
   },
   "gpt-5_6-luna": {
    "acc": 0.5901,
    "n": 161
   },
   "gpt-5_6-sol": {
    "acc": 0.913,
    "n": 161
   },
   "gpt-5_6-terra": {
    "acc": 0.4348,
    "n": 161
   }
  }
 },
 "bbeh_dyck": {
  "qwen": {
   "baseline": {
    "file": "data/bbeh_dyck/qwen_unconstrained_n200.jsonl",
    "acc": 0.205,
    "n": 200
   },
   "mask": {
    "file": "data/bbeh_dyck/qwen_mask_n200.jsonl",
    "acc": 0.3,
    "n": 200,
    "flips": {
     "lost": 4,
     "gained": 23,
     "n": 200
    }
   }
  },
  "llama": {
   "baseline": {
    "file": "data/bbeh_dyck/llama_unconstrained_n200.jsonl",
    "acc": 0.025,
    "n": 200
   },
   "mask": {
    "file": "data/bbeh_dyck/llama_mask_n200.jsonl",
    "acc": 0.005,
    "n": 200,
    "flips": {
     "lost": 4,
     "gained": 0,
     "n": 200
    }
   }
  },
  "frontier": {
   "claude-opus-4-7": {
    "acc": 0.7,
    "n": 200
   },
   "claude-opus-4-8": {
    "acc": 0.345,
    "n": 200
   },
   "claude-sonnet-5": {
    "acc": 0.225,
    "n": 200
   },
   "gpt-5_6-luna": {
    "acc": 0.555,
    "n": 200
   },
   "gpt-5_6-sol": {
    "acc": 0.84,
    "n": 200
   },
   "gpt-5_6-terra": {
    "acc": 0.725,
    "n": 200
   }
  }
 },
 "structural": {
  "countdown/qwen": {
   "n": 256,
   "fails": 50,
   "parse_reject": 17,
   "incomplete": 19,
   "semantic": 14
  },
  "countdown/llama": {
   "n": 256,
   "fails": 222,
   "parse_reject": 64,
   "incomplete": 148,
   "semantic": 10
  },
  "nested_arith/qwen": {
   "n": 384,
   "fails": 232,
   "parse_reject": 167,
   "incomplete": 62,
   "semantic": 3
  },
  "nested_arith/llama": {
   "n": 384,
   "fails": 309,
   "parse_reject": 178,
   "incomplete": 40,
   "semantic": 91
  },
  "planetarium/qwen": {
   "n": 254,
   "fails": 169,
   "parse_reject": 2,
   "incomplete": 2,
   "semantic": 165
  },
  "planetarium/llama": {
   "n": 256,
   "fails": 253,
   "parse_reject": 2,
   "incomplete": 69,
   "semantic": 182
  }
 },
 "dyck_kl": {
  "config": {
   "quick": false,
   "hs": [
    2,
    4,
    8,
    16,
    32
   ],
   "n_samples": 100000,
   "em_iters": 100,
   "restarts": 2,
   "smooth": 0.001,
   "lm_seed": 7,
   "sample_seed": 1234,
   "em_seed": 1000
  },
  "language": {
   "depth": 3,
   "max_len_incl_eos": 12,
   "n_accepted": 1347,
   "nwa_states": 48,
   "lm_mass_on_accepted": 0.023153314044920457
  },
  "distillation": {
   "n_samples": 100000,
   "n_unique": 33198,
   "n_forced_eos": 17791,
   "per_h": {
    "2": {
     "avg_loglik": -13.994782996264572,
     "em_iters_used": 100,
     "distill_kl_mc": 4.078227244013312
    },
    "4": {
     "avg_loglik": -13.15911380651663,
     "em_iters_used": 100,
     "distill_kl_mc": 3.2405201988037593
    },
    "8": {
     "avg_loglik": -12.366249485576574,
     "em_iters_used": 100,
     "distill_kl_mc": 2.4476052910339057
    },
    "16": {
     "avg_loglik": -11.92665970131934,
     "em_iters_used": 100,
     "distill_kl_mc": 2.0088123023190465
    },
    "32": {
     "avg_loglik": -11.578717748637416,
     "em_iters_used": 100,
     "distill_kl_mc": 1.6594528339427308
    }
   }
  },
  "table": [
   {
    "policy": "mask-only",
    "h": null,
    "kl_fwd": 0.5310182411550299,
    "kl_rev": 0.5516616319620444,
    "sum_p": 1.0,
    "guide_mc_kl": null
   },
   {
    "policy": "ctrlg",
    "h": 2,
    "kl_fwd": 1.9992174110894245,
    "kl_rev": 1.0392534771829336,
    "sum_p": 1.0000000000000004,
    "guide_mc_kl": 4.078227244013312
   },
   {
    "policy": "ctrlg",
    "h": 4,
    "kl_fwd": 1.7284096916719445,
    "kl_rev": 1.0090787228214624,
    "sum_p": 0.9999999999999998,
    "guide_mc_kl": 3.2405201988037593
   },
   {
    "policy": "ctrlg",
    "h": 8,
    "kl_fwd": 1.1733715958351294,
    "kl_rev": 0.6541745569739358,
    "sum_p": 0.9999999999999996,
    "guide_mc_kl": 2.4476052910339057
   },
   {
    "policy": "ctrlg",
    "h": 16,
    "kl_fwd": 0.8072083795865748,
    "kl_rev": 0.47696071923941025,
    "sum_p": 0.9999999999999997,
    "guide_mc_kl": 2.0088123023190465
   },
   {
    "policy": "ctrlg",
    "h": 32,
    "kl_fwd": 0.4811768698605567,
    "kl_rev": 0.33703712896819793,
    "sum_p": 1.0000000000000004,
    "guide_mc_kl": 1.6594528339427308
   },
   {
    "policy": "oracle",
    "h": null,
    "kl_fwd": -4.595798209410908e-16,
    "kl_rev": 4.595798209410923e-16,
    "sum_p": 1.0000000000000007,
    "guide_mc_kl": null
   }
  ],
  "runtime_s": 652.6919736862183
 },
 "nestful": {
  "provenance": "IBM vendored scorer output, test-200 (train-100 for the grid); job logs 2026-07-16..19, see EXPERIMENT_LOG.md",
  "test200": {
   "unconstrained": {
    "fsm": 0.145,
    "partial": 0.183,
    "f1_func": 0.703,
    "parse_err": 20,
    "valid": null
   },
   "mask_only": {
    "fsm": 0.235,
    "partial": 0.251,
    "f1_func": null,
    "parse_err": 1,
    "valid": "~178-196/200"
   },
   "ctrlg_old_guide": {
    "fsm": 0.21,
    "partial": 0.271,
    "f1_func": 0.931,
    "parse_err": 6,
    "valid": "194/200"
   },
   "ctrlg_walker_l1": {
    "fsm": 0.125,
    "partial": 0.211,
    "f1_func": 0.745,
    "parse_err": 0,
    "valid": "200/200"
   },
   "ctrlg_walker_l01": {
    "fsm": 0.23,
    "partial": 0.286,
    "f1_func": null,
    "parse_err": 0,
    "valid": "200/200"
   }
  },
  "lambda_grid_train100": {
   "mask_only": {
    "fsm": 0.22,
    "partial": 0.271,
    "valid": 99
   },
   "0.1": {
    "fsm": 0.2,
    "partial": 0.266,
    "valid": 99
   },
   "0.2": {
    "fsm": 0.2,
    "partial": 0.264,
    "valid": 99
   },
   "0.5": {
    "fsm": 0.17,
    "partial": 0.24,
    "valid": 100
   },
   "1.0": {
    "fsm": 0.125,
    "partial": 0.211,
    "valid": 100
   }
  },
  "frontier_no_reasoning": {
   "claude-opus-4-7": 0.36,
   "claude-sonnet-5": 0.35,
   "claude-opus-4-8": 0.33,
   "gpt-5.6-sol": 0.315,
   "gpt-5.6-terra": 0.3,
   "gpt-5.6-luna": 0.255
  }
 },
 "live": {
  "name": "nested_arith test-384 \u00b7 full Ctrl-G h256 \u03bb=0.5",
  "done": 384,
  "total": 384,
  "acc": 0.151
 }
};