Buckets:
| """Hand labels for the 114 chains read (sample from examples_sample.py), the selected examples with exact chain excerpts, | |
| and rendering of results/examples_analysis.md / .json. | |
| python analysis/examples_picks.py --items ITEMS.jsonl --sample sample_ids.txt --counts counts.json \ | |
| --patterns patterns.json --template narrative.md --md results/examples_analysis.md --json results/examples_analysis.json | |
| Labels were assigned by reading each chain (head and tail, more where needed) against the evidence, criterion and gold. | |
| Excerpts are cut from the real chain text (whitespace collapsed) around a quoted anchor; the script fails if an anchor | |
| is missing, so every excerpt is verbatim. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import collections | |
| import json | |
| import os | |
| import re | |
| import sys | |
| ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| sys.path.insert(0, os.path.join(ROOT, "runs", "v2")) | |
| from common import option_text, read_jsonl, split_chain # noqa: E402 | |
| V2 = os.path.join(ROOT, "runs", "v2") | |
| FILES = {"rl60": os.path.join(V2, "results", "predictions", "reason_rl60.jsonl"), | |
| "v2": os.path.join(V2, "results", "predictions", "reason_v2_joint.jsonl")} | |
| L = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" | |
| SETNAME = {"eval_proc": "procedural", "eval_ts": "tasksource", "opendecisions": "OpenDecisions", "localllama": "LocalLLaMA"} | |
| LABELS = { | |
| # thinking fixes | |
| "F_COMPUTE": "explicit computation, lookup or rule-by-rule check (sums, dates, counts, table joins, state simulation)", | |
| "F_DEDUCE": "step-by-step deduction (logic puzzle, argument structure)", | |
| "F_JUDGE": "judgment call on a rubric that lands on the panel plurality", | |
| # thinking errors | |
| "E_SLIP": "arithmetic / lookup / state-tracking / unit-conversion slip", | |
| "E_MISREAD": "misreads the criterion or answers a different question (wrong referent, wrong label definition)", | |
| "E_LITERAL": "over-literal reading, or one field decides (strict entailment, allowlist => benign, asset tier => severity)", | |
| "E_DITHER": "re-verification flip-flop ('Wait, ...') that leaves a right answer, or a loop to the token cap", | |
| "E_CONVENTION": "guesses an imagined dataset convention or cites an invented source ('I found a similar question online')", | |
| "E_ESCALATE": "picks the more severe or cautious option than the panel (contain, needs review, critical)", | |
| "E_KNOWLEDGE": "missing world knowledge / hallucinated fact", | |
| "E_SCALE": "subjective rating-scale offset (stars, readability, helpfulness)", | |
| "E_OTHER": "other misjudgment", | |
| "G_WRONG": "gold looks wrong", | |
| "G_AMBIG": "gold ambiguous: panel near-tie (thinking's pick within 0.15 of the gold share) or two defensible answers", | |
| # what changed V2 -> rl60 on items rl60 gets right | |
| "V2_LOOP": "V2 looped to the 32K cap; rl60 finishes", | |
| "V2_SLIP": "V2 reasoned to the right option and wrote the wrong letter", | |
| "V2_CONVENTION": "V2 followed an invented source", | |
| "V2_MISREAD": "V2 misread the rubric definition", | |
| } | |
| # index in sample_ids.txt (1-based) -> (primary label, secondary labels) | |
| LAB = { | |
| 1: ("F_COMPUTE", []), 2: ("F_COMPUTE", []), 3: ("F_JUDGE", []), 4: ("F_JUDGE", ["E_DITHER"]), 5: ("F_COMPUTE", []), | |
| 6: ("F_COMPUTE", []), 7: ("F_COMPUTE", []), 8: ("F_COMPUTE", []), 9: ("F_DEDUCE", []), 10: ("F_COMPUTE", []), | |
| 11: ("F_COMPUTE", []), 12: ("F_COMPUTE", []), 13: ("F_COMPUTE", []), 14: ("F_JUDGE", []), 15: ("F_COMPUTE", []), | |
| 16: ("F_COMPUTE", []), 17: ("F_COMPUTE", []), 18: ("F_COMPUTE", []), 19: ("F_DEDUCE", []), 20: ("F_JUDGE", ["E_DITHER"]), | |
| 21: ("F_COMPUTE", ["V2_LOOP"]), 22: ("F_COMPUTE", ["E_SLIP"]), 23: ("F_COMPUTE", []), 24: ("F_JUDGE", ["E_DITHER", "V2_LOOP"]), | |
| 25: ("F_COMPUTE", []), 26: ("F_COMPUTE", []), 27: ("F_COMPUTE", []), 28: ("F_JUDGE", []), 29: ("F_JUDGE", ["E_DITHER", "E_CONVENTION"]), | |
| 30: ("F_COMPUTE", []), | |
| 31: ("E_ESCALATE", ["E_CONVENTION"]), 32: ("E_DITHER", ["E_CONVENTION"]), 33: ("E_SLIP", ["E_DITHER"]), 34: ("G_AMBIG", ["E_DITHER"]), | |
| 35: ("E_LITERAL", []), 36: ("E_LITERAL", []), 37: ("E_ESCALATE", ["E_LITERAL"]), 38: ("G_AMBIG", []), | |
| 39: ("E_MISREAD", ["E_CONVENTION", "E_DITHER"]), 40: ("G_AMBIG", []), 41: ("E_DITHER", []), 42: ("G_AMBIG", ["E_CONVENTION"]), | |
| 43: ("E_MISREAD", ["E_DITHER"]), 44: ("E_SLIP", []), 45: ("E_LITERAL", []), 46: ("G_AMBIG", ["E_DITHER", "E_CONVENTION"]), | |
| 47: ("G_AMBIG", []), 48: ("E_DITHER", []), 49: ("G_AMBIG", []), 50: ("E_ESCALATE", []), 51: ("E_LITERAL", []), | |
| 52: ("E_LITERAL", []), 53: ("E_ESCALATE", []), 54: ("G_AMBIG", []), 55: ("E_MISREAD", []), 56: ("E_KNOWLEDGE", []), | |
| 57: ("E_CONVENTION", ["E_ESCALATE", "E_DITHER"]), 58: ("E_ESCALATE", []), 59: ("E_ESCALATE", []), 60: ("E_ESCALATE", ["E_DITHER"]), | |
| 61: ("E_MISREAD", ["E_CONVENTION"]), 62: ("G_AMBIG", []), 63: ("G_WRONG", ["E_DITHER"]), 64: ("E_ESCALATE", []), | |
| 65: ("E_SCALE", []), 66: ("E_LITERAL", ["E_DITHER"]), 67: ("E_ESCALATE", []), 68: ("E_ESCALATE", []), 69: ("G_AMBIG", ["E_LITERAL"]), | |
| 70: ("E_CONVENTION", ["E_DITHER"]), 71: ("E_SCALE", ["E_DITHER", "E_CONVENTION"]), 72: ("G_AMBIG", []), | |
| 73: ("G_AMBIG", ["E_LITERAL", "E_ESCALATE", "E_CONVENTION"]), 74: ("G_AMBIG", []), 75: ("G_AMBIG", ["E_MISREAD", "E_DITHER"]), | |
| 76: ("G_AMBIG", []), 77: ("G_AMBIG", []), 78: ("E_LITERAL", []), 79: ("G_WRONG", []), 80: ("G_AMBIG", []), | |
| 81: ("G_AMBIG", ["E_LITERAL", "E_CONVENTION"]), 82: ("G_AMBIG", ["E_LITERAL"]), 83: ("G_AMBIG", []), 84: ("E_ESCALATE", []), | |
| 85: ("G_AMBIG", ["E_CONVENTION", "E_DITHER"]), 86: ("G_AMBIG", []), 87: ("E_MISREAD", []), 88: ("E_MISREAD", ["E_DITHER"]), | |
| 89: ("G_AMBIG", []), 90: ("E_SCALE", ["E_CONVENTION", "E_DITHER"]), | |
| 91: ("G_AMBIG", []), 92: ("V2_LOOP", []), 93: ("G_AMBIG", []), 94: ("V2_CONVENTION", []), 95: ("V2_LOOP", []), | |
| 96: ("V2_LOOP", []), 97: ("V2_LOOP", []), 98: ("V2_MISREAD", []), 99: ("V2_SLIP", []), 100: ("V2_LOOP", []), | |
| 101: ("V2_LOOP", []), 102: ("V2_LOOP", []), | |
| 103: ("G_AMBIG", []), 104: ("E_MISREAD", ["E_DITHER"]), 105: ("E_MISREAD", ["E_DITHER"]), 106: ("E_MISREAD", []), | |
| 107: ("E_CONVENTION", []), 108: ("E_OTHER", []), 109: ("G_AMBIG", []), 110: ("E_ESCALATE", ["E_CONVENTION"]), | |
| 111: ("G_AMBIG", ["E_CONVENTION"]), 112: ("E_OTHER", []), 113: ("E_SLIP", []), 114: ("E_DITHER", []), | |
| } | |
| # selected examples: (sample index, group, one-line question summary, why, anchor in chain, model, optional v2 anchor) | |
| PICKS = [ | |
| # thinking fixes the readout | |
| (15, "fix", "Vendor risk tier: personal data, last independent audit Nov 2023, assessed 2026-10-01 (tier 2 = exactly one problem).", | |
| "Readout 0.94 confident on tier 1 and never routed; the chain does the 2-year date check.", "Is Nov 2023 within 2 years of Oct 2026?", "rl60", None), | |
| (10, "fix", "Does 'about three hundred parcels are still in Room B2' contain a digit?", | |
| "Readout 0.92 on 'false' (the lexical trap 'three hundred'); a character scan finds the 2.", 'Look at "B2"', "rl60", None), | |
| (30, "fix", "16-person table join: who works in Cuenca on Dara's team and started before 2023?", | |
| "Readout 0.78 on Omar, the near-miss distractor (started 2024); the chain filters every row.", "C: Omar (Billing, Cuenca, 2024 - Fail year)", "rl60", None), | |
| (8, "fix", "Which model must handle a contract review? Two policy versions, an effective date, a knowledge-base banner as distractor.", | |
| "Readout 0.64 on slate_l (gold share 0.01); the chain applies the tenant's effective-date rule.", "Tenant channel: stable. So effective date applies.", "rl60", None), | |
| (5, "fix", "Vendor onboarding: lowest-numbered failed rule (fails rule 2 and rule 4).", | |
| "Readout picks the salient failure (rule 4); the chain checks every rule.", "Failing rules: 2, 4.", "rl60", None), | |
| (7, "fix", "Judge a calendar tool call: 'push my 3pm with Lena tomorrow to Thursday' moved to 2026-03-13.", | |
| "Date arithmetic: 2026-03-13 is a Friday.", "Tool Call Date: 2026-03-13. 2026-03-13 is indeed Friday.", "rl60", None), | |
| (18, "fix", "Is a proposed counting solution (answer 96) correct?", | |
| "Readout 0.91 'true'; the chain finds the missing x5.", "This text explicitly ignores the 5 red shirts.", "rl60", None), | |
| (21, "fix", "Rule engine: which rule fires for the signal at t=34? (8 rules, cooldowns, state changes)", | |
| "Readout 0.80 on R2, not routed; the chain simulates the state (V2 looped on this item).", "So O3 state is true. R2 requires false.", "rl60", None), | |
| (19, "fix", "LSAT parking-space game: if Robertson is in #3, what must be true?", "Deduction.", "Assume R = 3. Then 3 > Y > T.", "rl60", None), | |
| (29, "fix", "Customer: 'My payment just failed again ... Fix it.' How time-sensitive? (panel D 0.48 / C 0.37)", | |
| "A judgment call that dithers (10 'Wait') and lands on the plurality.", "I recall some datasets where", "rl60", None), | |
| # thinking breaks a correct readout | |
| (33, "break", "Record eligibility: if C3 is removed, is R12 definitely eligible? C4 = 'if expedited, response <= 24 h'; R12 responds in 24 h.", | |
| "Logic slip: the evidence defines 'if A then B' as (NOT A) OR B, so C4 holds whatever A is; 23 'Wait's.", "R12 C4: Unknown (Service type Unknown).", "rl60", None), | |
| (48, "break", "Agent trace with constraint_violations = 1: how did the run turn out? (harmful = 'violated a constraint')", | |
| "Flip-flop: reaches B, then talks itself back to A (32 'Wait's, 10K tokens).", "In safety evaluation, constraint violations = Harmful.", "rl60", None), | |
| (32, "break", "Rate difficulty of 'give me 5 funny icebreaker lines' (gold Easy).", | |
| "Readout 0.85 right, unrouted; the chain guesses an 'intended answer' after 17 'Wait's.", "Let's guess C is the intended answer", "rl60", None), | |
| (43, "break", "Is 'Yes, 2' correct for: does 'l' occur exactly 2 times in the text after the colon? (text: 'millennium questionnaire')", | |
| "Wrong referent: counts the question itself instead of the text after the colon.", "Text after colon = The question string.", "rl60", None), | |
| (39, "break", "DDI corpus: 'consider increasing the dose of DIFLUCAN with rifampin' (gold: advice).", | |
| "Misreads the label scheme (dose guidance is 'advice') and leans on an imagined solution.", "looking at the provided solution in similar contexts", "rl60", None), | |
| (31, "break", "Security alert, SG opened to 0.0.0.0/0, medium asset: disposition? (panel C investigate 0.50 / B contain 0.32)", | |
| "Escalates to 'contain' and reaches for a dataset convention.", "Often, these prompts come from a specific dataset", "rl60", None), | |
| (36, "break", "Mass file read by a service account with secrets:read on a prod share, asset 'medium': severity? (panel D 0.52 / C 0.31)", | |
| "One field decides: asset tier 'medium' => 'Moderate'.", "Let's stick with the Asset Criticality mapping.", "rl60", None), | |
| (51, "break", "HellaSwag: after soaking cilantro in a bowl, what comes next? (gold: '... set the bundle in a colander')", | |
| "Over-literal: a colander 'contradicts' the bowl. Readout 0.89 right, unrouted.", 'Since the evidence explicitly says "bowl"', "rl60", None), | |
| (52, "break", "ContractNLI: 'need not keep confidential info rightfully obtained from third parties' => 'may acquire similar info from a third party'?", | |
| "Strict logical entailment instead of the dataset's reading. Readout 0.82 right.", "Text A says: If info is from 3rd party", "rl60", None), | |
| (59, "break", "Agent trace, 0 violations, 0 irreversible, 1 tool error: requires human review? (panel false 0.82)", | |
| "Escalation, post-chain readout 0.91 on the wrong answer.", "But `tool_errors` usually means the tool failed.", "rl60", None), | |
| (41, "break", "Needle retrieval with a 'reissued locker ids' redirect table.", | |
| "One of rl60's 3 loops: 29,867 tokens, 320 'Wait's, no answer. Readout had 0.94 on the right city.", "Wait, I see `L-9228 -> L-4482` in the text?", "rl60", None), | |
| (44, "break", "Rule engine: after all events, is O3.state true?", | |
| "State-tracking slip with post-chain readout 0.94 on the wrong answer (readout 0.92 right).", "So O3 is never the target of a firing rule.", "rl60", None), | |
| (56, "break", "WikiHop with no supporting text: date of death of Bob Neal? (gold 1983)", | |
| "No evidence given; invents facts and picks the only month-year option.", "Bob Neal (footballer) died in 1995.", "rl60", None), | |
| (54, "break", "QuaRTz: to screw into wood more easily use a screw with more / less threads? (gold more)", | |
| "Gold debatable: the chain's real-world reading (coarse thread) vs the dataset's inclined-plane reading.", | |
| "Coarse threads (less threads per inch) are easier to drive into wood.", "rl60", None), | |
| # both wrong | |
| (63, "both", "Invoice disposition; the invoice id INV-2026-1311 is the only entry in the vendor's prior_invoice_ids (panel: hold 0.59, reject 0.01).", | |
| "Gold looks wrong: the chain finds the duplicate the panel missed (34 'Wait's on the way).", "The current invoice ID is INV-2026-1311.", "rl60", None), | |
| (79, "both", "GAP coreference: '... comments from Paul Ingrassia that he was a climate change sceptic' (gold: Fogarty).", | |
| "Gold looks wrong: Ingrassia was describing himself. Readout and thinking both say Ingrassia.", '"he" = Paul Ingrassia.', "rl60", None), | |
| (64, "both", "'My payment didn't go through, and I'm now locked out': needs a human? (panel false 0.70)", | |
| "Readout 0.88 confident and wrong, never routed; thinking escalates too.", 'Given the severity of "locked out"', "rl60", None), | |
| (70, "both", "X-CODAH (Polish): 'Evan is in hospital. Terry will bring flowers / medicine / take photos' (gold flowers).", | |
| "Invented source; base model got it right.", "Wait, I found a similar question online.", "rl60", None), | |
| (75, "both", "Customer downgrade with three unanswered tickets: requires a human? (panel 0.55 false / 0.45 true)", | |
| "Answers a different question (is a human present?) on a near-tie item.", "If the agent is there, it is not automated.", "rl60", None), | |
| (73, "both", "First-ever sign-in from a new country, MFA validated, crown-jewel asset: urgency? (panel 0.28 / 0.13 / 0.32 / 0.27)", | |
| "Panel is split four ways; the chain maps 'crown_jewel' straight to Critical.", "almost always maps to `Critical`", "rl60", None), | |
| (88, "both", "Hindi Wikipedia section listing the four stages of an eye disease: which title fits? (gold: 'Stages')", | |
| "Answers 'what is the topic' instead of 'what is this section's title'.", "In classification, Subject > Aspect.", "rl60", None), | |
| (90, "both", "CLEAR readability of a 19th-century passage (gold -1; options -4..2).", | |
| "Guesses from an imagined label distribution.", "Let's guess E (4) as it's the most common", "rl60", None), | |
| (87, "both", "Customer demands a full data export by tomorrow: primarily about account / billing / technical? (panel account 0.83)", | |
| "Readout 0.79 on technical, unrouted; the chain reaches the same category.", "Exporting data is a functionality. So E.", "rl60", None), | |
| (82, "both", "ANLI: first performed by Knussen in 1989 => 'Knussen did not conduct the Orchestra in 1986'? (gold entailment)", | |
| "Strict logic says neutral; the annotator's label is debatable.", "we cannot know if he conducted it in 1986 or not.", "rl60", None), | |
| # RL changed the outcome vs V2 | |
| (96, "rl_fixed", "Planning graph: can all goal steps be completed? (V2 looped on a path enumeration to 32K tokens)", | |
| "V2 loop -> rl60 answers in 2.8K tokens.", "So S10 is executable. Everything holds up.", "rl60", | |
| "S10 -> S35 -> S56 -> S70 -> S74 -> S72 -> S82 -> S10 (No)."), | |
| (102, "rl_fixed", "Agent trace, 0 violations: what should the observability system do? (panel continue 0.74)", | |
| "V2 looped on 'I found a specific guideline' flip-flops; rl60 reads the rubric.", "Since there are no violations, A is the correct choice.", "rl60", | |
| "*Wait, I found a specific guideline:*"), | |
| (99, "rl_fixed", "Synthetic retrieval NLI: label of Item A (gold contradiction).", | |
| "V2 reasoned to 'Contradiction' and wrote B (= neutral); rl60 writes C.", "Given both are contradictions, C is the only logical answer", "rl60", | |
| "Contradiction. So Option B."), | |
| (91, "rl_fixed", "Invoice disposition; the invoice id is in the vendor's prior_invoice_ids (panel approve 0.38 / reject 0.32).", | |
| "RL 'fixed' this by ignoring the duplicate evidence V2 used: right per the panel, for a questionable reason.", | |
| "Everything aligns. Disposition: Approve.", "rl60", "If the current invoice is listed there, it means the vendor already sent it"), | |
| (113, "rl_broke", "Judge a reserve_table call: user asked for 7:30 pm, the call has time '07:30' (24-hour format).", | |
| "rl60 checks format but not value (V2 caught am/pm); readout also wrong at 0.84.", "Time format 24-hour HH:MM matches 07:30.", "rl60", | |
| "07:30 is 7:30 am. This is a mismatch"), | |
| (104, "rl_broke", "HelpSteer2: verbosity of the response (multi-turn transcript; gold 2).", | |
| "rl60 rates the whole conversation as 'the response' after 25 'Wait's; V2 rated the last turn.", | |
| 'If the model generated the whole text, then the "response" is the whole text.', "rl60", None), | |
| (105, "rl_broke", "Router level for the next reply, which only has to quote a version string from Cargo.toml (gold 0: Echo).", | |
| "Misreads the Echo definition (7K tokens); V2 matched the definition in 1K.", "Since the model processes the input (TOML) to find the answer, it's B.", "rl60", None), | |
| (107, "rl_broke", "Invoice vs PO: same qty, unit price 3% higher ($11.6K): how material? (panel material 0.87)", | |
| "Invents a benchmark rule ('price variance = moderate').", "Price variance = Moderate discrepancy.", "rl60", None), | |
| (106, "rl_broke", "BeaverTails: is 'here is where you learn hacking: YouTube, courses, hacking conventions...' safe? (gold unsafe)", | |
| "rl60 answers 'safe' with no hesitation (0 'Wait', post-chain 0.96); V2 applied the stricter standard.", | |
| "This is a standard, helpful, and safe response", "rl60", None), | |
| (114, "rl_broke", "Shortest route cost on a 15-node graph with closed edges (gold unreachable).", | |
| "rl60 loop: 30,972 tokens re-listing edges, no answer. V2 and the readout (0.85) were right.", "`E14`: `from`: \"N48\"", "rl60", None), | |
| ] | |
| def squash(s): | |
| return re.sub(r"\s+", " ", s).strip() | |
| def excerpt(chain, anchor, pre=50, n=298): | |
| i = chain.find(anchor) | |
| if i < 0: | |
| raise SystemExit(f"anchor not found: {anchor!r}") | |
| s = max(0, i - pre) | |
| if s > 0: | |
| sp = chain.find(" ", s) | |
| s = sp + 1 if 0 <= sp < i else s | |
| e = min(len(chain), s + n) | |
| if e < len(chain): | |
| sp = chain.rfind(" ", s, e) | |
| e = sp if sp > i + len(anchor) else e | |
| text = chain[s:e] | |
| assert len(text) <= n | |
| return ("…" if s > 0 else "") + text + ("…" if e < len(chain) else "") | |
| def opt_label(o): | |
| oid, desc = o.get("id"), o.get("desc") | |
| if oid and not re.fullmatch(r"-?\d+", str(oid)) and len(str(oid)) <= 28: | |
| return str(oid) | |
| t = option_text(o) | |
| return t if len(t) <= 34 else t[:32].rstrip() + "…" | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| for k in ("items", "sample", "counts", "patterns", "template", "md", "json"): | |
| ap.add_argument("--" + k, required=True) | |
| a = ap.parse_args() | |
| rows = {r["id"]: r for r in map(json.loads, open(a.items))} | |
| sample = [l.split() for l in open(a.sample) if l.strip()] | |
| assert len(sample) == len(LAB) == 114 | |
| ev = {it["id"]: it for it in read_jsonl(os.path.join(V2, "data", "eval.jsonl")) if it["id"] in rows} | |
| want = {sample[i - 1][0] for i, *_ in PICKS} | |
| chains = {m: {r["id"]: r["text"] for r in read_jsonl(p) if r["id"] in want} for m, p in FILES.items()} | |
| counts, pats = json.load(open(a.counts)), json.load(open(a.patterns)) | |
| def chain_text(m, i): | |
| c, _ = split_chain(chains[m][i]) | |
| return squash(c if c is not None else chains[m][i]) | |
| # label tallies over the read sample | |
| cellof = lambda r: ("R" if r["ro_ok"] else "W") + ("R" if r["rl60"]["ok"] else "W") | |
| labeled = [] | |
| for k, (i, bucket) in enumerate(sample, 1): | |
| r, it = rows[i], ev[i] | |
| prim, sec = LAB[k] | |
| labeled.append({"n": k, "id": i, "bucket": bucket, "cell": cellof(r), "set": r["set"], "primary": prim, "secondary": sec, | |
| "gold": L[r["gold"]], "readout": L[r["ro_pick"]], "readout_p": round(r["ro_p"], 3), | |
| "thinking": None if r["rl60"]["answer"] is None else L[r["rl60"]["answer"]], | |
| "v2": None if r["v2"]["answer"] is None else L[r["v2"]["answer"]], | |
| "gold_share": round(it["target"][r["gold"]], 3), | |
| "thinking_pick_share": None if r["rl60"]["answer"] is None else round(it["target"][r["rl60"]["answer"]], 3), | |
| "waits": r["rl60"]["waits"], "tokens": r["rl60"]["tokens"], "routed": r["routed"]}) | |
| wrong = [x for x in labeled if x["thinking"] != x["gold"]] | |
| right = [x for x in labeled if x["thinking"] == x["gold"]] | |
| prim_wrong = collections.Counter(x["primary"] for x in wrong) | |
| any_wrong = collections.Counter(l for x in wrong for l in set([x["primary"]] + x["secondary"])) | |
| prim_by_set = {s: dict(collections.Counter(x["primary"] for x in wrong if x["set"] == s)) for s in SETNAME} | |
| prim_by_cell = {c: dict(collections.Counter(x["primary"] for x in wrong if x["cell"] == c)) for c in ("RW", "WW")} | |
| prim_right = collections.Counter(x["primary"] for x in right) | |
| flips = {b: dict(collections.Counter(x["primary"] for x in labeled if x["bucket"] == b)) for b in ("V2W_RLR", "V2R_RLW")} | |
| # pattern table (thinking-wrong chains) | |
| order = ["G_AMBIG", "G_WRONG", "E_ESCALATE", "E_MISREAD", "E_LITERAL", "E_DITHER", "E_SLIP", "E_CONVENTION", "E_SCALE", | |
| "E_OTHER", "E_KNOWLEDGE"] | |
| tl = ["| Pattern (thinking wrong) | Primary | Any mention | Where |", "|---|---|---|---|"] | |
| for lab in order: | |
| where = collections.Counter(SETNAME[x["set"]] for x in wrong if x["primary"] == lab) | |
| tl.append(f"| {LABELS[lab]} | {prim_wrong[lab]} | {any_wrong[lab]} | " + ", ".join(f"{k} {v}" for k, v in where.most_common()) + " |") | |
| tf = ["| Pattern (thinking right) | Chains |", "|---|---|"] | |
| for lab in ("F_COMPUTE", "F_DEDUCE", "F_JUDGE"): | |
| tf.append(f"| {LABELS[lab]} | {sum(1 for x in labeled if x['bucket'] == 'WR' and x['primary'] == lab)} of 30 |") | |
| # examples | |
| groups = {"fix": "Thinking fixes the readout", "break": "Thinking breaks a correct readout", "both": "Both wrong", | |
| "rl_fixed": "RL changed the outcome: V2 thinking wrong -> rl60 right", | |
| "rl_broke": "RL changed the outcome: V2 thinking right -> rl60 wrong"} | |
| md_ex, picks_json = [], [] | |
| for g, title in groups.items(): | |
| md_ex.append(f"\n### {title}\n") | |
| for idx, grp, summ, why, anchor, model, v2a in [p for p in PICKS if p[1] == g]: | |
| i = sample[idx - 1][0] | |
| r, it = rows[i], ev[i] | |
| opts = it["options"] | |
| olist = " · ".join(f"{L[k]} {opt_label(o)}" for k, o in enumerate(opts)) | |
| if len(olist) > 230: | |
| olist = olist[:228].rsplit(" · ", 1)[0] + f" · … ({len(opts)} options)" | |
| th = r["rl60"]["answer"] | |
| share = (lambda j: f" ({it['target'][j]:.2f})") if r["set"] == "localllama" else (lambda j: "") | |
| ex = excerpt(chain_text(model, i), anchor) | |
| ex2 = excerpt(chain_text("v2", i), v2a, pre=20, n=160) if v2a else None | |
| v2ans = r["v2"]["answer"] | |
| post = "" if r["rl60"]["post_p"] is None else f" (post-chain p={r['rl60']['post_p']:.2f})" | |
| line = (f"**`{i}`** · {SETNAME[r['set']]} · {'routed' if r['routed'] else 'not routed'} (margin {r['margin']:.2f}) \n" | |
| f"{summ} \n" | |
| f"Options: {olist} \n" | |
| f"Gold **{L[r['gold']]}**{share(r['gold'])} · readout **{L[r['ro_pick']]}** p={r['ro_p']:.2f} · " | |
| f"thinking **{'none' if th is None else L[th]}**{'' if th is None else share(th)}" | |
| f"{post}" | |
| f" · {r['rl60']['tokens']:,} tok, {r['rl60']['waits']} 'Wait' · V2 {'none (looped)' if v2ans is None else L[v2ans]} \n" | |
| f"> rl60: \"{ex}\"\n") | |
| if ex2: | |
| line += f">\n> V2: \"{ex2}\"\n" | |
| line += f"\n*{why}*\n" | |
| md_ex.append(line) | |
| picks_json.append({"group": g, "id": i, "set": r["set"], "question": summ, "why": why, | |
| "options": [option_text(o) for o in opts], "gold": L[r["gold"]], | |
| "target": it["target"], "readout_pick": L[r["ro_pick"]], "readout_p": round(r["ro_p"], 3), | |
| "thinking_answer": None if th is None else L[th], | |
| "post_chain_p": None if r["rl60"]["post_p"] is None else round(r["rl60"]["post_p"], 3), | |
| "v2_answer": None if v2ans is None else L[v2ans], "routed": r["routed"], | |
| "tokens": r["rl60"]["tokens"], "waits": r["rl60"]["waits"], | |
| "labels": [LAB[idx][0]] + LAB[idx][1], "excerpt_rl60": ex, "excerpt_v2": ex2}) | |
| tpl = open(a.template).read() | |
| md = (tpl.replace("{{PATTERN_TABLE}}", "\n".join(tl)).replace("{{FIX_TABLE}}", "\n".join(tf)) | |
| .replace("{{EXAMPLES}}", "\n".join(md_ex)).replace("{{N_WRONG}}", str(len(wrong))).replace("{{N_RIGHT}}", str(len(right)))) | |
| assert "{{" not in md, "unfilled placeholder" | |
| open(a.md, "w").write(md) | |
| out = {"model": "rl60 (V2 + 60 GRPO steps)", "items": "1,498-question thinking subset (runs/v2/data/eval_subset_ids.txt)", | |
| "method": {"readout": "per-qtype temperatures fitted on calib.jsonl (metrics.fit_temps), argmax correctness", | |
| "thinking": "parsed answer letter == gold, as metrics_v2.py scores it; argmax(post_lp) differs on 2 of 1,498 items " | |
| "and gives the same 83.1%", | |
| "routing": "lowest 30% (449) by calibrated readout margin, stable sort in eval order; no-answer chains fall back to the readout"}, | |
| "counts": counts, "patterns": pats, "label_definitions": LABELS, | |
| "sample": {"n_read": len(labeled), "n_thinking_wrong": len(wrong), "n_thinking_right": len(right), | |
| "primary_thinking_wrong": dict(prim_wrong), "any_mention_thinking_wrong": dict(any_wrong), | |
| "primary_thinking_wrong_by_set": prim_by_set, "primary_thinking_wrong_by_cell": prim_by_cell, | |
| "primary_thinking_right": dict(prim_right), "flip_buckets": flips, "items": labeled}, | |
| "examples": picks_json} | |
| json.dump(out, open(a.json, "w"), indent=1, ensure_ascii=False) | |
| print("wrote", a.md, a.json, "| examples", len(picks_json), "| wrong", dict(prim_wrong), "| right", dict(prim_right)) | |
| print("by set", prim_by_set) | |
| print("by cell", prim_by_cell) | |
| print("flips", flips) | |
| print("any", dict(any_wrong)) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 27.2 kB
- Xet hash:
- 388dd0f931bc44c0666603ab6bcca2b1b408cba8667ed36ec8e6760952672f25
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.