Deploy PromptStat — UI shell + MiniCPM4.1-8B + 4-LoRA hybrid (Modal)

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+prompt_card/training/acquired/conversations.jsonl filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,15 +1,72 @@
 ---
-title: Promptstat
-emoji: 📊
-colorFrom: purple
-colorTo: yellow
+title: PromptStat
+emoji: 🎮
+colorFrom: gray
+colorTo: green
 sdk: gradio
-sdk_version: 6.18.0
-python_version: '3.13'
-app_file: app.py
+sdk_version: 6.6.0
+app_file: space_app.py
 pinned: false
 license: apache-2.0
-short_description: Your AI usage skill card from AI conversation
+short_description: Rate your AI-collaboration skill — MiniCPM-8B + LoRA
+tags:
+  - thousand-token-wood
+  - minicpm
+  - modal
+  - off-brand
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# 🎮 PromptStat
+
+Upload your **ChatGPT or Claude export** (or paste a conversation / share link) and get an esports-style
+**stat card** rating *how you collaborate with AI* across 5 observable axes:
+
+**Focus · Technique · Critical Engagement · Interaction · Input Quality** → an overall score + tier (D→S).
+
+> 🔗 **Live demo:** [demo link — TODO]  ·  🐦 **Write-up:** [social link — TODO]
+
+## What makes it real
+- **Scored by MiniCPM4.1-8B** (OpenBMB) using a validated observable-detection method — not vibes.
+  The model is **served on Modal** (vLLM on an A100-40GB, the 4 LoRA adapters loaded warm and addressable
+  by name). The card tags every score "✓ scored on MiniCPM-8B + LoRA hybrid"; if no model endpoint is
+  configured it falls back to a transparent heuristic and says so (amber tag).
+- **Per-category LoRA hybrid** (trained **on Modal**, H100): the 8B is fine-tuned only where it makes
+  correctable errors — `decomposition` 0.26→**0.66 κ**, `critical·source_request` 0.47→**0.66**, plus
+  `independent_verification` and `interaction` — and stays base everywhere it's already strong. Each adapter
+  overrides only its own feature, so there's no catastrophic forgetting.
+- **Honest validation**: every axis ships its measured **Cohen's κ** vs a 159-conversation human-labeled
+  set. We document where the 8B hits its ceiling (it can't out-agree the human rater) instead of inflating.
+
+## Two ways in
+- **Upload export** — full profile across your whole history (representative 30-conversation sample, every
+  turn within each scored; set `UI_MAX_CONVS=0` for the entire history).
+- **Paste a conversation or a ChatGPT/Claude share link** — fast single-conversation sample for quick
+  feedback / demo. Clearly labelled "Sample analysis — not your overall AI usage."
+
+## Privacy
+Your chat content is parsed **in memory only** — never written to disk, never logged. A pasted share link is
+fetched once to read the conversation and is likewise never stored.
+
+## Architecture (Modal end-to-end)
+- **Inference:** `modal_serve_minicpm.py` — vLLM serves base MiniCPM4.1-8B **plus** all 4 LoRA adapters from a
+  single warm A100; the app routes each axis to its adapter by model name over HTTP. Model weights are baked
+  into the image so container starts don't touch the Hub.
+- **Training:** `modal_train_lora.py` / `modal_eval_lora.py` — the LoRA adapters were trained and evaluated on
+  Modal GPUs against the human-labeled validation set.
+
+## Run locally
+```bash
+pip install -r requirements.txt
+# real scoring needs a MiniCPM endpoint (auto-loaded from eval/.secrets.env if present):
+export OPENBMB_BASE_URL=...   # OpenBMB free API, or your Modal vLLM URL
+export OPENBMB_TOKEN=...
+python -m ui.app              # or: python space_app.py   (or ./run_demo.sh for the LoRA hybrid)
+python -m pytest ui/tests -q  # 19 UI tests
+```
+Without the env vars the app still runs (heuristic fallback, amber tag). See `DEPLOY.md` for the Space deploy
++ Modal model-serving + speed tuning, and `handoff_to_card.md` for the `card_data` schema.
+
+## Built for the OpenBMB "Build Small" hackathon
+Models ≤32B, English. Uses **MiniCPM4.1-8B** (reasoning) — eligible model — with **Modal** for both LoRA
+training and warm inference. Maintainer notes: backend/ML workstream in `prompt_card/`, `modal_*.py`,
+`SPEC.md`; UI shell in `ui/`.

eval/_cache/cascade_results.json

@@ -0,0 +1,462 @@
+{
+ "baseline": {
+  "_new_calls": 0,
+  "technique": {
+   "headline": 0.12970513816185567,
+   "axis_any": 0.3282188125416949,
+   "cats": {
+    "zero_shot_role": [
+     0.5291833914588411,
+     {
+      "tp": 9,
+      "fp": 7,
+      "fn": 8,
+      "tn": 453
+     },
+     17
+    ],
+    "few_shot": [
+     0.0,
+     {
+      "tp": 0,
+      "fp": 0,
+      "fn": 7,
+      "tn": 470
+     },
+     7
+    ],
+    "thought_generation": [
+     0.0,
+     {
+      "tp": 0,
+      "fp": 2,
+      "fn": 0,
+      "tn": 475
+     },
+     0
+    ],
+    "decomposition": [
+     -0.0075597554196795,
+     {
+      "tp": 0,
+      "fp": 2,
+      "fn": 17,
+      "tn": 458
+     },
+     17
+    ],
+    "self_criticism": [
+     null,
+     {
+      "tp": 0,
+      "fp": 0,
+      "fn": 0,
+      "tn": 477
+     },
+     0
+    ],
+    "flipped": [
+     -0.002803083391738918,
+     {
+      "tp": 0,
+      "fp": 1,
+      "fn": 2,
+      "tn": 474
+     },
+     2
+    ]
+   },
+   "parse_fail": 3
+  },
+  "input_quality": {
+   "headline": 0.09211310491513845,
+   "axis_any": 0.1846153846153846,
+   "cats": {
+    "goal_stated": [
+     0.09353119536482576,
+     {
+      "tp": 31,
+      "fp": 219,
+      "fn": 6,
+      "tn": 221
+     },
+     37
+    ],
+    "specific_specification": [
+     0.09069501446545114,
+     {
+      "tp": 42,
+      "fp": 6,
+      "fn": 249,
+      "tn": 180
+     },
+     291
+    ]
+   },
+   "parse_fail": 3
+  },
+  "interaction": {
+   "headline": 0.13385383129426917,
+   "counts": {
+    "tp": 176,
+    "fp": 392,
+    "fn": 30,
+    "tn": 218
+   },
+   "pos": 206,
+   "n": 816,
+   "parse_fail": 1
+  },
+  "focus": {
+   "headline": 0.39427812903901904,
+   "T": 0.55,
+   "recall": 0.684931506849315
+  },
+  "_secs": 70.5
+ },
+ "v2": {
+  "_new_calls": 0,
+  "technique": {
+   "headline": 0.29018965857011875,
+   "axis_any": 0.551405033291953,
+   "cats": {
+    "zero_shot_role": [
+     0.9000069876318915,
+     {
+      "tp": 14,
+      "fp": 0,
+      "fn": 3,
+      "tn": 460
+     },
+     17
+    ],
+    "few_shot": [
+     0.0,
+     {
+      "tp": 0,
+      "fp": 0,
+      "fn": 7,
+      "tn": 470
+     },
+     7
+    ],
+    "thought_generation": [
+     0.0,
+     {
+      "tp": 0,
+      "fp": 1,
+      "fn": 0,
+      "tn": 476
+     },
+     0
+    ],
+    "decomposition": [
+     0.26075164664858347,
+     {
+      "tp": 3,
+      "fp": 2,
+      "fn": 14,
+      "tn": 458
+     },
+     17
+    ],
+    "self_criticism": [
+     null,
+     {
+      "tp": 0,
+      "fp": 0,
+      "fn": 0,
+      "tn": 477
+     },
+     0
+    ],
+    "flipped": [
+     0.0,
+     {
+      "tp": 0,
+      "fp": 0,
+      "fn": 2,
+      "tn": 475
+     },
+     2
+    ]
+   },
+   "parse_fail": 0
+  },
+  "input_quality": {
+   "headline": 0.3971921361556304,
+   "axis_any": 0.5678643257052781,
+   "cats": {
+    "goal_stated": [
+     0.226437865359231,
+     {
+      "tp": 8,
+      "fp": 14,
+      "fn": 29,
+      "tn": 426
+     },
+     37
+    ],
+    "specific_specification": [
+     0.5679464069520299,
+     {
+      "tp": 225,
+      "fp": 35,
+      "fn": 66,
+      "tn": 151
+     },
+     291
+    ]
+   },
+   "parse_fail": 0
+  },
+  "interaction": {
+   "headline": 0.07623546939015509,
+   "counts": {
+    "tp": 32,
+    "fp": 57,
+    "fn": 174,
+    "tn": 553
+   },
+   "pos": 206,
+   "n": 816,
+   "parse_fail": 0
+  },
+  "focus": {
+   "headline": 0.4046963209425367,
+   "T": 0.55,
+   "recall": 0.684931506849315
+  },
+  "_secs": 68.0
+ },
+ "v3": {
+  "_new_calls": 0,
+  "technique": {
+   "headline": 0.29018965857011875,
+   "axis_any": 0.551405033291953,
+   "cats": {
+    "zero_shot_role": [
+     0.9000069876318915,
+     {
+      "tp": 14,
+      "fp": 0,
+      "fn": 3,
+      "tn": 460
+     },
+     17
+    ],
+    "few_shot": [
+     0.0,
+     {
+      "tp": 0,
+      "fp": 0,
+      "fn": 7,
+      "tn": 470
+     },
+     7
+    ],
+    "thought_generation": [
+     0.0,
+     {
+      "tp": 0,
+      "fp": 1,
+      "fn": 0,
+      "tn": 476
+     },
+     0
+    ],
+    "decomposition": [
+     0.26075164664858347,
+     {
+      "tp": 3,
+      "fp": 2,
+      "fn": 14,
+      "tn": 458
+     },
+     17
+    ],
+    "self_criticism": [
+     null,
+     {
+      "tp": 0,
+      "fp": 0,
+      "fn": 0,
+      "tn": 477
+     },
+     0
+    ],
+    "flipped": [
+     0.0,
+     {
+      "tp": 0,
+      "fp": 0,
+      "fn": 2,
+      "tn": 475
+     },
+     2
+    ]
+   },
+   "parse_fail": 0
+  },
+  "input_quality": {
+   "headline": 0.36485526886734726,
+   "axis_any": 0.5401767615340956,
+   "cats": {
+    "goal_stated": [
+     0.1937101132639565,
+     {
+      "tp": 31,
+      "fp": 141,
+      "fn": 6,
+      "tn": 299
+     },
+     37
+    ],
+    "specific_specification": [
+     0.536000424470738,
+     {
+      "tp": 213,
+      "fp": 32,
+      "fn": 78,
+      "tn": 154
+     },
+     291
+    ]
+   },
+   "parse_fail": 4
+  },
+  "interaction": {
+   "headline": 0.3197441320340984,
+   "counts": {
+    "tp": 148,
+    "fp": 199,
+    "fn": 58,
+    "tn": 411
+   },
+   "pos": 206,
+   "n": 816,
+   "parse_fail": 0
+  },
+  "focus": {
+   "headline": 0.4046963209425367,
+   "T": 0.55,
+   "recall": 0.684931506849315
+  },
+  "_secs": 68.4
+ },
+ "v4": {
+  "_new_calls": 2204,
+  "technique": {
+   "headline": 0.1942222178493125,
+   "axis_any": 0.3235641692271324,
+   "cats": {
+    "zero_shot_role": [
+     0.631597013644554,
+     {
+      "tp": 8,
+      "fp": 0,
+      "fn": 9,
+      "tn": 460
+     },
+     17
+    ],
+    "few_shot": [
+     -0.0036822724881613316,
+     {
+      "tp": 0,
+      "fp": 1,
+      "fn": 7,
+      "tn": 469
+     },
+     7
+    ],
+    "thought_generation": [
+     0.0,
+     {
+      "tp": 0,
+      "fp": 1,
+      "fn": 0,
+      "tn": 476
+     },
+     0
+    ],
+    "decomposition": [
+     0.1489741302408574,
+     {
+      "tp": 2,
+      "fp": 5,
+      "fn": 15,
+      "tn": 455
+     },
+     17
+    ],
+    "self_criticism": [
+     null,
+     {
+      "tp": 0,
+      "fp": 0,
+      "fn": 0,
+      "tn": 477
+     },
+     0
+    ],
+    "flipped": [
+     0.0,
+     {
+      "tp": 0,
+      "fp": 0,
+      "fn": 2,
+      "tn": 475
+     },
+     2
+    ]
+   },
+   "parse_fail": 2
+  },
+  "input_quality": {
+   "headline": 0.19584802394592618,
+   "axis_any": 0.42787859824780966,
+   "cats": {
+    "goal_stated": [
+     0.1445087236840895,
+     {
+      "tp": 34,
+      "fp": 192,
+      "fn": 3,
+      "tn": 248
+     },
+     37
+    ],
+    "specific_specification": [
+     0.24718732420776285,
+     {
+      "tp": 96,
+      "fp": 7,
+      "fn": 195,
+      "tn": 179
+     },
+     291
+    ]
+   },
+   "parse_fail": 0
+  },
+  "interaction": {
+   "headline": 0.21034110636700612,
+   "counts": {
+    "tp": 152,
+    "fp": 276,
+    "fn": 54,
+    "tn": 334
+   },
+   "pos": 206,
+   "n": 816,
+   "parse_fail": 0
+  },
+  "focus": {
+   "headline": 0.4333764553686933,
+   "T": 0.5,
+   "recall": 0.4931506849315068
+  },
+  "_secs": 445.8
+ }
+}

eval/_cache/critical.json

@@ -0,0 +1,14 @@
+{
+ "per_type": {
+  "skepticism": 0.5761426708319434,
+  "rebuttal": 0.5209181524638488,
+  "source_request": 0.4667274384685487,
+  "independent_verification": 0.03541359607502818,
+  "re_questioning": 0.057768710404807265
+ },
+ "any": 0.5667524582590905,
+ "headline": 0.33139411364883525,
+ "mae": 0.33962264150943394,
+ "decision": "LoRA NEEDED (headline < 0.4)",
+ "parse_fail": 2
+}

eval/_cache/critical_hybrid.json

@@ -0,0 +1,13 @@
+{
+  "per_type": {
+    "skepticism": 0.576,
+    "rebuttal": 0.521,
+    "source_request": 0.664,
+    "independent_verification": 0.578,
+    "re_questioning": 0.058
+  },
+  "headline": 0.479,
+  "routing": {"base": ["skepticism","rebuttal","re_questioning"], "lora": ["source_request","independent_verification"]},
+  "lora_adapter": "critical_c_r16e5",
+  "note": "Phase-7 locked per-type hybrid. base per-type from critical.json; source_request+independent_verification from critical_c_r16e5."
+}

eval/_cache/lora_critical.json

@@ -0,0 +1,12 @@
+{
+ "axis": "critical",
+ "lora_per_type": {
+  "skepticism": 0.3848012434786635,
+  "rebuttal": 0.4037305065742538,
+  "source_request": 0.3970315398886822,
+  "independent_verification": 0.5821293774453546,
+  "re_questioning": 0.061319706731838106
+ },
+ "lora_headline": 0.36580247482375844,
+ "base_headline": 0.33139411364883525
+}

eval/_cache/lora_critical_c_r16e5.json

@@ -0,0 +1,13 @@
+{
+ "axis": "critical",
+ "lora_per_type": {
+  "skepticism": 0.4967512752003869,
+  "rebuttal": 0.5270003941663386,
+  "source_request": 0.663908996897621,
+  "independent_verification": 0.5775455588122841,
+  "re_questioning": 0.02836335760827934
+ },
+ "lora_headline": 0.45871391653698196,
+ "base_headline": 0.33139411364883525,
+ "adapter": "critical_c_r16e5"
+}

eval/_cache/lora_critical_c_r16e8.json

@@ -0,0 +1,13 @@
+{
+ "axis": "critical",
+ "lora_per_type": {
+  "skepticism": 0.5559570075297539,
+  "rebuttal": 0.554183813443073,
+  "source_request": 0.5518786625301633,
+  "independent_verification": 0.6110889509373758,
+  "re_questioning": 0.058456190046997245
+ },
+ "lora_headline": 0.46631292489747267,
+ "base_headline": 0.33139411364883525,
+ "adapter": "critical_c_r16e8"
+}

eval/_cache/lora_decomposition.json

@@ -0,0 +1,5 @@
+{
+ "axis": "decomposition",
+ "lora_kappa": 0.4521019986216408,
+ "base_kappa": 0.261
+}

eval/_cache/lora_decomposition_d_r16e5.json

@@ -0,0 +1,6 @@
+{
+ "axis": "decomposition",
+ "lora_kappa": 0.6560426882030562,
+ "base_kappa": 0.261,
+ "adapter": "decomposition_d_r16e5"
+}

eval/_cache/lora_decomposition_d_r16e8.json

@@ -0,0 +1,6 @@
+{
+ "axis": "decomposition",
+ "lora_kappa": 0.6560426882030562,
+ "base_kappa": 0.261,
+ "adapter": "decomposition_d_r16e8"
+}

eval/_cache/lora_decomposition_p6a.json

@@ -0,0 +1,6 @@
+{
+ "axis": "decomposition",
+ "lora_kappa": 0.48731249123790904,
+ "base_kappa": 0.261,
+ "adapter": "decomposition_p6a"
+}

eval/_cache/lora_decomposition_p6b.json

@@ -0,0 +1,6 @@
+{
+ "axis": "decomposition",
+ "lora_kappa": 0.5473769825132166,
+ "base_kappa": 0.261,
+ "adapter": "decomposition_p6b"
+}

eval/_cache/lora_decomposition_p6c.json

@@ -0,0 +1,6 @@
+{
+ "axis": "decomposition",
+ "lora_kappa": 0.5919589392643283,
+ "base_kappa": 0.261,
+ "adapter": "decomposition_p6c"
+}

eval/_cache/lora_decomposition_p6d.json

@@ -0,0 +1,6 @@
+{
+ "axis": "decomposition",
+ "lora_kappa": 0.6120374135827567,
+ "base_kappa": 0.261,
+ "adapter": "decomposition_p6d"
+}

eval/_cache/lora_goal_stated.json

@@ -0,0 +1,5 @@
+{
+ "axis": "goal_stated",
+ "lora_kappa": 0.4188278588656541,
+ "base_kappa": 0.226
+}

eval/_cache/lora_goal_stated_g_r16e5.json

@@ -0,0 +1,6 @@
+{
+ "axis": "goal_stated",
+ "lora_kappa": 0.4370514497145227,
+ "base_kappa": 0.226,
+ "adapter": "goal_stated_g_r16e5"
+}

eval/_cache/lora_interaction.json

@@ -0,0 +1,5 @@
+{
+ "axis": "interaction",
+ "lora_kappa": 0.4497702102011602,
+ "base_kappa": 0.3197441320340984
+}

eval/_cache/lora_interaction_i_r16e5.json

@@ -0,0 +1,6 @@
+{
+ "axis": "interaction",
+ "lora_kappa": 0.47302563796270025,
+ "base_kappa": 0.32,
+ "adapter": "interaction_i_r16e5"
+}

eval/_cache/lora_interaction_i_r16e8.json

@@ -0,0 +1,6 @@
+{
+ "axis": "interaction",
+ "lora_kappa": 0.4320932517529365,
+ "base_kappa": 0.32,
+ "adapter": "interaction_i_r16e8"
+}

eval/_cache/lora_interaction_i_r32e5.json

@@ -0,0 +1,6 @@
+{
+ "axis": "interaction",
+ "lora_kappa": 0.4458529518290473,
+ "base_kappa": 0.32,
+ "adapter": "interaction_i_r32e5"
+}

eval/_cache/step_c.json

@@ -0,0 +1,6 @@
+{
+ "goal": 0.1610096976611523,
+ "spec": 0.18226702324624555,
+ "iq_mean": 0.17163836045369893,
+ "interaction_sc": 0.30050531092090343
+}

eval/_cache/step_d.json

@@ -0,0 +1,7 @@
+{
+ "interaction_think": 0.22752203222903328,
+ "focus_think": 0.31570533819020363,
+ "focus_T": 0.55,
+ "int_parsefail": 436,
+ "foc_parsefail": 71
+}

eval/_ceiling/prep_p7.py

@@ -0,0 +1,82 @@
+"""Phase-7 ceiling-check prep for Interaction + Critical (goal/decomp reuse Phase-6 artifacts).
+
+Builds 50-unit stratified samples (positive-enriched so inter-model κ is stable) from the 159-conv
+validation set, with Kim GT attached, plus a BLIND version (context only) for Opus/Sonnet labelers.
+
+Interaction unit: (prev_user, this_user) -> refinement_attempt bool.
+Critical unit: (prev_assistant, this_user) -> set of CE types (5).
+
+Run:  python -m eval._ceiling.prep_p7
+"""
+from __future__ import annotations
+
+import json
+import os
+import random
+
+from eval import kappa as K
+from prompt_card.scoring import observable_axes as OA
+from eval.step_critical import CE
+
+HERE = os.path.dirname(__file__)
+SEED = 7
+N = 50
+
+
+def _interaction_units(gt, convs):
+    units = []
+    for r in gt:
+        ut = K.user_turns(convs[r["id"]])
+        for row in r["interaction"]:
+            i = int(row["turn"][1:]) - 1
+            if i < 1 or i >= len(ut):
+                continue
+            units.append({"cid": r["id"], "turn": row["turn"], "prev_user": ut[i - 1],
+                          "this_user": ut[i], "kim": bool(row["refinement"])})
+    return units
+
+
+def _critical_units(gt, convs):
+    units = []
+    for r in gt:
+        conv = convs[r["id"]]; ut = K.user_turns(conv)
+        for row in r["critical"]:
+            i = int(row["turn"][1:]) - 1
+            if i < 0 or i >= len(ut):
+                continue
+            units.append({"cid": r["id"], "turn": row["turn"],
+                          "prev_assistant": OA._prev_assistant(conv, i) or "",
+                          "this_user": ut[i], "kim": sorted(row["types"])})
+    return units
+
+
+def _stratified(units, is_pos, n, rng, pos_frac):
+    pos = [u for u in units if is_pos(u)]
+    neg = [u for u in units if not is_pos(u)]
+    rng.shuffle(pos); rng.shuffle(neg)
+    npos = min(len(pos), int(n * pos_frac))
+    sample = pos[:npos] + neg[:n - npos]
+    rng.shuffle(sample)
+    for k, u in enumerate(sample):
+        u["idx"] = k
+    return sample
+
+
+def _write(name, sample, blind_fields):
+    json.dump(sample, open(os.path.join(HERE, f"{name}_samples_p7.json"), "w"), ensure_ascii=False, indent=1)
+    blind = [{**{"idx": u["idx"]}, **{f: u[f] for f in blind_fields}} for u in sample]
+    json.dump(blind, open(os.path.join(HERE, f"{name}_blind_p7.json"), "w"), ensure_ascii=False, indent=1)
+    print(f"[prep_p7] {name}: {len(sample)} units ({sum(1 for u in sample if (u['kim'] if isinstance(u['kim'],bool) else u['kim']))} pos-ish)")
+
+
+def main():
+    rng = random.Random(SEED)
+    gt = K.load_gt(); convs = K.load_convs()
+    inter = _stratified(_interaction_units(gt, convs), lambda u: u["kim"], N, rng, 0.4)
+    _write("interaction", inter, ["prev_user", "this_user"])
+    crit = _stratified(_critical_units(gt, convs), lambda u: len(u["kim"]) > 0, N, rng, 0.5)
+    _write("critical", crit, ["prev_assistant", "this_user"])
+
+
+if __name__ == "__main__":
+    main()

eval/cascade.py

@@ -0,0 +1,129 @@
+"""Cascade runner: measure base-8B per axis under different prompt/inference strategies, compare to
+baseline, apply the decision gate. Reuses eval.kappa scoring; the 8B cache makes unchanged prompts free.
+
+Usage:
+  python -m eval.cascade baseline v2        # measure these versions, print before/after κ table
+Versions are registered in VERSIONS (a builders module per key). Results persist to
+eval/_cache/cascade_results.json so later steps accumulate.
+"""
+from __future__ import annotations
+
+import importlib
+import json
+import os
+import sys
+import time
+
+from eval import kappa as K
+from prompt_card.scoring import observable_axes as OA
+
+RESULTS = os.path.join(os.path.dirname(__file__), "_cache", "cascade_results.json")
+VERSIONS = {"baseline": OA, "v2": "eval.prompts_v2", "v3": "eval.prompts_v3", "v4": "eval.prompts_v4"}
+
+
+def _builders(v):
+    b = VERSIONS[v]
+    return importlib.import_module(b) if isinstance(b, str) else b
+
+
+def measure(builders, gt, convs, embedder, client):
+    """Return per-axis headline κ + detail. Cache-served calls are free."""
+    prompts, plan, geom = K.build_prompts(gt, convs, embedder, builders=builders)
+    pi = {}
+    for it in plan:
+        pi.setdefault(it[0], []).append(it)
+    n_before = client.misses
+    responses = client.run_all(prompts)
+    new_calls = client.misses - n_before
+
+    out = {"_new_calls": new_calls}
+    # technique / input_quality (per-category + axis-level "any")
+    for axis, fields in (("technique", K.TECH), ("input_quality", K.IQ)):
+        per, fail = K.score_binary_axis(gt, responses, pi, axis, fields)
+        cats = {f: K.cohen_kappa(*per[f]) for f in fields}
+        n = len(per[fields[0]][0])
+        anyt = [int(any(per[f][0][j] for f in fields)) for j in range(n)]
+        anyp = [int(any(per[f][1][j] for f in fields)) for j in range(n)]
+        feat_ks = [cats[f] for f in fields if sum(per[f][0]) > 0]  # only categories with positives
+        headline = (sum(feat_ks) / len(feat_ks)) if feat_ks else None
+        out[axis] = {"headline": headline, "axis_any": K.cohen_kappa(anyt, anyp),
+                     "cats": {f: (cats[f], K.binary_counts(*per[f]), sum(per[f][0])) for f in fields},
+                     "parse_fail": fail}
+    # interaction
+    yt, yp, fail = K.score_interaction(gt, responses, pi)
+    out["interaction"] = {"headline": K.cohen_kappa(yt, yp), "counts": K.binary_counts(yt, yp),
+                          "pos": sum(yt), "n": len(yt), "parse_fail": fail}
+    # focus (sweep best T)
+    best = None
+    for T in [0.40, 0.45, 0.50, 0.55, 0.60, 0.65, 0.70]:
+        fyt, fyp, info = K.score_focus(gt, responses, pi, geom, T)
+        k = K.cohen_kappa(fyt, fyp)
+        cand = (k if k is not None else -9, info["recall"] or 0)
+        if best is None or cand > best[0]:
+            best = (cand, T, k, info["recall"])
+    out["focus"] = {"headline": best[2], "T": best[1], "recall": best[3]}
+    return out
+
+
+def gate(k):
+    if k is None:
+        return "N/A"
+    if k >= 0.6:
+        return "SOLID"
+    if k >= 0.4:
+        return "OK"
+    if k >= 0.2:
+        return "try-next"
+    return "must-next"
+
+
+def main(version_keys):
+    base_url = os.environ.get("OPENBMB_BASE_URL"); token = os.environ.get("OPENBMB_TOKEN")
+    if not base_url or not token:
+        print("ERROR: set OPENBMB_BASE_URL and OPENBMB_TOKEN", file=sys.stderr); sys.exit(2)
+    from prompt_card.llm.minicpm import MiniCPMClient
+    gt = K.load_gt(); convs = K.load_convs(); embedder = K.FastEmbedder()
+    client = K.CachedClient(MiniCPMClient(base_url, token), workers=8)
+
+    prior = {}
+    if os.path.exists(RESULTS):
+        prior = json.load(open(RESULTS))
+
+    results = dict(prior)
+    for v in version_keys:
+        t0 = time.time()
+        print(f"\n=== measuring '{v}' ===", flush=True)
+        r = measure(_builders(v), gt, convs, embedder, client)
+        r["_secs"] = round(time.time() - t0, 1)
+        results[v] = r
+        print(f"  new 8B calls: {r['_new_calls']}  ·  {r['_secs']}s", flush=True)
+    json.dump(results, open(RESULTS, "w"), indent=1, default=str)
+
+    axes = ["technique", "input_quality", "interaction", "focus"]
+    print("\n================ κ comparison (headline per axis) ================")
+    head = "axis".ljust(16) + "".join(v.ljust(12) for v in version_keys) + "gate(last)"
+    print(head)
+    for ax in axes:
+        row = ax.ljust(16)
+        last = None
+        for v in version_keys:
+            k = results[v][ax]["headline"]; last = k
+            row += (f"{k:+.3f}" if k is not None else "N/A").ljust(12)
+        print(row + gate(last))
+    # per-category technique/IQ detail for the last version
+    last = version_keys[-1]
+    print(f"\n--- per-category detail ({last}) ---")
+    for ax in ("technique", "input_quality"):
+        for f, (k, c, npos) in results[last][ax]["cats"].items():
+            ks = f"{k:+.3f}" if k is not None else "N/A"
+            print(f"  {ax[:4]}.{f:22} κ={ks}  pos={npos}  [TN {c['tn']} FP {c['fp']} FN {c['fn']} TP {c['tp']}]")
+    fi = results[last]["interaction"]; c = fi["counts"]
+    print(f"  interaction.refinement      κ={fi['headline']:+.3f}  pos={fi['pos']}/{fi['n']}  "
+          f"[TN {c['tn']} FP {c['fp']} FN {c['fn']} TP {c['tp']}]")
+    ff = results[last]["focus"]
+    print(f"  focus.topic_shift           κ={ff['headline']:+.3f}  T={ff['T']}  recall={ff['recall']:.2f}")
+
+
+if __name__ == "__main__":
+    keys = sys.argv[1:] or ["baseline", "v2"]
+    main(keys)

eval/kappa.py

@@ -0,0 +1,412 @@
+"""Phase-2 measurement harness: base MiniCPM4.1-8B + bge-small embedder + rules vs Kim-verified
+ground truth. Computes per-axis Cohen's kappa, per-category breakdowns (sparse → N/A), confusion
+matrices, an OOD report, and embedder boundary-recall vs hand-identified Focus candidates.
+
+Pure metrics (cohen_kappa, confusion) are dependency-free and unit-tested in tests/test_kappa.py.
+8B calls go through a disk cache (eval/_cache/preds_8b.jsonl) keyed by sha1(prompt) + a thread pool,
+so runs are resumable and Phase 3 (Critical) reuses every cached response.
+
+Credentials (shared OpenBMB hackathon key) are read from the environment, never hardcoded:
+  OPENBMB_BASE_URL, OPENBMB_TOKEN
+Run:  python -m eval.kappa            # 4 Phase-2 axes (Focus, Technique, Interaction, Input Quality)
+      python -m eval.kappa --limit 5 # quick probe on 5 convs (latency/parse check before full run)
+"""
+from __future__ import annotations
+
+import concurrent.futures as cf
+import hashlib
+import json
+import os
+import sys
+import threading
+
+import numpy as np
+
+from prompt_card.scoring import observable_axes as OA
+from prompt_card.scoring.embedding import FastEmbedder
+
+HERE = os.path.dirname(__file__)
+ROOT = os.path.dirname(HERE)
+VP = os.path.join(ROOT, "prompt_card", "training", "validation_pool")
+CACHE = os.path.join(HERE, "_cache", "preds_8b.jsonl")
+
+TECH = ["zero_shot_role", "few_shot", "thought_generation", "decomposition", "self_criticism", "flipped"]
+IQ = ["goal_stated", "specific_specification"]
+CE = ["skepticism", "rebuttal", "source_request", "independent_verification", "re_questioning"]
+
+
+# ---------------- pure metrics (no deps) ----------------
+
+def confusion(y_true, y_pred, labels):
+    idx = {l: i for i, l in enumerate(labels)}
+    m = [[0] * len(labels) for _ in labels]
+    for t, p in zip(y_true, y_pred):
+        m[idx[t]][idx[p]] += 1
+    return m
+
+
+def cohen_kappa(y_true, y_pred):
+    """Cohen's kappa. Returns None when undefined (empty, or both raters use one identical label
+    → chance agreement is 1, kappa degenerate). Caller reports those as N/A."""
+    n = len(y_true)
+    if n == 0:
+        return None
+    labels = sorted(set(y_true) | set(y_pred))
+    if len(labels) == 1:
+        return None  # degenerate: everything one class (e.g. an all-negative sparse category)
+    m = confusion(y_true, y_pred, labels)
+    po = sum(m[i][i] for i in range(len(labels))) / n
+    rows = [sum(r) for r in m]
+    cols = [sum(m[i][j] for i in range(len(labels))) for j in range(len(labels))]
+    pe = sum((rows[i] / n) * (cols[i] / n) for i in range(len(labels)))
+    if pe >= 1.0:
+        return None
+    return (po - pe) / (1 - pe)
+
+
+def binary_counts(y_true, y_pred):
+    tp = sum(1 for t, p in zip(y_true, y_pred) if t and p)
+    fp = sum(1 for t, p in zip(y_true, y_pred) if not t and p)
+    fn = sum(1 for t, p in zip(y_true, y_pred) if t and not p)
+    tn = sum(1 for t, p in zip(y_true, y_pred) if not t and not p)
+    return dict(tp=tp, fp=fp, fn=fn, tn=tn)
+
+
+def prf(c):
+    tp, fp, fn = c["tp"], c["fp"], c["fn"]
+    p = tp / (tp + fp) if tp + fp else None
+    r = tp / (tp + fn) if tp + fn else None
+    f1 = (2 * p * r / (p + r)) if (p and r) else None
+    return p, r, f1
+
+
+# ---------------- cached concurrent 8B ----------------
+
+class CachedClient:
+    """Wraps any `generate(prompt)->str` client with a sha1-keyed disk cache + thread pool."""
+
+    def __init__(self, base, workers=8):
+        self.base = base
+        self.workers = workers
+        self._lock = threading.Lock()
+        self.cache = {}
+        if os.path.exists(CACHE):
+            for line in open(CACHE):
+                try:
+                    d = json.loads(line)
+                    self.cache[d["k"]] = d["v"]
+                except Exception:
+                    pass
+        self.hits = 0
+        self.misses = 0
+
+    @staticmethod
+    def _key(prompt):
+        return hashlib.sha1(prompt.encode("utf-8")).hexdigest()
+
+    def _persist(self, k, v):
+        with self._lock:
+            with open(CACHE, "a") as f:
+                f.write(json.dumps({"k": k, "v": v}, ensure_ascii=False) + "\n")
+
+    def run_all(self, prompts):
+        """Return {prompt: response} for a list of prompts, using cache + concurrency."""
+        uniq = list(dict.fromkeys(prompts))
+        todo = [p for p in uniq if self._key(p) not in self.cache]
+        self.hits += len(uniq) - len(todo)
+
+        def work(p):
+            v = self.base.generate(p)
+            k = self._key(p)
+            self.cache[k] = v
+            self._persist(k, v)
+            return p
+
+        if todo:
+            with cf.ThreadPoolExecutor(max_workers=self.workers) as ex:
+                for i, _ in enumerate(ex.map(work, todo), 1):
+                    self.misses += 1
+                    if i % 50 == 0:
+                        print(f"  ... {i}/{len(todo)} new 8B calls", flush=True)
+        return {p: self.cache[self._key(p)] for p in uniq}
+
+
+# ---------------- data loading ----------------
+
+def load_convs():
+    convs = {}
+    for fn in ("pool.jsonl", "supplement.jsonl"):
+        for line in open(os.path.join(VP, fn)):
+            c = json.loads(line)
+            convs[c["id"]] = c
+    return convs
+
+
+def user_turns(conv):
+    return [t["text"] for t in conv["turns"] if t["role"] == "user"]
+
+
+def load_gt():
+    return [json.loads(l) for l in open(os.path.join(VP, "ground_truth.jsonl"))]
+
+
+# ---------------- prediction assembly ----------------
+
+def build_prompts(gt, convs, embedder, focus_tmax=0.70, builders=OA,
+                  axes=("technique", "input_quality", "interaction", "focus")):
+    """Return (prompts, plan, focus_geom). `builders` supplies build_*_prompt (swap for prompt versions);
+    `axes` selects which axes to assemble (lets the cascade re-run only changed axes)."""
+    prompts, plan = [], []
+    focus_geom = {}  # conv_id -> list of (i, cosine) for every adjacent user-turn boundary
+    for r in gt:
+        cid = r["id"]
+        ut = user_turns(convs[cid])
+        if "technique" in axes:
+            for row in r["technique"]:
+                i = int(row["turn"][1:]) - 1
+                p = builders.build_technique_prompt(ut[i])
+                prompts.append(p); plan.append(("technique", cid, row["turn"], p))
+        if "input_quality" in axes:
+            for row in r["input_quality"]:
+                i = int(row["turn"][1:]) - 1
+                p = builders.build_input_quality_prompt(ut[i])
+                prompts.append(p); plan.append(("input_quality", cid, row["turn"], p))
+        if "interaction" in axes:
+            for row in r["interaction"]:
+                i = int(row["turn"][1:]) - 1
+                p = builders.build_interaction_prompt(ut[i - 1], ut[i])
+                prompts.append(p); plan.append(("interaction", cid, row["turn"], p))
+        # focus: embedder geometry over ALL adjacent boundaries; 8B only on cos<focus_tmax
+        if "focus" in axes and len(ut) >= 2:
+            vecs = embedder.embed(ut)
+            geom = []
+            for i in range(len(ut) - 1):
+                cos = float(np.dot(vecs[i], vecs[i + 1]))
+                geom.append((i, cos))
+                if cos < focus_tmax:
+                    p = builders.build_focus_boundary_prompt(ut[i], ut[i + 1])
+                    prompts.append(p); plan.append(("focus", cid, f"U{i+1}→U{i+2}", p))
+            focus_geom[cid] = geom
+    return prompts, plan, focus_geom
+
+
+# ---------------- axis scoring ----------------
+
+def score_binary_axis(gt, responses, plan_index, axis, fields):
+    """For a per-turn multi-binary axis (technique/input_quality): return per-field (yt,yp) + parse fails."""
+    per = {f: ([], []) for f in fields}
+    parse_fail = 0
+    gtmap = {r["id"]: r for r in gt}
+    key = "types" if axis in ("technique", "critical") else "features"
+    for (ax, cid, turn, prompt) in plan_index[axis]:
+        row = next(rr for rr in gtmap[cid][axis] if rr["turn"] == turn)
+        pred = OA.parse(responses[prompt], fields)
+        if pred is None:
+            parse_fail += 1
+            pred = {f: False for f in fields}
+        for f in fields:
+            per[f][0].append(int(f in row[key]))
+            per[f][1].append(int(bool(pred.get(f))))
+    return per, parse_fail
+
+
+def score_interaction(gt, responses, plan_index):
+    yt, yp = [], []
+    parse_fail = 0
+    gtmap = {r["id"]: r for r in gt}
+    for (ax, cid, turn, prompt) in plan_index["interaction"]:
+        row = next(rr for rr in gtmap[cid]["interaction"] if rr["turn"] == turn)
+        pred = OA.parse(responses[prompt], ("refinement_attempt",))
+        if pred is None:
+            parse_fail += 1
+            pred = {}
+        yt.append(int(bool(row["refinement"])))
+        yp.append(int(bool(pred.get("refinement_attempt"))))
+    return yt, yp, parse_fail
+
+
+def score_focus(gt, responses, plan_index, focus_geom, threshold):
+    """Binary shift vs not over EVERY adjacent boundary; embedder gates 8B at `threshold`.
+    Also returns embedder recall on GT topic_shift boundaries and a 3-class tally on co-identified."""
+    rel_pred = {(cid, turn): OA.parse(responses[p], ("relation",)) for (ax, cid, turn, p) in plan_index["focus"]}
+    gtmap = {r["id"]: r for r in gt}
+    yt, yp = [], []
+    gt_shift_total = recall_hit = 0
+    cand_total = 0
+    three = {}  # (gt_rel, pred_rel) -> count on co-identified candidates
+    for r in gt:
+        cid = r["id"]
+        gt_rel = {f"U{c['a'][1:]}→U{c['b'][1:]}" if False else f"{c['a']}→{c['b']}": c["relation"] for c in r["focus"]}
+        for (i, cos) in focus_geom.get(cid, []):
+            bkey = f"U{i+1}→U{i+2}"
+            gtr = gt_rel.get(bkey)
+            is_gt_shift = (gtr == "topic_shift")
+            yt.append(int(is_gt_shift))
+            is_cand = cos < threshold
+            cand_total += int(is_cand)
+            pred_shift = 0
+            if is_cand:
+                pr = rel_pred.get((cid, bkey)) or {}
+                prel = pr.get("relation")
+                pred_shift = int(prel == "topic_shift")
+                if gtr is not None:  # co-identified: both GT and embedder flagged this boundary
+                    three[(gtr, prel)] = three.get((gtr, prel), 0) + 1
+            yp.append(pred_shift)
+            if is_gt_shift:
+                gt_shift_total += 1
+                recall_hit += int(is_cand)
+    recall = recall_hit / gt_shift_total if gt_shift_total else None
+    return yt, yp, dict(recall=recall, gt_shift=gt_shift_total, cand_total=cand_total, three=three)
+
+
+# ---------------- OOD ----------------
+
+def ood_report(gt, convs, embedder):
+    ids = [r["id"] for r in gt]
+    centroids = []
+    for cid in ids:
+        ut = user_turns(convs[cid])
+        v = embedder.embed(ut)
+        c = v.mean(axis=0)
+        centroids.append(c / (np.linalg.norm(c) + 1e-9))
+    M = np.array(centroids)
+    glob = M.mean(axis=0); glob /= np.linalg.norm(glob) + 1e-9
+    sims = M @ glob
+    mu, sd = float(sims.mean()), float(sims.std())
+    flagged = [(ids[i], float(sims[i])) for i in range(len(ids)) if sims[i] < mu - 2 * sd]
+    return dict(mu=mu, sd=sd, flagged=sorted(flagged, key=lambda x: x[1]))
+
+
+# ---------------- main ----------------
+
+def _fmt_k(k):
+    return "N/A" if k is None else f"{k:+.3f}"
+
+
+def main(limit=None, workers=8):
+    base_url = os.environ.get("OPENBMB_BASE_URL")
+    token = os.environ.get("OPENBMB_TOKEN")
+    if not base_url or not token:
+        print("ERROR: set OPENBMB_BASE_URL and OPENBMB_TOKEN in the environment (shared hackathon key).",
+              file=sys.stderr)
+        sys.exit(2)
+    from prompt_card.llm.minicpm import MiniCPMClient
+    base = MiniCPMClient(base_url, token)
+
+    gt = load_gt()
+    if limit:
+        gt = gt[:limit]
+    convs = load_convs()
+    embedder = FastEmbedder()
+    print(f"[measure] {len(gt)} convs; embedding + assembling prompts ...", flush=True)
+    prompts, plan, focus_geom = build_prompts(gt, convs, embedder)
+    plan_index = {}
+    for item in plan:
+        plan_index.setdefault(item[0], []).append(item)
+    print(f"[measure] {len(prompts)} prompts "
+          f"(tech {len(plan_index.get('technique',[]))}, iq {len(plan_index.get('input_quality',[]))}, "
+          f"inter {len(plan_index.get('interaction',[]))}, focus {len(plan_index.get('focus',[]))})", flush=True)
+
+    client = CachedClient(base, workers=workers)
+    responses = client.run_all(prompts)
+    print(f"[measure] 8B done (cache hits {client.hits}, new {client.misses})", flush=True)
+
+    # axis scoring
+    tech_per, tech_fail = score_binary_axis(gt, responses, plan_index, "technique", TECH)
+    iq_per, iq_fail = score_binary_axis(gt, responses, plan_index, "input_quality", IQ)
+    inter_yt, inter_yp, inter_fail = score_interaction(gt, responses, plan_index)
+    # focus threshold sweep
+    sweep = {}
+    for T in [0.40, 0.45, 0.50, 0.55, 0.60, 0.65, 0.70]:
+        yt, yp, info = score_focus(gt, responses, plan_index, focus_geom, T)
+        sweep[T] = (cohen_kappa(yt, yp), info["recall"], info["cand_total"], yt, yp, info)
+    # pick T maximizing kappa (tie-break higher recall)
+    bestT = max(sweep, key=lambda T: (sweep[T][0] if sweep[T][0] is not None else -9, sweep[T][1] or 0))
+    fk, frec, fcand, fyt, fyp, finfo = sweep[bestT]
+
+    write_results(gt, tech_per, tech_fail, iq_per, iq_fail, inter_yt, inter_yp, inter_fail,
+                  sweep, bestT, ood_report(gt, convs, embedder))
+    print("[measure] wrote eval/measurement_results.md", flush=True)
+
+
+def write_results(gt, tech_per, tech_fail, iq_per, iq_fail, inter_yt, inter_yp, inter_fail,
+                  sweep, bestT, ood):
+    L = ["# Phase-2 measurement — base MiniCPM4.1-8B + bge-small + rules vs ground truth", ""]
+    L.append(f"Validation set: {len(gt)} conversations. Metric: Cohen's κ (per axis & per category). "
+             "Sparse categories (no positives) → κ undefined → **N/A**. Confusion as [TN FP / FN TP].")
+    L.append("")
+
+    def axis_block(title, scope, per, fail, fields):
+        L.append(f"## {title}")
+        L.append(f"_{scope}_  ·  parse failures: {fail}")
+        L.append("| category | κ | TN | FP | FN | TP | precision | recall | f1 |")
+        L.append("|---|---|---|---|---|---|---|---|---|")
+        for f in fields:
+            yt, yp = per[f]
+            k = cohen_kappa(yt, yp); c = binary_counts(yt, yp); p, r, f1 = prf(c)
+            note = "  ← N/A (0 positives)" if sum(yt) == 0 else ""
+            L.append(f"| {f} | {_fmt_k(k)}{note} | {c['tn']} | {c['fp']} | {c['fn']} | {c['tp']} | "
+                     f"{'-' if p is None else f'{p:.2f}'} | {'-' if r is None else f'{r:.2f}'} | "
+                     f"{'-' if f1 is None else f'{f1:.2f}'} |")
+        # axis-level "any" binary
+        anyt = [int(any(per[f][0][j] for f in fields)) for j in range(len(per[fields[0]][0]))]
+        anyp = [int(any(per[f][1][j] for f in fields)) for j in range(len(per[fields[0]][1]))]
+        L.append(f"\n**Axis-level (any {title.split()[0].lower()} present): κ = {_fmt_k(cohen_kappa(anyt, anyp))}** "
+                 f"(n={len(anyt)} turns, {sum(anyt)} positive).")
+        L.append("")
+
+    axis_block("Technique (6 binary)", "first ≤3 user turns/conv", tech_per, tech_fail, TECH)
+    axis_block("Input Quality (2 binary)", "first ≤3 user turns/conv", iq_per, iq_fail, IQ)
+
+    L.append("## Interaction (refinement_attempt)")
+    L.append(f"_all follow-up turns_  ·  parse failures: {inter_fail}")
+    k = cohen_kappa(inter_yt, inter_yp); c = binary_counts(inter_yt, inter_yp); p, r, f1 = prf(c)
+    L.append(f"- **κ = {_fmt_k(k)}**  ·  n={len(inter_yt)}, positives={sum(inter_yt)}")
+    L.append(f"- confusion [TN {c['tn']} · FP {c['fp']} / FN {c['fn']} · TP {c['tp']}]  ·  "
+             f"P {'-' if p is None else f'{p:.2f}'} R {'-' if r is None else f'{r:.2f}'} F1 {'-' if f1 is None else f'{f1:.2f}'}")
+    L.append("")
+
+    L.append("## Focus (topic_shift detection, embedder-gated)")
+    L.append("Binary shift-vs-not over **every** adjacent user-turn boundary; the embedder gates which "
+             "boundaries the 8B classifies (cosine < T). Threshold swept; κ-maximizing T selected.")
+    L.append("| T | κ (shift) | embedder recall@T (on GT shifts) | candidates flagged |")
+    L.append("|---|---|---|---|")
+    for T in sorted(sweep):
+        k, rec, cand, *_ = sweep[T]
+        mark = " ← selected" if T == bestT else ""
+        L.append(f"| {T:.2f} | {_fmt_k(k)} | {'-' if rec is None else f'{rec:.2f}'} | {cand} |{mark}")
+    fk, frec, fcand, fyt, fyp, finfo = sweep[bestT]
+    cc = binary_counts(fyt, fyp)
+    rec_str = "-" if frec is None else f"{frec:.2f}"
+    hits = "" if frec is None else f" ({int(round(frec * finfo['gt_shift']))}/{finfo['gt_shift']} GT shifts flagged)"
+    L.append("")
+    L.append(f"**Selected T={bestT:.2f}: κ = {_fmt_k(fk)}**, embedder boundary-recall = {rec_str}{hits}. "
+             f"Confusion [TN {cc['tn']} · FP {cc['fp']} / FN {cc['fn']} · TP {cc['tp']}].")
+    if finfo["three"]:
+        L.append("\n3-class agreement on co-identified candidates (GT rel → 8B rel):")
+        for (g, p2), n in sorted(finfo["three"].items(), key=lambda x: -x[1]):
+            L.append(f"- {g} → {p2}: {n}")
+    L.append("\n_Embedder boundary-recall is the model-agnostic metric Kim requested: of the boundaries Kim "
+             "hand-identified as topic_shift, how many the production embedder surfaces as candidates._")
+    L.append("")
+
+    L.append("## OOD report (validation-set self-check)")
+    L.append(f"Per-conv mean-user-turn embedding vs the set centroid: μ_sim={ood['mu']:.3f}, σ={ood['sd']:.3f}. "
+             f"Flagged (sim < μ−2σ, i.e. atypical conversations): {len(ood['flagged'])}.")
+    for cid, s in ood["flagged"][:15]:
+        L.append(f"- `{cid[:12]}`  sim={s:.3f}")
+    L.append("\n_In production this same distance flags user uploads far from the validation distribution "
+             "(low-confidence / out-of-scope scoring)._")
+    L.append("")
+
+    with open(os.path.join(HERE, "measurement_results.md"), "w") as f:
+        f.write("\n".join(L) + "\n")
+
+
+if __name__ == "__main__":
+    import argparse
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--limit", type=int, default=None)
+    ap.add_argument("--workers", type=int, default=8)
+    args = ap.parse_args()
+    main(limit=args.limit, workers=args.workers)

eval/measure_lora.py

@@ -0,0 +1,113 @@
+"""Phase 3 Step 5 — re-measure a trained LoRA on the validation set and compare to base-8B.
+
+Builds the SAME validation prompts used for the base measurement (apples-to-apples), runs them through
+the Modal base+adapter batch function, parses, and computes Cohen's κ vs the Kim ground truth.
+
+Run (from a Modal-authed machine):  python -m eval.measure_lora interaction
+                                     python -m eval.measure_lora critical
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+from eval import kappa as K
+from eval.prompts_v3 import build_interaction_prompt
+from eval.step_critical import build_critical_prompt, CE
+from eval.step_c import goal_prompt, decomp_prompt
+from prompt_card.scoring import observable_axes as OA
+
+# base-8B κ per axis/category (from the cascade) for the lock-vs-keep comparison
+BASE_KAPPA = {"interaction": 0.320, "goal_stated": 0.226, "decomposition": 0.261, "re_questioning": 0.058}
+# single-feature axes: (prompt builder over the scored turns, the feature name, which GT axis holds it)
+SINGLE = {"goal_stated": (goal_prompt, "goal_stated", "input_quality"),
+          "decomposition": (decomp_prompt, "decomposition", "technique")}
+
+
+def build(axis, gt, convs):
+    prompts, truth = [], []
+    for r in gt:
+        conv = convs[r["id"]]
+        ut = OA._user_turns(conv)
+        if axis == "interaction":
+            for row in r["interaction"]:
+                i = int(row["turn"][1:]) - 1
+                prompts.append(build_interaction_prompt(ut[i - 1], ut[i]))
+                truth.append(set(["refinement_attempt"]) if row["refinement"] else set())
+        elif axis == "critical":
+            for row in r["critical"]:
+                i = int(row["turn"][1:]) - 1
+                prompts.append(build_critical_prompt(OA._prev_assistant(conv, i), ut[i]))
+                truth.append(set(row["types"]))
+        elif axis in SINGLE:
+            builder, feat, gt_axis = SINGLE[axis]
+            key = "features" if gt_axis == "input_quality" else "types"
+            for row in r[gt_axis]:
+                i = int(row["turn"][1:]) - 1
+                prompts.append(builder(ut[i]))
+                truth.append({feat} if feat in row[key] else set())
+    return prompts, truth
+
+
+def main(axis, adapter=""):
+    from modal_eval_lora import app, evaluate
+    gt = K.load_gt(); convs = K.load_convs()
+    prompts, truth = build(axis, gt, convs)
+    tag = f"{axis} (adapter={adapter or axis})"
+    print(f"[lora-eval] {tag}: {len(prompts)} prompts -> Modal base+adapter ...", flush=True)
+    with app.run():
+        resp = evaluate.remote(axis, prompts, adapter=adapter)
+
+    if axis == "interaction":
+        fields = ("refinement_attempt",)
+    elif axis in SINGLE:
+        fields = (SINGLE[axis][1],)
+    else:
+        fields = CE
+    fail = 0
+    preds = []
+    for rtext in resp:
+        d = OA.parse(rtext, fields)
+        if d is None:
+            fail += 1; d = {}
+        preds.append({f for f in fields if d.get(f)})
+
+    print(f"[lora-eval] {axis}: parse_fail {fail}/{len(prompts)}")
+    if axis == "interaction" or axis in SINGLE:
+        feat = "refinement_attempt" if axis == "interaction" else SINGLE[axis][1]
+        yt = [int(feat in s) for s in truth]
+        yp = [int(feat in s) for s in preds]
+        k = K.cohen_kappa(yt, yp)
+        base_k = BASE_KAPPA[axis]
+        print(f"  {axis} κ: base {base_k:+.3f} -> LoRA {k:+.3f}  [{K.binary_counts(yt, yp)}]")
+        result = {"axis": axis, "lora_kappa": k, "base_kappa": base_k}
+    else:
+        base = json.load(open(os.path.join(os.path.dirname(__file__), "_cache", "critical.json")))
+        per = {}
+        for t in CE:
+            yt = [int(t in s) for s in truth]; yp = [int(t in s) for s in preds]
+            per[t] = K.cohen_kappa(yt, yp)
+            print(f"  {t:24} base {base['per_type'][t]:+.3f} -> LoRA {(per[t] if per[t] is not None else float('nan')):+.3f}")
+        valid = [v for v in per.values() if v is not None]
+        head = sum(valid) / len(valid)
+        print(f"  per-type mean: base {base['headline']:+.3f} -> LoRA {head:+.3f}")
+        result = {"axis": axis, "lora_per_type": per, "lora_headline": head, "base_headline": base["headline"]}
+
+    result["adapter"] = adapter or axis
+    json.dump(result, open(os.path.join(os.path.dirname(__file__), "_cache", f"lora_{adapter or axis}.json"), "w"),
+              indent=1, default=str)
+    print(f"  decision: {'LOCK LoRA' if _wins(result, axis) else 'KEEP base'}")
+
+
+def _wins(result, axis):
+    # Phase-5 rule: LoRA κ ≥ base + 0.1 → use LoRA.
+    if "lora_kappa" in result:
+        return (result["lora_kappa"] or 0) >= (result["base_kappa"] or 0) + 0.1
+    return (result["lora_headline"] or 0) >= (result["base_headline"] or 0) + 0.1
+
+
+if __name__ == "__main__":
+    # usage: python -m eval.measure_lora <axis> [adapter_dir_name]
+    main(sys.argv[1] if len(sys.argv) > 1 else "interaction",
+         sys.argv[2] if len(sys.argv) > 2 else "")

eval/prompts_v2.py

@@ -0,0 +1,112 @@
+"""STEP A prompt rewrites (v2). Same field names/JSON shape as schemas.py so eval.kappa.parse is
+unchanged — only the instruction text changes (→ new cache keys, clean A/B vs baseline).
+
+Each prompt now carries: (1) the exact GT definition from observable_schemas_review.md, (2) positive
+AND negative examples chosen to counter the measured base-8B biases, (3) a strict JSON template.
+
+Measured baseline biases this targets:
+  goal_stated  R0.84/P0.12  → model too LIBERAL → make strict, show task-only negatives
+  specific_specification R0.14/P0.88 → model too STRICT → make liberal, show what counts
+  refinement   R0.85/P0.31  → model over-fires → tighten, default NO, show new-subtask negatives
+  few_shot 0/7, decomposition 0/17 → model too strict → concrete positives in WildChat style
+"""
+from __future__ import annotations
+
+
+def _ask(instr: str, template: str, payload_label: str, payload: str) -> str:
+    return (f"{instr}\n\nReturn ONLY a JSON object of EXACTLY this shape (no prose, no markdown):\n"
+            f"{template}\n\n{payload_label}:\n\"\"\"\n{payload}\n\"\"\"\nJSON:")
+
+
+# ---------------- Technique ----------------
+
+_TECH_TEMPLATE = ('{"zero_shot_role": <bool>, "few_shot": <bool>, "thought_generation": <bool>, '
+                  '"decomposition": <bool>, "self_criticism": <bool>, "flipped": <bool>}')
+
+
+def build_technique_prompt(user_msg: str) -> str:
+    instr = (
+        "Detect which prompting techniques THIS single user message uses. Mark true only when the feature "
+        "is genuinely present in the user's own words. Judge each independently.\n"
+        "- zero_shot_role: the user assigns a role/persona to the AI. ▸ 'You are a senior tax lawyer…', "
+        "'Act as a Linux terminal'. ✗ the user describing THEMSELF ('I'm a law student') — that is NOT this.\n"
+        "- few_shot: the user gives one or more concrete in-context examples / a sample to imitate. "
+        "▸ 'Format like: Input: 3 → Output: nine', 'Here's an example email: <text>. Write three more like it', "
+        "'Translate these: cat=gato, dog=perro, bird=?'. ✗ 'for example, keep it short' (no actual exemplar).\n"
+        "- thought_generation: the user explicitly asks the AI to REASON/show its thinking. ▸ 'think step by "
+        "step', 'show your reasoning', 'explain your reasoning before answering'. ✗ 'write a step-by-step guide' "
+        "(that asks for step-by-step OUTPUT, not reasoning).\n"
+        "- decomposition: the user breaks the request into multiple ordered/numbered sub-tasks or parts. "
+        "▸ 'First summarize it, then list risks, then propose fixes', '1) do X 2) do Y 3) do Z', a numbered "
+        "list of distinct requirements. ✗ a single undifferentiated ask.\n"
+        "- self_criticism: the user asks the AI to check/critique/verify ITS OWN answer. ▸ 'double-check your "
+        "answer for errors', 'review what you wrote and flag mistakes'. ✗ 'check my code' (the user's work, not the AI's).\n"
+        "- flipped: the user asks the AI to ask THEM clarifying questions first. ▸ 'ask me any questions you "
+        "need before starting'. ✗ the user simply asking a question."
+    )
+    return _ask(instr, _TECH_TEMPLATE, "User message", user_msg)
+
+
+# ---------------- Input Quality ----------------
+
+_IQ_TEMPLATE = '{"goal_stated": <bool>, "specific_specification": <bool>}'
+
+
+def build_input_quality_prompt(user_msg: str) -> str:
+    instr = (
+        "Detect two INDEPENDENT features of this single user message. Apply each rule exactly — they have "
+        "deliberately different strictness.\n\n"
+        "goal_stated — STRICT. True ONLY if the user states a WHY / purpose / intended outcome, not merely the "
+        "task. Look for 'so that', 'for', 'because', 'to help me…', 'I'm trying to…', a use-for.\n"
+        "  ▸ TRUE: 'Dedupe my survey data FOR ANALYSIS', 'I'm trying to speed up my API, help me add caching', "
+        "'Summarize this so I can brief my team'.\n"
+        "  ✗ FALSE (task only, no why): 'Write a sorting function', 'Dedupe this list', 'Translate this email', "
+        "'Give me 10 marketing ideas'. If you only see WHAT to do and not WHY, it is FALSE.\n\n"
+        "specific_specification — LIBERAL. True if the message contains ANY concrete detail of ANY kind: a "
+        "number, version, named tool/library/framework, file/data/schema, an explicit constraint, OR the user's "
+        "OWN background/role/skill level. Be generous — a single concrete detail is enough.\n"
+        "  ▸ TRUE: 'Python 3.11, under 100ms', 'I'm a beginner', 'here's my CSV with columns date,revenue', "
+        "'in React with TypeScript', 'a 1000-word post', 'using pandas'.\n"
+        "  ✗ FALSE only when fully vague: 'make it fast', 'help me with my code', 'write something good'.\n"
+        "Note: a role assigned to the AI ('you are an expert') is NOT input quality — ignore it here."
+    )
+    return _ask(instr, _IQ_TEMPLATE, "User message", user_msg)
+
+
+# ---------------- Interaction ----------------
+
+_INT_TEMPLATE = '{"refinement_attempt": <bool>}'
+
+
+def build_interaction_prompt(prev_user: str, this_user: str) -> str:
+    instr = (
+        "Did the user, in THIS follow-up, attempt to REFINE the PRIOR request to reshape the SAME output? "
+        "Detect the attempt only (not whether it worked). Default to FALSE unless it clearly reshapes the "
+        "previous output.\n"
+        "  ▸ TRUE (reshapes the same deliverable): 'make it shorter', 'use Python instead', 'actually I meant X', "
+        "'more formal tone', 'that's wrong, fix the loop', 'redo it without the intro'.\n"
+        "  ✗ FALSE — a NEW request or NEW sub-task on the same project: 'now write tests for it', 'also add a "
+        "README', 'next, explain how it works', a brand-new question, 'thanks', or simply accepting the output. "
+        "Producing a NEW artifact is NOT a refinement; only editing/adjusting the PREVIOUS one is."
+    )
+    payload = f"PRIOR user request:\n{prev_user}\n\nTHIS follow-up:\n{this_user}"
+    return _ask(instr, _INT_TEMPLATE, "Turns", payload)
+
+
+# ---------------- Focus boundary ----------------
+
+_FOCUS_TEMPLATE = '{"relation": "same_topic" | "related_sub_task" | "topic_shift"}'
+
+
+def build_focus_boundary_prompt(turn_a: str, turn_b: str) -> str:
+    instr = (
+        "Classify how the SECOND user turn relates to the FIRST. Pick exactly one.\n"
+        "- same_topic: clearly the same subject/question. ▸ 'explain binary search' → 'what's its time complexity?'\n"
+        "- related_sub_task: a different sub-task that serves the SAME concrete project/artifact/goal (same "
+        "project, NOT merely the same field). ▸ 'write the Flask endpoint' → 'now write pytest tests for it'.\n"
+        "- topic_shift: a genuinely unrelated change, OR two separate questions that merely share a domain. "
+        "▸ 'debug my Python loop' → 'what's a good carbonara recipe?'  ▸ 'explain recursion' → 'explain dynamic "
+        "programming' (same field, different question = topic_shift)."
+    )
+    payload = f"FIRST user turn:\n{turn_a}\n\nSECOND user turn:\n{turn_b}"
+    return _ask(instr, _FOCUS_TEMPLATE, "Turns", payload)

eval/prompts_v3.py

@@ -0,0 +1,51 @@
+"""STEP A iter 2 (v3). Keeps v2's winners unchanged (technique, specific_specification, focus → cache
+reused) and only re-balances the two v2 over-corrections:
+  - interaction: v2 over-tightened (recall 0.85→0.16). Broaden TRUE to corrections / added constraints /
+    redo-differently while still excluding genuinely NEW deliverables.
+  - goal_stated: v2 over-strict (recall 0.84→0.22). Accept implied purpose / use / audience, not just
+    explicit 'so that'.
+specific_specification keeps the v2 (liberal) wording verbatim so its cache entries are reused.
+"""
+from __future__ import annotations
+
+from eval.prompts_v2 import build_technique_prompt, build_focus_boundary_prompt, _ask  # reused as-is
+
+_IQ_TEMPLATE = '{"goal_stated": <bool>, "specific_specification": <bool>}'
+_INT_TEMPLATE = '{"refinement_attempt": <bool>}'
+
+
+def build_input_quality_prompt(user_msg: str) -> str:
+    instr = (
+        "Detect two INDEPENDENT features of this single user message. Apply each rule exactly.\n\n"
+        "goal_stated. True if the user conveys a PURPOSE, use, audience, or reason — stated OR clearly implied "
+        "by context — for the request. False if it is only the bare task with no purpose/use/audience.\n"
+        "  ▸ TRUE: 'summarize this for my class', 'I'm building a budgeting app, write the DB schema', "
+        "'help me debug so the tests pass', 'dedupe my survey data for analysis', 'a cover letter for a "
+        "data-analyst job', 'explain it so a beginner gets it'.\n"
+        "  ✗ FALSE (task only, no why/use/audience): 'write a poem', 'fix this code', 'list 10 startup names', "
+        "'translate this', 'write a sorting function'.\n\n"
+        "specific_specification — LIBERAL. True if the message contains ANY concrete detail of ANY kind: a "
+        "number, version, named tool/library/framework, file/data/schema, an explicit constraint, OR the user's "
+        "OWN background/role/skill level. Be generous — a single concrete detail is enough.\n"
+        "  ▸ TRUE: 'Python 3.11, under 100ms', 'I'm a beginner', 'here's my CSV with columns date,revenue', "
+        "'in React with TypeScript', 'a 1000-word post', 'using pandas'.\n"
+        "  ✗ FALSE only when fully vague: 'make it fast', 'help me with my code', 'write something good'.\n"
+        "A role assigned to the AI ('you are an expert') is NOT input quality — ignore it here."
+    )
+    return _ask(instr, _IQ_TEMPLATE, "User message", user_msg)
+
+
+def build_interaction_prompt(prev_user: str, this_user: str) -> str:
+    instr = (
+        "Did the user, in THIS follow-up, attempt to REFINE the PRIOR request — i.e. adjust, correct, or "
+        "re-direct it to change the SAME deliverable? Detect the attempt only (not whether it worked).\n"
+        "  ▸ TRUE (modifies the same output): shorten/lengthen, change tone/format/language, corrections "
+        "('that's wrong', 'fix the loop', 'no, I meant…'), add or drop a constraint on the same output, "
+        "'redo it but more formal', 'try again', 'make the intro punchier'. If the follow-up is clearly "
+        "changing the previous answer, choose TRUE.\n"
+        "  ✗ FALSE only for a genuinely NEW deliverable or NEW question: 'now write tests for it', 'also build "
+        "a UI', a fresh unrelated ask, or mere acceptance / 'thanks'. Producing a NEW artifact is not a "
+        "refinement; adjusting the PREVIOUS one is."
+    )
+    payload = f"PRIOR user request:\n{prev_user}\n\nTHIS follow-up:\n{this_user}"
+    return _ask(instr, _INT_TEMPLATE, "Turns", payload)

eval/prompts_v4.py

@@ -0,0 +1,128 @@
+"""STEP B (few-shot). Best-per-axis prompts (technique=v2, input_quality=v2, interaction=v3, focus=v2)
+PLUS a hand-authored few-shot block per axis. All few-shot examples are SYNTHETIC/illustrative and
+deliberately DISJOINT from the 159 validation conversations — no train-on-test leakage.
+
+Targets the STEP-A residual failures: few_shot (0/7), decomposition (0.26), goal_stated (oscillating
+~0.2), interaction false positives.
+"""
+from __future__ import annotations
+
+from eval.prompts_v2 import build_focus_boundary_prompt as _focus_v2  # focus unchanged (already OK)
+
+# import the v2/v3 instruction bodies by rebuilding with an examples block appended.
+from eval.prompts_v2 import _ask, _TECH_TEMPLATE, _IQ_TEMPLATE
+from eval.prompts_v3 import _INT_TEMPLATE
+
+
+# ---------------- Technique (v2 defs + few-shot) ----------------
+
+_TECH_FEWSHOT = (
+    "\n\nExamples (message → JSON):\n"
+    "'You are an experienced chef. Suggest a tasting menu.' → "
+    '{"zero_shot_role": true, "few_shot": false, "thought_generation": false, "decomposition": false, "self_criticism": false, "flipped": false}\n'
+    "'Rewrite in a friendly tone. Example: \"Submit the form.\" → \"Pop your details in!\" Now: \"Payment failed.\"' → "
+    '{"zero_shot_role": false, "few_shot": true, "thought_generation": false, "decomposition": false, "self_criticism": false, "flipped": false}\n'
+    "'Plan my week: first list my goals, then schedule them by priority, then add breaks.' → "
+    '{"zero_shot_role": false, "few_shot": false, "thought_generation": false, "decomposition": true, "self_criticism": false, "flipped": false}\n'
+    "'Solve this problem. Think step by step and show your reasoning.' → "
+    '{"zero_shot_role": false, "few_shot": false, "thought_generation": true, "decomposition": false, "self_criticism": false, "flipped": false}\n'
+    "'Write the function, then review your own code and flag any bugs.' → "
+    '{"zero_shot_role": false, "few_shot": false, "thought_generation": false, "decomposition": false, "self_criticism": true, "flipped": false}\n'
+    "'I want to plan a trip. Ask me whatever you need to know before suggesting anything.' → "
+    '{"zero_shot_role": false, "few_shot": false, "thought_generation": false, "decomposition": false, "self_criticism": false, "flipped": true}\n'
+    "'Write a step-by-step guide to baking bread.' → "
+    '{"zero_shot_role": false, "few_shot": false, "thought_generation": false, "decomposition": false, "self_criticism": false, "flipped": false}  (step-by-step OUTPUT, not reasoning; one undivided task)\n'
+    "'Translate to French, for example keep it natural.' → "
+    '{"zero_shot_role": false, "few_shot": false, "thought_generation": false, "decomposition": false, "self_criticism": false, "flipped": false}  (no actual exemplar)'
+)
+
+_TECH_INSTR = (
+    "Detect which prompting techniques THIS single user message uses. Mark true only when genuinely present "
+    "in the user's own words; judge each independently.\n"
+    "- zero_shot_role: assigns a role/persona to the AI ('You are a tax lawyer'). The user describing THEMSELF is not this.\n"
+    "- few_shot: the user gives a concrete in-context EXAMPLE / sample to imitate (input→output pair, a sample to copy).\n"
+    "- thought_generation: explicitly asks the AI to REASON / 'think step by step' / show reasoning (not 'write a step-by-step guide').\n"
+    "- decomposition: breaks the request into multiple ordered/numbered sub-tasks ('first… then… then…', '1) 2) 3)').\n"
+    "- self_criticism: asks the AI to check/critique/verify ITS OWN answer (not 'check my code').\n"
+    "- flipped: asks the AI to ask THEM clarifying questions first."
+)
+
+
+def build_technique_prompt(user_msg: str) -> str:
+    return _ask(_TECH_INSTR + _TECH_FEWSHOT, _TECH_TEMPLATE, "User message", user_msg)
+
+
+# ---------------- Input Quality (v2 defs + few-shot) ----------------
+
+_IQ_FEWSHOT = (
+    "\n\nExamples (message → JSON):\n"
+    "'Write a sorting function.' → {\"goal_stated\": false, \"specific_specification\": false}\n"
+    "'Write a sorting function in Python 3.11.' → {\"goal_stated\": false, \"specific_specification\": true}\n"
+    "'Summarize this paper so I can present it to my class.' → {\"goal_stated\": true, \"specific_specification\": false}\n"
+    "'I'm a beginner. Help me dedupe my CSV for analysis.' → {\"goal_stated\": true, \"specific_specification\": true}\n"
+    "'Draft a cover letter for a marketing internship.' → {\"goal_stated\": true, \"specific_specification\": false}\n"
+    "'Give me 10 startup names.' → {\"goal_stated\": false, \"specific_specification\": false}\n"
+    "'Make it better.' → {\"goal_stated\": false, \"specific_specification\": false}\n"
+    "'Optimize this query, it's slow on our 2M-row orders table.' → {\"goal_stated\": true, \"specific_specification\": true}"
+)
+
+_IQ_INSTR = (
+    "Detect two INDEPENDENT features of this single user message.\n"
+    "goal_stated: TRUE if the user conveys a PURPOSE / use / audience / reason (stated or clearly implied), "
+    "FALSE if it is only the bare task. specific_specification: LIBERAL — TRUE if ANY concrete detail is "
+    "present (number, version, named tool, file/data, constraint, or the user's OWN background), FALSE only "
+    "when fully vague. A role assigned to the AI is NOT input quality."
+)
+
+
+def build_input_quality_prompt(user_msg: str) -> str:
+    return _ask(_IQ_INSTR + _IQ_FEWSHOT, _IQ_TEMPLATE, "User message", user_msg)
+
+
+# ---------------- Interaction (v3 defs + few-shot) ----------------
+
+_INT_FEWSHOT = (
+    "\n\nExamples (prior → follow-up → JSON):\n"
+    "'write a poem about the sea' → 'make it rhyme' → {\"refinement_attempt\": true}\n"
+    "'summarize this article' → 'too long, give me 3 bullets' → {\"refinement_attempt\": true}\n"
+    "'translate to French' → 'actually, Spanish instead' → {\"refinement_attempt\": true}\n"
+    "'write the SQL for top customers' → 'also include their signup date' → {\"refinement_attempt\": true}\n"
+    "'write a poem about the sea' → 'now write one about mountains' → {\"refinement_attempt\": false}\n"
+    "'write the login function' → 'now write unit tests for it' → {\"refinement_attempt\": false}\n"
+    "'give me startup ideas' → 'thanks, that helps' → {\"refinement_attempt\": false}\n"
+    "'explain photosynthesis' → 'what about cellular respiration?' → {\"refinement_attempt\": false}"
+)
+
+_INT_INSTR = (
+    "Did the user, in THIS follow-up, REFINE the PRIOR request — adjust, correct, or re-direct it to change "
+    "the SAME deliverable? TRUE for shorten/lengthen, tone/format/language change, corrections ('that's "
+    "wrong', 'fix the loop', 'no, I meant…'), adding/dropping a constraint on the same output, or 'redo it "
+    "differently'. FALSE for a genuinely NEW deliverable or NEW question, or mere acceptance/'thanks'."
+)
+
+
+def build_interaction_prompt(prev_user: str, this_user: str) -> str:
+    payload = f"PRIOR user request:\n{prev_user}\n\nTHIS follow-up:\n{this_user}"
+    return _ask(_INT_INSTR + _INT_FEWSHOT, _INT_TEMPLATE, "Turns", payload)
+
+
+# ---------------- Focus (v2 + few-shot) ----------------
+
+_FOCUS_TEMPLATE = '{"relation": "same_topic" | "related_sub_task" | "topic_shift"}'
+_FOCUS_FEWSHOT = (
+    "\n\nExamples (first → second → JSON):\n"
+    "'explain binary search' → 'what's its time complexity?' → {\"relation\": \"same_topic\"}\n"
+    "'write the login API' → 'now write tests for it' → {\"relation\": \"related_sub_task\"}\n"
+    "'debug my python loop' → 'what's a good carbonara recipe?' → {\"relation\": \"topic_shift\"}\n"
+    "'explain recursion' → 'explain dynamic programming' → {\"relation\": \"topic_shift\"}"
+)
+_FOCUS_INSTR = (
+    "Classify how the SECOND user turn relates to the FIRST. same_topic = same subject/question. "
+    "related_sub_task = a different sub-task serving the SAME concrete project/artifact (same project, not "
+    "merely the same field). topic_shift = unrelated change, OR two separate questions sharing only a domain."
+)
+
+
+def build_focus_boundary_prompt(turn_a: str, turn_b: str) -> str:
+    payload = f"FIRST user turn:\n{turn_a}\n\nSECOND user turn:\n{turn_b}"
+    return _ask(_FOCUS_INSTR + _FOCUS_FEWSHOT, _FOCUS_TEMPLATE, "Turns", payload)

eval/step_c.py

@@ -0,0 +1,137 @@
+"""STEP C — per-feature dedicated prompts + self-consistency. Compares to the A+B best.
+
+C1: Input Quality as TWO single-feature binary prompts (goal_stated solo, specific_specification solo),
+    removing any trade-off the combined prompt forced. Deterministic, cached.
+C2: Self-consistency for Interaction — sample the v3 prompt 3× at temperature 0.6, majority vote
+    (variance reduction on the noisy axis). Uses the vLLM `n` param (one request → 3 completions).
+
+Run:  python -m eval.step_c
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+import requests
+
+from eval import kappa as K
+from eval.prompts_v2 import _ask
+from eval.prompts_v3 import build_interaction_prompt as int_v3
+
+_GOAL_T = '{"goal_stated": <bool>}'
+_SPEC_T = '{"specific_specification": <bool>}'
+
+
+def goal_prompt(msg):
+    instr = (
+        "Does THIS user message state a PURPOSE / use / audience / reason (a WHY) for the request — not just "
+        "the task itself? TRUE only if a why/use/audience is present (stated or clearly implied).\n"
+        "  ▸ TRUE: 'summarize for my class', 'I'm building a budgeting app, write the schema', 'dedupe my data "
+        "for analysis'.  ✗ FALSE (task only): 'write a sorting function', 'fix this code', 'list 10 names'."
+    )
+    return _ask(instr, _GOAL_T, "User message", msg)
+
+
+_DECOMP_T = '{"decomposition": <bool>}'
+
+
+def decomp_prompt(msg):
+    instr = (
+        "Does THIS user message break the REQUEST into multiple ordered/numbered SUB-TASKS the AI must "
+        "perform? TRUE for 'first X, then Y, then Z' or '1) … 2) … 3) …' of distinct sub-tasks.\n"
+        "  ▸ TRUE: 'First summarize it, then list 3 risks, then propose fixes', '1) clean the data 2) plot it'.\n"
+        "  ✗ FALSE: a single ask ('write a sorting function'); 'write a step-by-step guide' (output, not a "
+        "decomposed request); 'think step by step' (that's reasoning); a numbered list of data/options."
+    )
+    return _ask(instr, _DECOMP_T, "User message", msg)
+
+
+def spec_prompt(msg):
+    instr = (
+        "Does THIS user message contain ANY concrete detail of ANY kind — a number, version, named "
+        "tool/library/framework, file/data/schema, an explicit constraint, OR the user's OWN "
+        "background/role/skill level? Be LIBERAL: a single concrete detail is enough.\n"
+        "  ▸ TRUE: 'Python 3.11', 'I'm a beginner', 'under 100ms', 'here's my CSV', 'using pandas', "
+        "'a 1000-word post'.  ✗ FALSE only when fully vague: 'make it fast', 'help me', 'write something good'."
+    )
+    return _ask(instr, _SPEC_T, "User message", msg)
+
+
+class SamplingClient:
+    """vLLM OpenAI-style endpoint with n completions per request (for self-consistency)."""
+
+    def __init__(self, base_url, token, timeout=90):
+        self.base = base_url.rstrip("/"); self.token = token; self.timeout = timeout
+
+    def sample(self, prompt, n=3, temperature=0.6):
+        body = {"model": "MiniCPM4.1-8B", "messages": [{"role": "user", "content": prompt}],
+                "temperature": temperature, "n": n, "max_tokens": 256,
+                "chat_template_kwargs": {"enable_thinking": False}}
+        r = requests.post(f"{self.base}/v1/chat/completions",
+                          headers={"Authorization": f"Bearer {self.token}", "Content-Type": "application/json"},
+                          json=body, timeout=self.timeout)
+        r.raise_for_status()
+        from prompt_card.llm.minicpm import clean_content
+        return [clean_content(c["message"]["content"]) for c in r.json()["choices"]]
+
+
+def main():
+    base = os.environ.get("OPENBMB_BASE_URL"); token = os.environ.get("OPENBMB_TOKEN")
+    if not base or not token:
+        print("ERROR: set OPENBMB_BASE_URL/OPENBMB_TOKEN", file=sys.stderr); sys.exit(2)
+    from prompt_card.llm.minicpm import MiniCPMClient
+    gt = K.load_gt(); convs = K.load_convs()
+    client = K.CachedClient(MiniCPMClient(base, token), workers=8)
+
+    # ---- C1: per-feature IQ ----
+    goal_prompts, spec_prompts, rows = [], [], []
+    for r in gt:
+        ut = K.user_turns(convs[r["id"]])
+        for row in r["input_quality"]:
+            i = int(row["turn"][1:]) - 1
+            goal_prompts.append(goal_prompt(ut[i])); spec_prompts.append(spec_prompt(ut[i]))
+            rows.append((int("goal_stated" in row["features"]), int("specific_specification" in row["features"])))
+    gres = client.run_all(goal_prompts); sres = client.run_all(spec_prompts)
+    gyt = [r[0] for r in rows]; gyp = [int(bool((K.OA.parse(gres[p], ("goal_stated",)) or {}).get("goal_stated"))) for p in goal_prompts]
+    syt = [r[1] for r in rows]; syp = [int(bool((K.OA.parse(sres[p], ("specific_specification",)) or {}).get("specific_specification"))) for p in spec_prompts]
+    gk = K.cohen_kappa(gyt, gyp); sk = K.cohen_kappa(syt, syp)
+    print("=== C1 per-feature IQ (dedicated single-feature prompts) ===")
+    print(f"  goal_stated           κ={gk:+.3f}  [TN/FP/FN/TP {K.binary_counts(gyt,gyp)}]   (A+B best v2: +0.226)")
+    print(f"  specific_specification κ={sk:+.3f}  [TN/FP/FN/TP {K.binary_counts(syt,syp)}]   (A+B best v2: +0.568)")
+    print(f"  IQ headline (mean)    κ={(gk+sk)/2:+.3f}   (A+B best: +0.397)")
+
+    # ---- C2: self-consistency interaction ----
+    sc = SamplingClient(base, token)
+    iyt, iyp = [], []
+    items = []
+    for r in gt:
+        ut = K.user_turns(convs[r["id"]])
+        for row in r["interaction"]:
+            i = int(row["turn"][1:]) - 1
+            items.append((int(bool(row["refinement"])), int_v3(ut[i - 1], ut[i])))
+    print(f"\n=== C2 self-consistency interaction (3×, temp 0.6, majority) over {len(items)} turns ===", flush=True)
+    import concurrent.futures as cf
+    cache = {}
+
+    def vote(prompt):
+        outs = sc.sample(prompt, n=3, temperature=0.6)
+        yes = 0
+        for o in outs:
+            d = K.OA.parse(o, ("refinement_attempt",)) or {}
+            yes += int(bool(d.get("refinement_attempt")))
+        return int(yes >= 2)
+
+    prompts = [p for _, p in items]
+    with cf.ThreadPoolExecutor(max_workers=8) as ex:
+        votes = list(ex.map(vote, prompts))
+    iyt = [t for t, _ in items]; iyp = votes
+    ik = K.cohen_kappa(iyt, iyp)
+    print(f"  interaction (self-consistency) κ={ik:+.3f}  [TN/FP/FN/TP {K.binary_counts(iyt,iyp)}]   (A+B best v3: +0.320)")
+
+    json.dump({"goal": gk, "spec": sk, "iq_mean": (gk + sk) / 2, "interaction_sc": ik},
+              open(os.path.join(os.path.dirname(__file__), "_cache", "step_c.json"), "w"), indent=1)
+
+
+if __name__ == "__main__":
+    main()

eval/step_critical.py

@@ -0,0 +1,114 @@
+"""Phase 3 Step 1 — Critical Engagement (5 types) base-8B measurement, STEP-A style (sharp GT-aligned
+definitions, NO few-shot). Per-type κ + confusion + overall. Reuses the base-8B cache (:8001).
+
+Run:  python -m eval.step_critical
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+from eval import kappa as K
+from eval.prompts_v2 import _ask
+from prompt_card.scoring import observable_axes as OA
+
+CE = ["skepticism", "rebuttal", "source_request", "independent_verification", "re_questioning"]
+_CE_T = ('{"skepticism": <bool>, "rebuttal": <bool>, "source_request": <bool>, '
+         '"independent_verification": <bool>, "re_questioning": <bool>}')
+
+
+def build_critical_prompt(prev_assistant: str, this_user: str) -> str:
+    instr = (
+        "Detect critical-engagement TYPES in THE USER's turn as it reacts to what the AI just said. Multiple "
+        "types can be true at once; mark true only when genuinely present.\n"
+        "- skepticism: doubts/questions the AI's claim WITHOUT giving a reason. ▸ 'really?', 'are you sure?', "
+        "'that doesn't sound right'. ✗ 'ok, thanks' (acceptance).\n"
+        "- rebuttal: pushes back with the user's OWN counter-argument or correction. ▸ 'that's wrong — if X "
+        "were true, Y wouldn't happen', 'no, it returns 0 not 1'. ✗ a bare 'are you sure?' (that is skepticism).\n"
+        "- source_request: asks for a citation / evidence / source. ▸ 'what's your source?', 'cite that'. "
+        "Note: asking for REASONING ('why did you pick X?') is NOT source_request.\n"
+        "- independent_verification: the user states an EXPLICIT external check they performed. ▸ 'I ran it and "
+        "it throws on empty input', 'I checked the docs, it says X'. ✗ a bare confident assertion with no stated "
+        "check (that is rebuttal).\n"
+        "- re_questioning: RE-ASKS the same question because the AI's answer was unsatisfactory. ▸ 'that's not "
+        "what I asked — I meant the async case'. ✗ a NEW/different question."
+    )
+    payload = f"AI just said:\n{prev_assistant}\n\nUser's turn:\n{this_user}"
+    return _ask(instr, _CE_T, "Turns", payload)
+
+
+def main():
+    base = os.environ.get("OPENBMB_BASE_URL"); token = os.environ.get("OPENBMB_TOKEN")
+    if not base or not token:
+        print("ERROR: creds", file=sys.stderr); sys.exit(2)
+    from prompt_card.llm.minicpm import MiniCPMClient
+    gt = K.load_gt(); convs = K.load_convs()
+    client = K.CachedClient(MiniCPMClient(base, token), workers=8)
+
+    prompts, rows = [], []
+    for r in gt:
+        conv = convs[r["id"]]
+        for j, row in enumerate(r["critical"]):
+            prev = OA._prev_assistant(conv, int(row["turn"][1:]) - 1)
+            users = OA._user_turns(conv)
+            utext = users[int(row["turn"][1:]) - 1]
+            prompts.append(build_critical_prompt(prev, utext))
+            rows.append(set(row["types"]))
+    print(f"[critical] {len(prompts)} turn-prompts", flush=True)
+    resp = client.run_all(prompts)
+    print(f"[critical] 8B done (new {client.misses})", flush=True)
+
+    preds = []
+    fail = 0
+    for p in prompts:
+        d = OA.parse(resp[p], CE)
+        if d is None:
+            fail += 1; d = {}
+        preds.append({t for t in CE if d.get(t)})
+
+    print("\n=== Critical Engagement — base-8B per-type κ (vs Kim ground truth) ===")
+    print(f"parse failures: {fail}/{len(prompts)}")
+    per_k = {}
+    for t in CE:
+        yt = [int(t in s) for s in rows]; yp = [int(t in s) for s in preds]
+        k = K.cohen_kappa(yt, yp); c = K.binary_counts(yt, yp); p, rc, f1 = K.prf(c)
+        per_k[t] = k
+        ks = f"{k:+.3f}" if k is not None else "N/A"
+        print(f"  {t:24} κ={ks}  pos={sum(yt)}  [TN {c['tn']} FP {c['fp']} FN {c['fn']} TP {c['tp']}]  "
+              f"P{'-' if p is None else f'{p:.2f}'} R{'-' if rc is None else f'{rc:.2f}'}")
+    # any-CE per turn
+    anyt = [int(bool(s)) for s in rows]; anyp = [int(bool(s)) for s in preds]
+    anyk = K.cohen_kappa(anyt, anyp)
+    # distinct-type count per conv (the product signal): agreement
+    gi = 0; gt_counts = []; pr_counts = []
+    for r in gt:
+        n = len(r["critical"])
+        gtypes = set(); ptypes = set()
+        for _ in range(n):
+            gtypes |= rows[gi]; ptypes |= preds[gi]; gi += 1
+        gt_counts.append(len(gtypes)); pr_counts.append(len(ptypes))
+    # mean abs error on the 0-5 distinct-type signal
+    mae = sum(abs(a - b) for a, b in zip(gt_counts, pr_counts)) / len(gt_counts)
+    valid = [per_k[t] for t in CE if per_k[t] is not None]
+    headline = sum(valid) / len(valid) if valid else None
+    print(f"\n  any-CE-present (per turn) κ = {anyk:+.3f}")
+    print(f"  per-type mean κ (headline) = {headline:+.3f}")
+    print(f"  distinct-type count (0-5) MAE per conv = {mae:.2f}")
+
+    if headline is None:
+        decision = "N/A"
+    elif headline >= 0.6:
+        decision = "NO LoRA — base 8B solid"
+    elif headline >= 0.4:
+        decision = "NO LoRA — borderline, document caveats (save time)"
+    else:
+        decision = "LoRA NEEDED (headline < 0.4)"
+    print(f"\n  >>> CRITICAL LoRA DECISION: {decision}")
+    json.dump({"per_type": per_k, "any": anyk, "headline": headline, "mae": mae,
+               "decision": decision, "parse_fail": fail},
+              open(os.path.join(os.path.dirname(__file__), "_cache", "critical.json"), "w"), indent=1, default=str)
+
+
+if __name__ == "__main__":
+    main()

eval/step_d.py

@@ -0,0 +1,170 @@
+"""STEP D — model swap to MiniCPM-V-4.6-Thinking (:8004) for the reasoning-heavy axes (Focus, Interaction;
+and re-usable for Phase-3 Critical). Compares to the base-8B A+B best (focus 0.433, interaction 0.320).
+
+The Thinking model emits a reasoning block then JSON, sometimes with a dangling `</think>` (no opening
+tag), so we parse robustly: take text after the last `</think>`, then json/regex-extract the field.
+Separate cache namespace (think_8004.jsonl) so identical prompt text doesn't collide with base-8B cache.
+"""
+from __future__ import annotations
+
+import concurrent.futures as cf
+import hashlib
+import json
+import os
+import re
+import sys
+import threading
+
+import requests
+
+from eval import kappa as K
+from eval.prompts_v4 import build_focus_boundary_prompt
+from eval.prompts_v3 import build_interaction_prompt
+
+CACHE = os.path.join(os.path.dirname(__file__), "_cache", "think_8004.jsonl")
+MODEL = "MiniCPM-V-4.6-Thinking"
+
+
+def parse_think(text, field):
+    """Robust to dangling </think> + reasoning noise. Returns {field: value} or None."""
+    t = text or ""
+    if "</think>" in t:
+        t = t.rsplit("</think>", 1)[-1]
+    t = re.sub(r"^```[a-zA-Z0-9]*\n?|\n?```$", "", t.strip()).strip()
+    try:
+        d = json.loads(t)
+        if isinstance(d, dict) and field in d:
+            return {field: d[field]}
+    except Exception:
+        pass
+    # regex fallback: "field": value
+    m = re.search(rf'"{field}"\s*:\s*("?[a-zA-Z_]+"?|true|false)', t)
+    if m:
+        v = m.group(1).strip('"')
+        if v in ("true", "false"):
+            return {field: v == "true"}
+        return {field: v}
+    return None
+
+
+class ThinkClient:
+    def __init__(self, base, token, workers=8):
+        self.base = base.rstrip("/"); self.token = token; self.workers = workers
+        self._lock = threading.Lock(); self.cache = {}
+        if os.path.exists(CACHE):
+            for line in open(CACHE):
+                try:
+                    d = json.loads(line); self.cache[d["k"]] = d["v"]
+                except Exception:
+                    pass
+
+    @staticmethod
+    def _key(p):
+        return hashlib.sha1(("THINK::" + p).encode()).hexdigest()
+
+    def _call(self, prompt, retries=4):
+        body = {"model": MODEL, "messages": [{"role": "user", "content": prompt}], "temperature": 0,
+                "max_tokens": 1536, "chat_template_kwargs": {"enable_thinking": True}}
+        last = None
+        for attempt in range(retries):
+            try:
+                r = requests.post(f"{self.base}/v1/chat/completions",
+                                  headers={"Authorization": f"Bearer {self.token}", "Content-Type": "application/json"},
+                                  json=body, timeout=240)
+                r.raise_for_status()
+                return r.json()["choices"][0]["message"]["content"]
+            except (requests.exceptions.RequestException,) as e:
+                last = e  # transient endpoint hiccup — backoff and retry
+                import time
+                time.sleep(2 * (attempt + 1))
+        raise last
+
+    def run_all(self, prompts):
+        uniq = list(dict.fromkeys(prompts))
+        todo = [p for p in uniq if self._key(p) not in self.cache]
+
+        def work(p):
+            v = self._call(p); k = self._key(p)
+            with self._lock:
+                self.cache[k] = v
+                with open(CACHE, "a") as f:
+                    f.write(json.dumps({"k": k, "v": v}, ensure_ascii=False) + "\n")
+            return p
+        if todo:
+            with cf.ThreadPoolExecutor(max_workers=self.workers) as ex:
+                for i, _ in enumerate(ex.map(work, todo), 1):
+                    if i % 50 == 0:
+                        print(f"  ... {i}/{len(todo)} thinking calls", flush=True)
+        return {p: self.cache[self._key(p)] for p in uniq}
+
+
+def main():
+    base = os.environ.get("OPENBMB_BASE_URL"); token = os.environ.get("OPENBMB_TOKEN")
+    if not base or not token:
+        print("ERROR: creds", file=sys.stderr); sys.exit(2)
+    base = base.replace("8001", "8004")
+    gt = K.load_gt(); convs = K.load_convs(); embedder = K.FastEmbedder()
+    client = ThinkClient(base, token, workers=8)
+
+    # interaction
+    int_prompts, int_rows = [], []
+    for r in gt:
+        ut = K.user_turns(convs[r["id"]])
+        for row in r["interaction"]:
+            i = int(row["turn"][1:]) - 1
+            int_prompts.append(build_interaction_prompt(ut[i - 1], ut[i]))
+            int_rows.append(int(bool(row["refinement"])))
+    # focus (embedder-gated, T<=0.70 to cover the sweep)
+    foc_tasks = []
+    geom = {}
+    for r in gt:
+        ut = K.user_turns(convs[r["id"]])
+        if len(ut) < 2:
+            continue
+        import numpy as np
+        vecs = embedder.embed(ut); g = []
+        for i in range(len(ut) - 1):
+            cos = float(np.dot(vecs[i], vecs[i + 1])); g.append((i, cos))
+            if cos < 0.70:
+                foc_tasks.append((r["id"], f"U{i+1}->U{i+2}", build_focus_boundary_prompt(ut[i], ut[i + 1])))
+        geom[r["id"]] = g
+
+    print(f"[step_d] thinking calls: interaction {len(int_prompts)} + focus {len(foc_tasks)}", flush=True)
+    all_prompts = int_prompts + [t[2] for t in foc_tasks]
+    resp = client.run_all(all_prompts)
+
+    # interaction kappa
+    iyp = [int(bool((parse_think(resp[p], "refinement_attempt") or {}).get("refinement_attempt"))) for p in int_prompts]
+    ifail = sum(1 for p in int_prompts if parse_think(resp[p], "refinement_attempt") is None)
+    ik = K.cohen_kappa(int_rows, iyp)
+
+    # focus: rebuild per-boundary relation preds, sweep T
+    rel = {(cid, bk): (parse_think(resp[p], "relation") or {}).get("relation") for (cid, bk, p) in foc_tasks}
+    ffail = sum(1 for (cid, bk, p) in foc_tasks if parse_think(resp[p], "relation") is None)
+    best = None
+    for T in [0.40, 0.45, 0.50, 0.55, 0.60, 0.65, 0.70]:
+        yt, yp = [], []; hit = tot = 0
+        for r in gt:
+            gtrel = {f"{c['a']}->{c['b']}": c["relation"] for c in r["focus"]}
+            for (i, cos) in geom.get(r["id"], []):
+                bk = f"U{i+1}->U{i+2}"; isgt = (gtrel.get(bk) == "topic_shift")
+                yt.append(int(isgt))
+                pred = int(cos < T and rel.get((r["id"], bk)) == "topic_shift")
+                yp.append(pred)
+                if isgt:
+                    tot += 1; hit += int(cos < T)
+        k = K.cohen_kappa(yt, yp); rec = hit / tot if tot else None
+        cand = (k if k is not None else -9, rec or 0)
+        if best is None or cand > best[0]:
+            best = (cand, T, k, rec)
+
+    print("\n=== STEP D (MiniCPM-V-4.6-Thinking, :8004) vs base-8B best ===")
+    print(f"  interaction  κ={ik:+.3f}  parse_fail={ifail}/{len(int_prompts)}   (base-8B best v3: +0.320)")
+    print(f"  focus        κ={best[2]:+.3f}  T={best[1]} recall={best[3]:.2f}  parse_fail={ffail}/{len(foc_tasks)}   (base-8B best v4: +0.433)")
+    json.dump({"interaction_think": ik, "focus_think": best[2], "focus_T": best[1],
+               "int_parsefail": ifail, "foc_parsefail": ffail},
+              open(os.path.join(os.path.dirname(__file__), "_cache", "step_d.json"), "w"), indent=1)
+
+
+if __name__ == "__main__":
+    main()

prompt_card/adapters/chatgpt.py

@@ -0,0 +1,62 @@
+"""ChatGPT export adapter. `conversations.json` is a list of conversations, each with a
+`mapping` tree of nodes keyed by id; we walk parent->children from the root to recover order."""
+from __future__ import annotations
+
+from ..schema import Conversation, Turn, normalize_role
+
+
+def _text_from_parts(content: dict) -> str:
+    """Join string parts only; multimodal parts (dicts) are dropped."""
+    if not isinstance(content, dict):
+        return ""
+    parts = content.get("parts") or []
+    return "\n".join(p for p in parts if isinstance(p, str)).strip()
+
+
+def _walk(mapping: dict) -> list[Turn]:
+    # find root: node whose parent is None (or missing)
+    root_id = None
+    for node_id, node in mapping.items():
+        if not node.get("parent"):
+            root_id = node_id
+            break
+    if root_id is None:
+        root_id = next(iter(mapping))
+
+    turns: list[Turn] = []
+    cur = root_id
+    seen = set()
+    while cur and cur in mapping and cur not in seen:
+        seen.add(cur)
+        node = mapping[cur]
+        msg = node.get("message")
+        if msg:
+            role = normalize_role((msg.get("author") or {}).get("role", ""))
+            text = _text_from_parts(msg.get("content") or {})
+            if role and text:
+                ct = msg.get("create_time")
+                turns.append(Turn(role=role, text=text, time=str(ct) if ct is not None else None))
+        children = node.get("children") or []
+        cur = children[0] if children else None
+    return turns
+
+
+def parse(data) -> list[Conversation]:
+    if not isinstance(data, list):
+        raise ValueError("ChatGPT export must be a list of conversations")
+    convs: list[Conversation] = []
+    for conv in data:
+        if not isinstance(conv, dict):
+            continue
+        mapping = conv.get("mapping")
+        if not isinstance(mapping, dict):
+            continue
+        ct = conv.get("create_time")
+        convs.append(
+            Conversation(
+                provider="chatgpt",
+                created_at=str(ct) if ct is not None else None,
+                turns=_walk(mapping),
+            )
+        )
+    return convs

prompt_card/adapters/claude.py

@@ -0,0 +1,29 @@
+"""Claude export adapter. List of conversations, each with `chat_messages`
+(`sender`: human|assistant, `text`)."""
+from __future__ import annotations
+
+from ..schema import Conversation, Turn, normalize_role
+
+
+def parse(data) -> list[Conversation]:
+    if not isinstance(data, list):
+        raise ValueError("Claude export must be a list of conversations")
+    convs: list[Conversation] = []
+    for conv in data:
+        if not isinstance(conv, dict):
+            continue
+        msgs = conv.get("chat_messages")
+        if not isinstance(msgs, list):
+            continue
+        turns: list[Turn] = []
+        for m in msgs:
+            if not isinstance(m, dict):
+                continue
+            role = normalize_role(m.get("sender", ""))
+            text = (m.get("text") or "").strip()
+            if role and text:
+                turns.append(Turn(role=role, text=text, time=m.get("created_at")))
+        convs.append(
+            Conversation(provider="claude", created_at=conv.get("created_at"), turns=turns)
+        )
+    return convs

prompt_card/adapters/gemini.py

@@ -0,0 +1,29 @@
+"""Gemini export adapter. List of conversations, each with `turns`
+(`role`: user|model, `text`)."""
+from __future__ import annotations
+
+from ..schema import Conversation, Turn, normalize_role
+
+
+def parse(data) -> list[Conversation]:
+    if not isinstance(data, list):
+        raise ValueError("Gemini export must be a list of conversations")
+    convs: list[Conversation] = []
+    for conv in data:
+        if not isinstance(conv, dict):
+            continue
+        raw_turns = conv.get("turns")
+        if not isinstance(raw_turns, list):
+            continue
+        turns: list[Turn] = []
+        for t in raw_turns:
+            if not isinstance(t, dict):
+                continue
+            role = normalize_role(t.get("role", ""))
+            text = (t.get("text") or "").strip()
+            if role and text:
+                turns.append(Turn(role=role, text=text, time=t.get("create_time")))
+        convs.append(
+            Conversation(provider="gemini", created_at=conv.get("create_time"), turns=turns)
+        )
+    return convs

prompt_card/adapters/paste.py

@@ -0,0 +1,52 @@
+"""Paste fallback. Parses pasted free-text into user/assistant turns.
+
+Priority: explicit role markers at line start ("User:", "Assistant:", "You:", "AI:", "ChatGPT:",
+"Human:", "Q:", "A:", …) group multi-line turns. With no markers, blank-line-separated blocks are
+treated as an alternating user/assistant transcript starting with the user. This matters because the
+observable axes (interaction/critical/focus) need assistant turns, not an all-user blob.
+"""
+from __future__ import annotations
+
+import re
+
+from ..schema import Conversation, Turn
+
+_USER = re.compile(r"^\s*(user|you|human|me|prompt|q)\s*[:>\-]\s*", re.I)
+_ASST = re.compile(r"^\s*(assistant|ai|chatgpt|chat gpt|claude|gpt|bot|gemini|a|answer|response)\s*[:>\-]\s*", re.I)
+
+
+def parse(text: str) -> list[Conversation]:
+    if not isinstance(text, str):
+        raise ValueError("Paste input must be a string")
+    if not text.strip():
+        raise ValueError("No non-empty lines to parse")
+
+    marked = []
+    cur_role, cur_buf = None, []
+
+    def flush():
+        if cur_role and cur_buf:
+            t = "\n".join(cur_buf).strip()
+            if t:
+                marked.append((cur_role, t))
+
+    for ln in text.splitlines():
+        if _USER.match(ln):
+            flush(); cur_role, cur_buf = "user", [_USER.sub("", ln, count=1)]
+        elif _ASST.match(ln):
+            flush(); cur_role, cur_buf = "assistant", [_ASST.sub("", ln, count=1)]
+        elif cur_role is not None:
+            cur_buf.append(ln)
+    flush()
+
+    if marked:
+        turns = [Turn(role=r, text=t, time=None) for r, t in marked]
+        return [Conversation(provider="paste", created_at=None, turns=turns)]
+
+    # No role markers: split on blank lines into blocks, alternate user/assistant starting with user.
+    blocks = [b.strip() for b in re.split(r"\n\s*\n", text) if b.strip()]
+    if not blocks:
+        blocks = [ln.strip() for ln in text.splitlines() if ln.strip()]
+    roles = ("user", "assistant")
+    turns = [Turn(role=roles[i % 2], text=b, time=None) for i, b in enumerate(blocks)]
+    return [Conversation(provider="paste", created_at=None, turns=turns)]

prompt_card/app_core.py

@@ -0,0 +1,89 @@
+"""App integrator (UI-agnostic): take uploaded export text or pasted text, produce the card
+PNG + tips. Kept separate from the Gradio layer so the whole flow is unit-testable.
+
+v2: scoring goes through the Modal LoRA evaluator (`lora_client`); tips text is written by a
+separate generative client (`tips_client`, optional). Privacy: uploads are read into memory and
+the file is deleted before scoring; nothing is written to disk or logged here.
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+
+from .adapters import chatgpt, claude, gemini, paste
+from .preprocess import InsufficientData
+from .evaluate import evaluate
+from .llm.lora_client import LoRAClient, InferenceUnavailable
+from .llm.client import LLMClient
+from .llm import tips as tips_mod
+from . import card
+
+_ADAPTERS = {"chatgpt": chatgpt.parse, "claude": claude.parse, "gemini": gemini.parse}
+
+
+class AnalysisError(Exception):
+    """User-facing error with a friendly message."""
+
+
+def detect_provider(raw) -> str:
+    """Sniff the export shape. ChatGPT has `mapping`; Claude `chat_messages`; Gemini `turns`."""
+    sample = raw[0] if isinstance(raw, list) and raw else raw
+    if isinstance(sample, dict):
+        if "mapping" in sample:
+            return "chatgpt"
+        if "chat_messages" in sample:
+            return "claude"
+        if "turns" in sample:
+            return "gemini"
+    raise AnalysisError("Unrecognized export format. Use a ChatGPT, Claude, or Gemini export, or paste text.")
+
+
+def _style_comment(result: dict) -> str:
+    """Placeholder style comment grounding the tips (real MiniCPM grounding is a later step)."""
+    weak = tips_mod.weak_axes(result["axes"])
+    return f"Weakest areas: {', '.join(weak)}." if weak else "Strong across the board."
+
+
+def _insufficient_message(err: InsufficientData) -> str:
+    m = re.search(r"found (\d+)", str(err))
+    found = int(m.group(1)) if m else 0
+    if found == 0:
+        return "We couldn't find any English messages to score. This tool currently evaluates English chats only."
+    return f"We could only read {found} English messages, but need at least 20 to build a card."
+
+
+def analyze(content: str, lora_client: LoRAClient, *, name: str = "", tips_client: LLMClient | None = None) -> dict:
+    """content = export JSON text or pasted free text. Returns {card_png, tips, result}."""
+    try:
+        raw = json.loads(content)
+        is_json = True
+    except (json.JSONDecodeError, TypeError):
+        raw, is_json = None, False
+
+    try:
+        if is_json:
+            provider = detect_provider(raw)
+            conversations = _ADAPTERS[provider](raw)
+        else:
+            conversations = paste.parse(content)
+        result = evaluate(conversations, lora_client)
+    except InsufficientData as e:
+        raise AnalysisError(_insufficient_message(e)) from e
+    except InferenceUnavailable as e:
+        raise AnalysisError("The scoring service is temporarily unavailable. Please try again in a moment.") from e
+    except ValueError as e:
+        raise AnalysisError(str(e)) from e
+
+    tips = tips_mod.generate_tips(result["axes"], tips_client, _style_comment(result)) if tips_client else []
+    png = card.render_png(result, name=name)
+    return {"card_png": png, "tips": tips, "result": result}
+
+
+def analyze_upload(path: str, lora_client: LoRAClient, *, name: str = "", tips_client: LLMClient | None = None, delete: bool = True) -> dict:
+    """Read the upload into memory, delete the file immediately, then analyze."""
+    with open(path, "r", encoding="utf-8", errors="ignore") as fh:
+        content = fh.read()
+    if delete and os.path.exists(path):
+        os.remove(path)
+    return analyze(content, lora_client, name=name, tips_client=tips_client)

prompt_card/card.py

@@ -0,0 +1,108 @@
+"""Player card rendering — pure Pillow first cut (no plotly/matplotlib/kaleido).
+
+`radar_points` is pure geometry (unit-tested). `render_png` composes the card: radar grid +
+score polygon, per-axis labels, overall rating and star tier. Swap for richer visuals later;
+the interactive Gradio view can use Plotly without changing this PNG path.
+"""
+from __future__ import annotations
+
+import io
+import math
+
+from PIL import Image, ImageDraw, ImageFont
+
+from .scoring.overall import AXES
+
+# short labels for the rim (full names live in AXES)
+SHORT = {
+    "Technique": "TEC",
+    "Context": "CTX",
+    "Interaction": "INT",
+    "Focus": "FOC",
+    "Critical Engagement": "CRI",
+}
+
+_BG = (18, 22, 33)
+_GRID = (60, 68, 86)
+_FILL = (90, 170, 255)
+_ACCENT = (255, 205, 90)
+_TEXT = (230, 235, 245)
+
+
+def radar_points(scores, center, max_radius):
+    """Map scores (0-10) to (x, y) points on a radar, first axis pointing straight up,
+    going clockwise. Score 0 -> center, score 10 -> rim."""
+    cx, cy = center
+    n = len(scores)
+    pts = []
+    for i, s in enumerate(scores):
+        angle = -math.pi / 2 + 2 * math.pi * i / n
+        r = (max(0.0, min(10.0, s)) / 10.0) * max_radius
+        pts.append((cx + r * math.cos(angle), cy + r * math.sin(angle)))
+    return pts
+
+
+def _font(size):
+    try:
+        return ImageFont.truetype("DejaVuSans.ttf", size)
+    except Exception:
+        return ImageFont.load_default()
+
+
+def _ring(draw, center, radius, n):
+    pts = radar_points([10] * n, center, radius)
+    draw.polygon(pts, outline=_GRID)
+
+
+def render_png(result: dict, name: str = "") -> bytes:
+    W, H = 600, 720
+    center = (W // 2, 300)
+    max_radius = 200
+
+    img = Image.new("RGB", (W, H), _BG)
+    d = ImageDraw.Draw(img, "RGBA")
+
+    axes = result["axes"]
+    labels = list(axes.keys())
+    scores = [axes[k] for k in labels]
+    n = len(labels)
+
+    # grid rings + spokes
+    for frac in (0.25, 0.5, 0.75, 1.0):
+        _ring(d, center, max_radius * frac, n)
+    for x, y in radar_points([10] * n, center, max_radius):
+        d.line([center, (x, y)], fill=_GRID)
+
+    # score polygon
+    poly = radar_points(scores, center, max_radius)
+    if n >= 3:
+        d.polygon(poly, fill=(_FILL[0], _FILL[1], _FILL[2], 90), outline=_FILL)
+    for x, y in poly:
+        d.ellipse([x - 4, y - 4, x + 4, y + 4], fill=_FILL)
+
+    # rim labels
+    lf = _font(20)
+    for (x, y), key in zip(radar_points([11.4] * n, center, max_radius), labels):
+        txt = SHORT.get(key, key[:3].upper())
+        bb = d.textbbox((0, 0), txt, font=lf)
+        d.text((x - (bb[2] - bb[0]) / 2, y - (bb[3] - bb[1]) / 2), txt, fill=_TEXT, font=lf)
+
+    # header
+    if name:
+        d.text((30, 24), name, fill=_TEXT, font=_font(34))
+
+    # footer: overall + stars + per-axis values
+    overall = result.get("overall", 0.0)
+    stars = result.get("stars", 1)
+    d.text((30, 560), f"OVERALL  {overall:.1f}", fill=_ACCENT, font=_font(30))
+    d.text((30, 600), "★" * stars + "☆" * (5 - stars), fill=_ACCENT, font=_font(30))
+
+    sf = _font(18)
+    y = 560
+    for key in labels:
+        d.text((330, y), f"{SHORT.get(key, key[:3])}  {axes[key]:.1f}", fill=_TEXT, font=sf)
+        y += 26
+
+    buf = io.BytesIO()
+    img.save(buf, format="PNG")
+    return buf.getvalue()

prompt_card/evaluate.py

@@ -0,0 +1,62 @@
+"""v2 evaluation orchestrator (replaces score.py's engine path).
+
+Flow: preprocess (English filter, min-20 gate, 500-cap, session grouping) -> stratified-sample
+the prompts for the per-prompt axes -> call the Modal LoRA evaluator with prompts + conversation
+payloads -> aggregate the returned labels via saturation curves / CE mean -> overall + stars.
+
+Per-prompt axes (Technique, Context) score the sampled prompts; per-conversation axes
+(Interaction, Focus, Critical Engagement) score the conversation payloads. `InferenceUnavailable`
+propagates to the caller (app layer), which renders a friendly message — analysis never crashes.
+"""
+from __future__ import annotations
+
+from .schema import Conversation, ROLE_USER
+from .preprocess import preprocess
+from .scoring import labels, overall
+from .scoring.sampling import stratified_sample, DEFAULT_N
+from .llm.lora_client import LoRAClient
+
+
+def conversation_payload(conv: Conversation) -> dict:
+    """Serialize a conversation for the per-conversation LoRA axes. Keeps assistant turns
+    (needed for clarifying-question / sycophancy detection); turn-count thresholds are applied
+    Modal-side."""
+    turns = [{"role": t.role, "text": t.text} for t in conv.turns]
+    user_turns = sum(1 for t in conv.turns if t.role == ROLE_USER)
+    return {"turns": turns, "user_turns": user_turns}
+
+
+def evaluate(
+    conversations: list[Conversation],
+    lora_client: LoRAClient,
+    *,
+    sample_n: int = DEFAULT_N,
+    seed: int = 0,
+    min_user_msgs: int = 20,
+) -> dict:
+    prepared = preprocess(conversations, min_user_msgs=min_user_msgs)
+    all_prompts = prepared.user_prompts
+    sampled = stratified_sample(all_prompts, n=sample_n, seed=seed)
+    conv_payloads = [conversation_payload(c) for c in prepared.conversations]
+
+    out = lora_client.evaluate(sampled, conv_payloads)  # may raise InferenceUnavailable
+
+    axes = labels.assemble_axes(
+        technique=out["technique"],
+        context=out["context"],
+        interaction=out["interaction"],
+        focus=out["focus"],
+        critical_engagement=out["critical_engagement"],
+    )
+    ov = overall.overall(axes)
+    return {
+        "axes": axes,
+        "overall": ov,
+        "stars": overall.star_tier(ov),
+        "raw": {
+            "n_user_prompts": len(all_prompts),
+            "n_prompts_scored": len(sampled),
+            "sampled": len(sampled) < len(all_prompts),
+            "n_conversations": len(conv_payloads),
+        },
+    }

prompt_card/llm/client.py

@@ -0,0 +1,22 @@
+"""LLM client interface. Phase 1 uses MockLLMClient; the real MiniCPM-on-Modal client
+is wired in Phase 2-B behind the same `generate` contract."""
+from __future__ import annotations
+
+from typing import Protocol
+
+
+class LLMClient(Protocol):
+    def generate(self, prompt: str) -> str:
+        ...
+
+
+class MockLLMClient:
+    """Deterministic stand-in: returns a fixed response and records every prompt it received."""
+
+    def __init__(self, response: str = ""):
+        self._response = response
+        self.calls: list[str] = []
+
+    def generate(self, prompt: str) -> str:
+        self.calls.append(prompt)
+        return self._response

prompt_card/llm/critical_router.py

@@ -0,0 +1,50 @@
+"""Per-type hybrid router for Critical Engagement (Kim's Option B).
+
+Validation κ vs the Phase-7 retrained Critical LoRA `critical_c_r16e5` (see eval/decisions.md):
+  skepticism   base 0.576 > LoRA 0.497    → base
+  rebuttal     base 0.521 ≈ LoRA 0.527    → base (tie, keep base default)
+  source_request base 0.467 < LoRA 0.664  → LoRA  (Phase-7 WIN: targeted weak-type synthesis, +0.197)
+  independent_verification base 0.035 << LoRA 0.578 → LoRA
+  re_questioning base 0.058 > LoRA 0.028  → base (documented limitation: insufficient training data)
+
+Hybrid per-type-best κ ≈ 0.479 (Phase 7; was 0.441 with indep_verif-only). The LoRA is routed for the two
+types it wins (independent_verification + source_request); the rest stay base, avoiding the catastrophic
+forgetting it caused on skepticism/re_questioning.
+"""
+from __future__ import annotations
+
+CE = ["skepticism", "rebuttal", "source_request", "independent_verification", "re_questioning"]
+
+# Types routed to the LoRA; everything else uses the base model.
+LORA_TYPES = frozenset({"independent_verification", "source_request"})
+
+
+def merge_critical(base_pred: dict | None, lora_pred: dict | None) -> dict:
+    """Combine two 5-bool predictions per the routing table → one 5-bool dict."""
+    base_pred = base_pred or {}
+    lora_pred = lora_pred or {}
+    out = {}
+    for t in CE:
+        src = lora_pred if t in LORA_TYPES else base_pred
+        out[t] = bool(src.get(t))
+    return out
+
+
+def merge_batch(base_preds: list, lora_preds: list) -> list:
+    """Element-wise merge of aligned base/LoRA prediction lists."""
+    return [merge_critical(b, l) for b, l in zip(base_preds, lora_preds)]
+
+
+def lora_critical_predict(prompts: list[str]) -> list[dict]:
+    """Run the Critical LoRA (base+adapter on Modal) over prompts → parsed 5-bool dicts.
+    One batched remote call per upload (cold-start amortized). Returns {} for parse failures."""
+    from modal_eval_lora import app, evaluate
+    from ..scoring.observable_axes import parse
+    from .lora_router import ADAPTER_DIR
+    with app.run():
+        raw = evaluate.remote("critical", prompts, adapter=ADAPTER_DIR.get("critical", ""))
+    out = []
+    for r in raw:
+        d = parse(r, CE)
+        out.append(d if d is not None else {})
+    return out

prompt_card/llm/judge.py

@@ -0,0 +1,72 @@
+"""MiniCPM-as-judge for Critical Engagement (the noisy axis).
+
+Tier-3 only runs on hard / low-confidence cases: when the heuristic markers leave the verdict
+ambiguous we ask the model; clear cases (no markers, or many) are decided by the heuristic alone.
+"""
+from __future__ import annotations
+
+import json
+import re
+
+from .client import LLMClient
+from ..scoring import heuristics as H
+
+ESCALATE_THRESHOLD = 0.6
+_JSON = re.compile(r"\{.*?\}", re.S)
+
+
+def build_ce_judge_prompt(prompts: list[str]) -> str:
+    joined = "\n".join(f"- {p}" for p in prompts)
+    return (
+        "You are evaluating whether a user critically engages with an AI assistant — "
+        "challenging answers, asking for sources, pushing back, or verifying claims, rather "
+        "than passively accepting everything.\n\n"
+        "User messages:\n"
+        f"{joined}\n\n"
+        'Respond ONLY with JSON: {"critical_engagement": true|false, "confidence": 0.0-1.0}'
+    )
+
+
+def parse_ce_verdict(response: str) -> dict:
+    m = _JSON.search(response or "")
+    if not m:
+        raise ValueError("no JSON object found in judge response")
+    data = json.loads(m.group(0))
+    return {
+        "critical_engagement": bool(data.get("critical_engagement", False)),
+        "confidence": float(data.get("confidence", 0.0)),
+    }
+
+
+def heuristic_confidence(prompts: list[str]) -> float:
+    """Confident when the marker signal is unambiguous: clearly none, or clearly frequent.
+    A small nonzero rate (a stray borderline marker) is the uncertain band that we escalate."""
+    rate = H.ce_rate(prompts)
+    if rate == 0.0:
+        return 0.9   # no markers at all -> confidently not critically engaged
+    if rate >= 0.5:
+        return 0.85  # challenges dominate -> confidently engaged
+    return 0.4       # markers present but noisy -> ambiguous, let the model decide
+
+
+def should_escalate(confidence: float, *, threshold: float = ESCALATE_THRESHOLD) -> bool:
+    return confidence < threshold
+
+
+def judge_ce(prompts: list[str], client: LLMClient, *, threshold: float = ESCALATE_THRESHOLD) -> dict:
+    """Return a CE verdict. Escalate to the model only when the heuristic is uncertain."""
+    confidence = heuristic_confidence(prompts)
+    heuristic_verdict = {
+        "critical_engagement": H.ce_rate(prompts) >= 0.5,
+        "confidence": confidence,
+        "escalated": False,
+    }
+    if not should_escalate(confidence, threshold=threshold):
+        return heuristic_verdict
+    try:
+        verdict = parse_ce_verdict(client.generate(build_ce_judge_prompt(prompts)))
+    except ValueError:
+        # model emitted unparseable output -> degrade to the heuristic rather than abort
+        return {**heuristic_verdict, "escalated": True, "parse_error": True}
+    verdict["escalated"] = True
+    return verdict

prompt_card/llm/lora_client.py

@@ -0,0 +1,79 @@
+"""Client to the Modal multi-LoRA evaluator endpoint.
+
+`ModalLoRAClient` POSTs {prompts, conversations} and returns label lists for all 5 axes. It
+retries transient failures with exponential backoff and, on final failure, raises
+`InferenceUnavailable` — the caller turns that into a friendly message. Inference failure must
+never crash the analysis (the v1 judge-crash bug). `MockLoRAClient` lets the app run end-to-end
+before real LoRAs exist; swap it for `ModalLoRAClient` once the endpoint is deployed.
+"""
+from __future__ import annotations
+
+import time
+from typing import Protocol, Sequence
+
+import requests
+
+_AXES = ["technique", "context", "interaction", "focus", "critical_engagement"]
+_TECH_FIELDS = ("zero_shot_role", "few_shot", "thought_generation",
+                "decomposition", "self_criticism", "flipped")
+
+
+class InferenceUnavailable(Exception):
+    """Raised when the Modal endpoint cannot be reached after retries."""
+
+
+class LoRAClient(Protocol):
+    def evaluate(self, prompts: Sequence[str], conversations: Sequence[dict]) -> dict:
+        ...
+
+
+class ModalLoRAClient:
+    def __init__(self, base_url: str, token: str, *, retries: int = 2, timeout: int = 180,
+                 backoff: float = 0.5, post=None, sleep=None):
+        self.base_url = base_url.rstrip("/")
+        self.token = token
+        self.retries = retries
+        self.timeout = timeout
+        self.backoff = backoff
+        self._post = post or requests.post
+        self._sleep = sleep or time.sleep
+
+    def evaluate(self, prompts: Sequence[str], conversations: Sequence[dict]) -> dict:
+        payload = {"prompts": list(prompts), "conversations": list(conversations)}
+        last_err = None
+        for attempt in range(self.retries + 1):
+            try:
+                resp = self._post(
+                    f"{self.base_url}/evaluate",
+                    headers={"Authorization": f"Bearer {self.token}", "Content-Type": "application/json"},
+                    json=payload,
+                    timeout=self.timeout,
+                )
+                resp.raise_for_status()
+                return resp.json()
+            except Exception as e:  # network error or HTTP error -> retry, then degrade
+                last_err = e
+                if attempt < self.retries:
+                    self._sleep(self.backoff * (2 ** attempt))
+        raise InferenceUnavailable(f"Modal evaluator unreachable after {self.retries + 1} attempts") from last_err
+
+
+class MockLoRAClient:
+    """Neutral canned labels sized to the inputs, so the full pipeline runs without real LoRAs.
+    Pass `responder(prompts, conversations) -> dict` to inject custom labels for testing."""
+
+    def __init__(self, responder=None):
+        self._responder = responder
+        self.calls: list = []
+
+    def evaluate(self, prompts: Sequence[str], conversations: Sequence[dict]) -> dict:
+        self.calls.append((list(prompts), list(conversations)))
+        if self._responder:
+            return self._responder(prompts, conversations)
+        return {
+            "technique": [{f: False for f in _TECH_FIELDS} for _ in prompts],
+            "context": [{"overall_richness": 1} for _ in prompts],
+            "interaction": [{"is_interactive_learning": False} for _ in conversations],
+            "focus": [{"is_focused": True} for _ in conversations],
+            "critical_engagement": [{"engagement_type": "passive_accept"} for _ in conversations],
+        }

prompt_card/llm/lora_router.py

@@ -0,0 +1,93 @@
+"""General per-category LoRA routing. Each locked adapter overrides ONE feature of ONE observable axis;
+all other features stay base 8B (per-category hybrid → no catastrophic forgetting). One Modal session per
+upload runs every enabled adapter.
+
+Locked adapters (validation κ, base → LoRA; see eval/decisions.md):
+  interaction      0.320 → 0.450   (refinement_attempt)
+  critical         per-type hybrid (Phase 7): LoRA for independent_verification 0.578 + source_request
+                   0.467→0.664; skepticism/rebuttal/re_questioning stay base. Hybrid headline 0.441→0.479.
+  goal_stated      0.226 → 0.419   (goal_stated, within Input Quality)
+  decomposition    0.261 → 0.656   (decomposition, within Technique; Phase-7 retrain, adapter decomposition_d_r16e5)
+
+Each adapter is TRUSTED only for its own feature(s); the LoRA's other outputs are ignored.
+"""
+from __future__ import annotations
+
+from eval.step_critical import CE
+from ..scoring.observable_axes import parse
+
+# axis -> (json fields parsed from the adapter output, the feature(s) the adapter is trusted for)
+ADAPTERS = {
+    "interaction": (("refinement_attempt",), {"refinement_attempt"}),
+    "critical": (CE, {"independent_verification"}),
+    "goal_stated": (("goal_stated",), {"goal_stated"}),
+    "decomposition": (("decomposition",), {"decomposition"}),
+}
+DEFAULT_ENABLED = frozenset(ADAPTERS)  # all locked adapters
+
+# Per-axis adapter-dir override on the Modal volume (/data/adapters/<dir>). When absent, evaluate()
+# defaults to the axis name. Phase 6 retrained decomposition into a suffixed dir (see eval/decisions.md);
+# point production at the winning adapter without copying volume files.
+ADAPTER_DIR = {
+    "decomposition": "decomposition_d_r16e5",  # Phase-7 winner: κ 0.656 (expanded data + hard negs; beat Phase-6 p6d 0.612)
+    "critical": "critical_c_r16e5",  # Phase-7: serves independent_verification + source_request (hybrid 0.441→0.479)
+}
+
+
+def run(requests: dict, *, workers=None, progress=None) -> dict:
+    """requests: {axis: [prompts]} -> {axis: [parsed dict]}. Returns {} for any axis whose call fails
+    (caller degrades that feature to base). `progress` (if given) is ticked per completed call.
+
+    PRODUCTION path (LORA_ENDPOINT=1): the configured OPENBMB endpoint is the Modal vLLM server that serves
+    every adapter by name (model=<adapter>), so LoRA inference is plain HTTP on the SAME warm server as base
+    — no `modal` package or Modal auth needed on the Space. Legacy path: per-adapter Modal H100 batch jobs."""
+    import os
+    if os.environ.get("LORA_ENDPOINT") and os.environ.get("OPENBMB_BASE_URL"):
+        return _run_via_endpoint(requests, workers=workers, progress=progress)
+    from modal_eval_lora import app, evaluate
+    out = {}
+    with app.run():
+        for axis, prompts in requests.items():
+            if not prompts:
+                out[axis] = []
+                continue
+            fields = ADAPTERS[axis][0]
+            raw = evaluate.remote(axis, prompts, adapter=ADAPTER_DIR.get(axis, ""))
+            out[axis] = [parse(r, fields) or {} for r in raw]
+            if progress is not None:
+                progress.tick(len(prompts))
+    return out
+
+
+def _run_via_endpoint(requests: dict, *, workers=None, progress=None) -> dict:
+    """Route ALL axes' prompts to the vLLM endpoint in ONE wide concurrent pool (not per-axis serial).
+    Each prompt goes to its adapter's served model name; one shared executor at `workers` concurrency
+    saturates the warm server. `progress` ticked per completed call."""
+    import os
+    import concurrent.futures as cf
+    from .minicpm import MiniCPMClient
+    base, token = os.environ["OPENBMB_BASE_URL"], os.environ.get("OPENBMB_TOKEN", "")
+    timeout = int(os.environ.get("OPENBMB_TIMEOUT") or "180")
+    workers = workers or max(1, int(os.environ.get("SCORE_CONCURRENCY") or "48"))
+    # one client per axis (each routes to its adapter's served model name on the same endpoint)
+    clients = {axis: MiniCPMClient(base, token, model=ADAPTER_DIR.get(axis, axis), max_tokens=96,
+                                   timeout=timeout) for axis in requests}
+    out = {axis: [None] * len(prompts) for axis, prompts in requests.items()}
+    jobs = [(axis, i, p) for axis, prompts in requests.items() for i, p in enumerate(prompts)]
+    if not jobs:
+        return {axis: [] for axis in requests}
+    with cf.ThreadPoolExecutor(max_workers=workers) as ex:
+        futs = {ex.submit(clients[axis].generate, p): (axis, i) for axis, i, p in jobs}
+        for f in cf.as_completed(futs):
+            axis, i = futs[f]
+            try:
+                out[axis][i] = parse(f.result(), ADAPTERS[axis][0]) or {}
+            except Exception:
+                out[axis][i] = {}   # this feature degrades to base for that turn
+            if progress is not None:
+                progress.tick()
+    return out
+
+
+def trusted(axis) -> set:
+    return ADAPTERS[axis][1]

prompt_card/llm/minicpm.py

@@ -0,0 +1,73 @@
+"""MiniCPM client over the OpenBMB free API (OpenAI-style vLLM endpoint).
+
+MiniCPM4.1-8B is a reasoning model: it can emit a `<think>...</think>` block and tends to wrap
+JSON in a markdown fence. We disable thinking (`enable_thinking=False`) for our simple JSON
+tasks and `clean_content` strips any leftover think-block / code fence so the existing judge and
+tips parsers work unchanged. HTTP is injectable (`post`) so unit tests need no network.
+"""
+from __future__ import annotations
+
+import re
+
+import requests
+
+DEFAULT_MODEL = "MiniCPM4.1-8B"
+_THINK = re.compile(r"<think>.*?</think>", re.S)
+
+
+def clean_content(text: str) -> str:
+    """Strip reasoning blocks and a surrounding markdown code fence."""
+    t = _THINK.sub("", text or "").strip()
+    if t.startswith("```"):
+        t = re.sub(r"^```[a-zA-Z0-9]*\n?", "", t)
+        t = re.sub(r"\n?```$", "", t)
+    return t.strip()
+
+
+class MiniCPMClient:
+    def __init__(
+        self,
+        base_url: str,
+        token: str,
+        *,
+        model: str = DEFAULT_MODEL,
+        max_tokens: int = 512,
+        timeout: int = 60,
+        enable_thinking: bool = False,
+        post=None,
+    ):
+        self.base_url = base_url.rstrip("/")
+        self.token = token
+        self.model = model
+        self.max_tokens = max_tokens
+        self.timeout = timeout
+        self.enable_thinking = enable_thinking
+        # Pool connections so the hundreds of concurrent scoring calls reuse TCP/TLS instead of a fresh
+        # handshake per request (a real per-call cost against a remote Modal endpoint). Injectable `post`
+        # (tests) bypasses the session entirely.
+        if post is not None:
+            self._post = post
+        else:
+            sess = requests.Session()
+            adapter = requests.adapters.HTTPAdapter(pool_connections=64, pool_maxsize=256)
+            sess.mount("http://", adapter)
+            sess.mount("https://", adapter)
+            self._post = sess.post
+
+    def generate(self, prompt: str) -> str:
+        body = {
+            "model": self.model,
+            "messages": [{"role": "user", "content": prompt}],
+            "temperature": 0,
+            "max_tokens": self.max_tokens,
+            "chat_template_kwargs": {"enable_thinking": self.enable_thinking},
+        }
+        resp = self._post(
+            f"{self.base_url}/v1/chat/completions",
+            headers={"Authorization": f"Bearer {self.token}", "Content-Type": "application/json"},
+            json=body,
+            timeout=self.timeout,
+        )
+        resp.raise_for_status()
+        content = resp.json()["choices"][0]["message"]["content"]
+        return clean_content(content)