Sync full submission repo state

.env.example

@@ -0,0 +1,62 @@
+FIGMENT_MODE=hosted
+MODEL_STACK=omni_native
+MODEL_BACKEND=
+AUDIO_BACKEND=none
+ENABLE_AUDIO_INTAKE=false
+ALLOW_LOCAL_ASR=false
+ALLOW_SELF_HOSTED_OMNI=false
+
+HF_MODEL_ID=nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16
+NVIDIA_MODEL_ID=nvidia/nemotron-3-nano-omni-30b-a3b-reasoning
+NVIDIA_BASE_URL=https://integrate.api.nvidia.com/v1
+NVIDIA_API_KEY=
+LOCAL_MODEL_ID=nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16
+HF_ENDPOINT_URL=
+OMNI_ENDPOINT_URL=
+HF_TOKEN=
+LLAMA_BASE_URL=http://127.0.0.1:8001/v1
+FIGMENT_TRACE_DIR=traces
+FIGMENT_SMOKE_ALLOW_NETWORK=false
+FIGMENT_SMOKE_TIMEOUT_SECONDS=8
+FIGMENT_SMOKE_TRACE_PATH=
+
+# Hosted-live Omni demo:
+# FIGMENT_MODE=hosted
+# MODEL_BACKEND=hosted_omni
+# NVIDIA_API_KEY=nvapi-...
+# AUDIO_BACKEND=omni_native
+# ENABLE_AUDIO_INTAKE=true
+
+# Local/offline proof. This can use the smaller local model, or self-hosted Omni
+# on adequate local hardware; smoke output records the configured LOCAL_MODEL_ID.
+# FIGMENT_MODE=local
+# MODEL_STACK=local_4b_parakeet
+# MODEL_BACKEND=llama_cpp
+# LLAMA_BASE_URL=http://127.0.0.1:8001/v1
+# LOCAL_MODEL_ID=nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16
+# AUDIO_BACKEND=none
+# FIGMENT_SMOKE_ALLOW_NETWORK=true
+#
+# Evidence bundle after the full-weight local endpoint is live:
+# PYTHON_DOTENV_DISABLED=true python3 scripts/run_local_4b_evidence.py --base-url "$LLAMA_BASE_URL"
+
+# Self-hosted Omni proof on adequate local hardware. This is the technical
+# Off the Grid Omni route when the endpoint is local and no cloud APIs are used.
+# FIGMENT_MODE=local
+# MODEL_STACK=omni_native
+# MODEL_BACKEND=llama_cpp
+# LOCAL_MODEL_ID=nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16
+# LLAMA_BASE_URL=http://127.0.0.1:8001/v1
+# ALLOW_SELF_HOSTED_OMNI=true
+
+# Optional local ASR proof. Enable only after Parakeet local ASR passes the gate.
+# MODEL_STACK=local_4b_parakeet
+# MODEL_BACKEND=llama_cpp
+# AUDIO_BACKEND=parakeet_nemo
+# ENABLE_AUDIO_INTAKE=true
+# ALLOW_LOCAL_ASR=true
+# PYTHON_DOTENV_DISABLED=true python3 scripts/run_local_asr_evidence.py --provider-payload <local-parakeet-provider-output.json>
+
+MODAL_PROFILE=
+MISTRAL_API_KEY=
+MINIMAX_API_KEY=

data/eval/field_workflow_holdout_v1_corrected_scoring_manifest.json

@@ -0,0 +1,1117 @@
+{
+  "changed_case_count": 6,
+  "changed_cases": [
+    {
+      "case_id": "field_workflow_holdout_v1-000019",
+      "changes": {
+        "expected_missing_observations": {
+          "after": [
+            "temperature if available",
+            "age or pregnancy status",
+            "mental status",
+            "neck stiffness report",
+            "rash report",
+            "hydration observations",
+            "available vital signs",
+            "confirmed intake status",
+            "deterministic rule results",
+            "retrieved protocol card IDs",
+            "navigator validation result",
+            "manual correction status for audio-derived fields",
+            "situation or reason for handoff",
+            "relevant background and timeline",
+            "objective observations only",
+            "red flags already fired",
+            "specific request or receiving pathway",
+            "source protocol card IDs",
+            "pregnancy or postpartum status",
+            "bleeding report",
+            "abdominal pain report",
+            "headache or vision symptoms",
+            "seizure or fainting report",
+            "fever report"
+          ],
+          "before": [
+            "temperature if available",
+            "age or pregnancy status",
+            "mental status",
+            "neck stiffness report",
+            "rash report",
+            "hydration observations",
+            "available vital signs",
+            "confirmed intake status",
+            "deterministic rule results",
+            "retrieved protocol card IDs",
+            "navigator validation result",
+            "manual correction status for audio-derived fields",
+            "situation or reason for handoff",
+            "relevant background and timeline",
+            "objective observations only",
+            "red flags already fired",
+            "specific request or receiving pathway",
+            "source protocol card IDs",
+            "pregnancy or postpartum status",
+            "bleeding report",
+            "abdominal pain report",
+            "headache or vision symptoms",
+            "seizure or fainting report",
+            "fever report",
+            "chest pain description",
+            "onset and duration",
+            "shortness of breath report",
+            "sweating or fainting report",
+            "radiation to arm, jaw, back, or shoulder"
+          ]
+        },
+        "expected_red_flag_rule_ids": {
+          "after": [
+            "PREG-001",
+            "FEVER-001"
+          ],
+          "before": [
+            "PREG-001",
+            "red_flag_chest_pain",
+            "FEVER-001"
+          ]
+        },
+        "expected_source_card_ids": {
+          "after": [
+            "FEVER-RED-FLAGS-v1",
+            "SAFETY-BOUNDARIES-v1",
+            "REFERRAL-SBAR-v1",
+            "PREG-DANGER-SIGNS-v1"
+          ],
+          "before": [
+            "FEVER-RED-FLAGS-v1",
+            "SAFETY-BOUNDARIES-v1",
+            "REFERRAL-SBAR-v1",
+            "PREG-DANGER-SIGNS-v1",
+            "CHEST-PAIN-ESCALATION-v1"
+          ]
+        }
+      }
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000050",
+      "changes": {
+        "expected_min_protocol_urgency": {
+          "after": "routine",
+          "before": "emergency"
+        },
+        "expected_missing_observations": {
+          "after": [
+            "confirmed intake status",
+            "deterministic rule results",
+            "retrieved protocol card IDs",
+            "navigator validation result",
+            "manual correction status for audio-derived fields",
+            "situation or reason for handoff",
+            "relevant background and timeline",
+            "objective observations only",
+            "red flags already fired",
+            "specific request or receiving pathway",
+            "source protocol card IDs"
+          ],
+          "before": [
+            "confirmed intake status",
+            "deterministic rule results",
+            "retrieved protocol card IDs",
+            "navigator validation result",
+            "manual correction status for audio-derived fields",
+            "situation or reason for handoff",
+            "relevant background and timeline",
+            "objective observations only",
+            "red flags already fired",
+            "specific request or receiving pathway",
+            "source protocol card IDs",
+            "chest pain description",
+            "onset and duration",
+            "shortness of breath report",
+            "sweating or fainting report",
+            "radiation to arm, jaw, back, or shoulder",
+            "available vital signs"
+          ]
+        },
+        "expected_red_flag_rule_ids": {
+          "after": [],
+          "before": [
+            "red_flag_chest_pain"
+          ]
+        },
+        "expected_source_card_ids": {
+          "after": [
+            "SAFETY-BOUNDARIES-v1",
+            "REFERRAL-SBAR-v1"
+          ],
+          "before": [
+            "SAFETY-BOUNDARIES-v1",
+            "REFERRAL-SBAR-v1",
+            "CHEST-PAIN-ESCALATION-v1"
+          ]
+        }
+      }
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000067",
+      "changes": {
+        "expected_missing_observations": {
+          "after": [
+            "temperature if available",
+            "age or pregnancy status",
+            "mental status",
+            "neck stiffness report",
+            "rash report",
+            "hydration observations",
+            "available vital signs",
+            "confirmed intake status",
+            "deterministic rule results",
+            "retrieved protocol card IDs",
+            "navigator validation result",
+            "manual correction status for audio-derived fields",
+            "situation or reason for handoff",
+            "relevant background and timeline",
+            "objective observations only",
+            "red flags already fired",
+            "specific request or receiving pathway",
+            "source protocol card IDs",
+            "pregnancy or postpartum status",
+            "bleeding report",
+            "abdominal pain report",
+            "headache or vision symptoms",
+            "seizure or fainting report",
+            "fever report"
+          ],
+          "before": [
+            "temperature if available",
+            "age or pregnancy status",
+            "mental status",
+            "neck stiffness report",
+            "rash report",
+            "hydration observations",
+            "available vital signs",
+            "confirmed intake status",
+            "deterministic rule results",
+            "retrieved protocol card IDs",
+            "navigator validation result",
+            "manual correction status for audio-derived fields",
+            "situation or reason for handoff",
+            "relevant background and timeline",
+            "objective observations only",
+            "red flags already fired",
+            "specific request or receiving pathway",
+            "source protocol card IDs",
+            "pregnancy or postpartum status",
+            "bleeding report",
+            "abdominal pain report",
+            "headache or vision symptoms",
+            "seizure or fainting report",
+            "fever report",
+            "chest pain description",
+            "onset and duration",
+            "shortness of breath report",
+            "sweating or fainting report",
+            "radiation to arm, jaw, back, or shoulder"
+          ]
+        },
+        "expected_red_flag_rule_ids": {
+          "after": [
+            "PREG-001",
+            "FEVER-001"
+          ],
+          "before": [
+            "PREG-001",
+            "red_flag_chest_pain",
+            "FEVER-001"
+          ]
+        },
+        "expected_source_card_ids": {
+          "after": [
+            "FEVER-RED-FLAGS-v1",
+            "SAFETY-BOUNDARIES-v1",
+            "REFERRAL-SBAR-v1",
+            "PREG-DANGER-SIGNS-v1"
+          ],
+          "before": [
+            "FEVER-RED-FLAGS-v1",
+            "SAFETY-BOUNDARIES-v1",
+            "REFERRAL-SBAR-v1",
+            "PREG-DANGER-SIGNS-v1",
+            "CHEST-PAIN-ESCALATION-v1"
+          ]
+        }
+      }
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000091",
+      "changes": {
+        "expected_missing_observations": {
+          "after": [
+            "temperature if available",
+            "age or pregnancy status",
+            "mental status",
+            "neck stiffness report",
+            "rash report",
+            "hydration observations",
+            "available vital signs",
+            "confirmed intake status",
+            "deterministic rule results",
+            "retrieved protocol card IDs",
+            "navigator validation result",
+            "manual correction status for audio-derived fields",
+            "situation or reason for handoff",
+            "relevant background and timeline",
+            "objective observations only",
+            "red flags already fired",
+            "specific request or receiving pathway",
+            "source protocol card IDs",
+            "pregnancy or postpartum status",
+            "bleeding report",
+            "abdominal pain report",
+            "headache or vision symptoms",
+            "seizure or fainting report",
+            "fever report"
+          ],
+          "before": [
+            "temperature if available",
+            "age or pregnancy status",
+            "mental status",
+            "neck stiffness report",
+            "rash report",
+            "hydration observations",
+            "available vital signs",
+            "confirmed intake status",
+            "deterministic rule results",
+            "retrieved protocol card IDs",
+            "navigator validation result",
+            "manual correction status for audio-derived fields",
+            "situation or reason for handoff",
+            "relevant background and timeline",
+            "objective observations only",
+            "red flags already fired",
+            "specific request or receiving pathway",
+            "source protocol card IDs",
+            "pregnancy or postpartum status",
+            "bleeding report",
+            "abdominal pain report",
+            "headache or vision symptoms",
+            "seizure or fainting report",
+            "fever report",
+            "chest pain description",
+            "onset and duration",
+            "shortness of breath report",
+            "sweating or fainting report",
+            "radiation to arm, jaw, back, or shoulder"
+          ]
+        },
+        "expected_red_flag_rule_ids": {
+          "after": [
+            "PREG-001",
+            "FEVER-001"
+          ],
+          "before": [
+            "PREG-001",
+            "red_flag_chest_pain",
+            "FEVER-001"
+          ]
+        },
+        "expected_source_card_ids": {
+          "after": [
+            "FEVER-RED-FLAGS-v1",
+            "SAFETY-BOUNDARIES-v1",
+            "REFERRAL-SBAR-v1",
+            "PREG-DANGER-SIGNS-v1"
+          ],
+          "before": [
+            "FEVER-RED-FLAGS-v1",
+            "SAFETY-BOUNDARIES-v1",
+            "REFERRAL-SBAR-v1",
+            "PREG-DANGER-SIGNS-v1",
+            "CHEST-PAIN-ESCALATION-v1"
+          ]
+        }
+      }
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000099",
+      "changes": {
+        "expected_missing_observations": {
+          "after": [
+            "situation or reason for handoff",
+            "relevant background and timeline",
+            "objective observations only",
+            "red flags already fired",
+            "specific request or receiving pathway",
+            "source protocol card IDs",
+            "confirmed intake status",
+            "deterministic rule results",
+            "retrieved protocol card IDs",
+            "navigator validation result",
+            "manual correction status for audio-derived fields",
+            "available vital signs",
+            "temperature if available",
+            "age or pregnancy status",
+            "mental status",
+            "neck stiffness report",
+            "rash report",
+            "hydration observations"
+          ],
+          "before": [
+            "situation or reason for handoff",
+            "relevant background and timeline",
+            "objective observations only",
+            "red flags already fired",
+            "specific request or receiving pathway",
+            "source protocol card IDs",
+            "confirmed intake status",
+            "deterministic rule results",
+            "retrieved protocol card IDs",
+            "navigator validation result",
+            "manual correction status for audio-derived fields",
+            "chest pain description",
+            "onset and duration",
+            "shortness of breath report",
+            "sweating or fainting report",
+            "radiation to arm, jaw, back, or shoulder",
+            "available vital signs",
+            "temperature if available",
+            "age or pregnancy status",
+            "mental status",
+            "neck stiffness report",
+            "rash report",
+            "hydration observations"
+          ]
+        },
+        "expected_red_flag_rule_ids": {
+          "after": [
+            "PREG-001",
+            "FEVER-001"
+          ],
+          "before": [
+            "PREG-001",
+            "red_flag_chest_pain",
+            "FEVER-001"
+          ]
+        },
+        "expected_source_card_ids": {
+          "after": [
+            "REFERRAL-SBAR-v1",
+            "SAFETY-BOUNDARIES-v1",
+            "FEVER-RED-FLAGS-v1"
+          ],
+          "before": [
+            "REFERRAL-SBAR-v1",
+            "SAFETY-BOUNDARIES-v1",
+            "CHEST-PAIN-ESCALATION-v1",
+            "FEVER-RED-FLAGS-v1"
+          ]
+        }
+      }
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000115",
+      "changes": {
+        "expected_missing_observations": {
+          "after": [
+            "temperature if available",
+            "age or pregnancy status",
+            "mental status",
+            "neck stiffness report",
+            "rash report",
+            "hydration observations",
+            "available vital signs",
+            "confirmed intake status",
+            "deterministic rule results",
+            "retrieved protocol card IDs",
+            "navigator validation result",
+            "manual correction status for audio-derived fields",
+            "situation or reason for handoff",
+            "relevant background and timeline",
+            "objective observations only",
+            "red flags already fired",
+            "specific request or receiving pathway",
+            "source protocol card IDs",
+            "pregnancy or postpartum status",
+            "bleeding report",
+            "abdominal pain report",
+            "headache or vision symptoms",
+            "seizure or fainting report",
+            "fever report"
+          ],
+          "before": [
+            "temperature if available",
+            "age or pregnancy status",
+            "mental status",
+            "neck stiffness report",
+            "rash report",
+            "hydration observations",
+            "available vital signs",
+            "confirmed intake status",
+            "deterministic rule results",
+            "retrieved protocol card IDs",
+            "navigator validation result",
+            "manual correction status for audio-derived fields",
+            "situation or reason for handoff",
+            "relevant background and timeline",
+            "objective observations only",
+            "red flags already fired",
+            "specific request or receiving pathway",
+            "source protocol card IDs",
+            "pregnancy or postpartum status",
+            "bleeding report",
+            "abdominal pain report",
+            "headache or vision symptoms",
+            "seizure or fainting report",
+            "fever report",
+            "chest pain description",
+            "onset and duration",
+            "shortness of breath report",
+            "sweating or fainting report",
+            "radiation to arm, jaw, back, or shoulder"
+          ]
+        },
+        "expected_red_flag_rule_ids": {
+          "after": [
+            "PREG-001",
+            "FEVER-001"
+          ],
+          "before": [
+            "PREG-001",
+            "red_flag_chest_pain",
+            "FEVER-001"
+          ]
+        },
+        "expected_source_card_ids": {
+          "after": [
+            "FEVER-RED-FLAGS-v1",
+            "SAFETY-BOUNDARIES-v1",
+            "REFERRAL-SBAR-v1",
+            "PREG-DANGER-SIGNS-v1"
+          ],
+          "before": [
+            "FEVER-RED-FLAGS-v1",
+            "SAFETY-BOUNDARIES-v1",
+            "REFERRAL-SBAR-v1",
+            "PREG-DANGER-SIGNS-v1",
+            "CHEST-PAIN-ESCALATION-v1"
+          ]
+        }
+      }
+    }
+  ],
+  "dataset_version": "field_workflow_holdout_v1_corrected_scoring",
+  "derived_from_dataset_version": "field_workflow_holdout_v1",
+  "generated_at": "2026-06-12T21:53:41.462695+00:00",
+  "output_path": "data/eval/field_workflow_holdout_v1_corrected_scoring.jsonl",
+  "output_sha256": "c1115a32cf85f19fb446e037a62a10bb33f649d4b2b87a00a5379d55ae24782e",
+  "policy": {
+    "preserve_case_ids": true,
+    "preserve_frozen_holdout_file": true,
+    "recompute_expected_labels_with_current_rules": true
+  },
+  "row_count": 150,
+  "row_hashes": [
+    {
+      "case_id": "field_workflow_holdout_v1-000000",
+      "sha256": "sha256:b523f686fe7a990545f1b898ae7cfc64b5ba93710a9c8f18f7af8f63119d8194"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000001",
+      "sha256": "sha256:f1ffa233e891324d9590f3bf4795cb0364cae62abe591c04078fff955f51a262"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000002",
+      "sha256": "sha256:64d335b68060349433a926484ec39f93bb5eb98961d0d262f7eb517951f905cf"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000003",
+      "sha256": "sha256:41d266fa381d6a1a080be2ca2e0f509cf0296a2faadc1348ee31624c4ece68aa"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000004",
+      "sha256": "sha256:9afed3c660b56908575057f0b47ff59c2e79d40c8a64bc0cfc188c49d8c59799"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000005",
+      "sha256": "sha256:ea10f38f2ce20376acb16693e6c3b4c9545afa136537c0b6bf4beadf69237afa"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000006",
+      "sha256": "sha256:b6673de2bca31ef5876da374313cafc10909ea6c295143aee98ec5d46971bb0d"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000007",
+      "sha256": "sha256:5926d69bfcce4b4f1bb0e473454253e381ed8976cc357fef090f6f9ccc74dffe"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000008",
+      "sha256": "sha256:aeca2d50f4c064a63a4881edf2c550dff8a0c72500cc8b22d9c100e5fac1dcdf"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000009",
+      "sha256": "sha256:a10e49049fde7e48d17bd59f8a5a563c8a8e8a141945d1e0bc297152a6bcdc90"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000010",
+      "sha256": "sha256:63cfafe761548e70850bfe7982ec96e2d1f41716ae5c19b858c4b8a5a26f4df5"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000011",
+      "sha256": "sha256:7470038c1eba7eebadb50dcd2be2516325033f09f400f4774230f9cc2d79e4a9"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000012",
+      "sha256": "sha256:47c593492f91709f052b4d939ac10e4b6351cbbfe0ac6477e8c7ba2b40ea84ca"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000013",
+      "sha256": "sha256:e830c436dfb88962078b278b35b4d9ca9e49d9d6b0e3e3c74ef8ab66d064e8af"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000014",
+      "sha256": "sha256:e1bf283f91612f2b7711ba6106cbff326811fb4ffbef920ebff8ea6fe2ed74a2"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000015",
+      "sha256": "sha256:9f4fbba186e2ab20a8c51e4aaebf724b84aa539888a289dad5ca9fb272cb4c7c"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000016",
+      "sha256": "sha256:a226740ec8e2bc8afd2d1703eafff4c794eb144df99d9047db9beb51d190d13c"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000017",
+      "sha256": "sha256:4c56deb575cd15ffccef6ab1dea0816c8b6fe519f37bd86c3092dcd4bfe77531"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000018",
+      "sha256": "sha256:1e2bbaefe7f677b9a95144dd90b0f5fba4c2d5a49d80a55be32ed038a5f203e5"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000019",
+      "sha256": "sha256:f20f362f9e7d3b02e84dc122ee1492f14e0c4d53b86ef8800fdee192b8e4ee63"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000020",
+      "sha256": "sha256:41c7cc8d2048010630305a9ca592e0ac39e419db29f903f2bc9debb5abd27fec"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000021",
+      "sha256": "sha256:e1edb67a260ba9a0e3c40a959b332e145daa7a95396942410b0aab00e9fd6398"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000022",
+      "sha256": "sha256:a99f8c8fdb162b05381e48ea48caa23d1073f87d590d7fad41a77b31d2b69864"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000023",
+      "sha256": "sha256:f0b56b9d2005d79a6e6bd85445be3f97c22883fb88f84a9aaebd064db4e65c42"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000024",
+      "sha256": "sha256:46a89b5ab600251cbce1564a6d96c539692a0925a65e93be6a966471d62ad359"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000025",
+      "sha256": "sha256:3e2643b3831ff12c0e3c7e5e853b0b3120928eb3752e64ef3dfb6cf4a3d7a5fa"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000026",
+      "sha256": "sha256:7c9c4dcac0c81074b40df927163d26b2626aa4b06a91db1d1511851eaf6b5f66"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000027",
+      "sha256": "sha256:a4331633305a47a31883a0d15f4934f68d1a2ac259b7e1a39f7f72effca42067"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000028",
+      "sha256": "sha256:0d43bbd590660fe7a246d6d7bcf52e940a91a102af78b6c03c4d7eb173d238de"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000029",
+      "sha256": "sha256:c796bf3fe85d4810235447bdaafbe575f6745918a6b2b087c4d1cb27c3642948"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000030",
+      "sha256": "sha256:f828c2279938bda39b2a26edfc9e0f19bd3da35bdd806e9ea74811ca7f7a116a"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000031",
+      "sha256": "sha256:70170d5679b7256bd760e7841ab98d4bd45e495e6681b0acc7778c7f39012b75"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000032",
+      "sha256": "sha256:a6fed05b1a4c411a321cd78f09d5d854d8fc30269cbbbc5649cd1b9268cf02ee"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000033",
+      "sha256": "sha256:d2ffee370489ee960f06ed4968ce342aa01e23e43b944d6e725430bc1bdd2f4e"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000034",
+      "sha256": "sha256:fd36f66ab06be4ebbc90d5f5c81d3eab9a76acf54d8566e26d78c85dd0cac1b5"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000035",
+      "sha256": "sha256:28e349af37dac4283b751aa5c8e3ed12f00df21627bee3204bf20086d7a01ab7"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000036",
+      "sha256": "sha256:6e36deb5336beda564e45ce709202f4e2cffe4094b447411b47a801e3b1ee9d1"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000037",
+      "sha256": "sha256:8cbe78cdf162cb2b86fe150df0d8e8e26b6415c6447739a4d530ceb1c1836576"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000038",
+      "sha256": "sha256:25230b1b4ad8f376669b7d7053cc1289f781f9247cb75c51796610f524184afa"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000039",
+      "sha256": "sha256:2c21af4d2dbc07b008b809c00dcf4b2ff56bef25e669fef6b1692a37008c8601"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000040",
+      "sha256": "sha256:bb27435b3aba56ac3a5a2ee5842283412f399d9dae7e6f02bfc572d031def445"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000041",
+      "sha256": "sha256:25680220ab169f115739fe6ccf90cc160f3d3d6df459cca0bae890dac7578b6d"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000042",
+      "sha256": "sha256:7c9cc913412ee6eb41ce75bd95d80e49fb803e51fc0493c7f055b92376a8d86f"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000043",
+      "sha256": "sha256:610aad3828bc47434a5ae26955cbe49029241cdbc055af66ff2c3f2b79c979a7"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000044",
+      "sha256": "sha256:bc77d99f54533f7667e8f4f2ac789c5b2753dcfbd0d9325089bdbefe9892b603"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000045",
+      "sha256": "sha256:79875a5a1079993271a3ab14d23a13f3da3d577fb89cc255a9959c795d04ea69"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000046",
+      "sha256": "sha256:eefd5fd5ebef634356ce28abfea0e332632bfcb880c28da8b0ec8f4d49a0f8ed"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000047",
+      "sha256": "sha256:4eadceba9bf87b4e767d481b8710766569f7940ef2769d2054e06912bdbeb82c"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000048",
+      "sha256": "sha256:914ede0e64a925cf9d7b7960ff9ca3378bffd9524caad04b38df41f8e7e80e7a"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000049",
+      "sha256": "sha256:61ef5af4dc111ae1ccbb4034983f7b680114ac15deb1a18eb61987f8dfbfe6da"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000050",
+      "sha256": "sha256:754b0f07eddeecaef5fc0723cf13506b6ea08efde251354c9edbd6c2a4eb5582"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000051",
+      "sha256": "sha256:f1c64b64c10d2b28178fdcd1cc9eb730c41351bd6e6734f9bbfbb63ae1eb793e"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000052",
+      "sha256": "sha256:ce98df4c4e3ec879f708ee749af88855c1b34d284671638a113835c33934d0c8"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000053",
+      "sha256": "sha256:edae1a4f4340597e9e04e40415719dfa3c8efabc01f912a19dbca76ad27059f1"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000054",
+      "sha256": "sha256:718935ca8cc0d7cc0db6cca893569a5030e97f962a8e33cb881b7b81ec616088"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000055",
+      "sha256": "sha256:1c6db577e78c1ca35a8179157a4b967fa42170cad85cf2abc4b7a652871e93d5"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000056",
+      "sha256": "sha256:a6d98b03f0f7e6bcbaee344c41369023183ab901addad9dddc0538a26d9da38b"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000057",
+      "sha256": "sha256:0dfb01c5c5d8fc546e97cc94b5eedd09d891d60b44edf379680e2e89335ff73d"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000058",
+      "sha256": "sha256:6f61bdbc37a4212c96f66bf65231e9a1d15ed0cab14a3884dd2db9ff6cbb085a"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000059",
+      "sha256": "sha256:003bb99e4d0ff746ce57c1662cdfe29ce8e62d69a7d2382577f7b58d069116ab"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000060",
+      "sha256": "sha256:441b222dc0b55c7c302f22aa4438afd8c51653609286efb913d275cd8b560768"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000061",
+      "sha256": "sha256:7fca0d9e1543f1cfaddc15eb7e54ed42ce75101fe6794398b134cf77ec52dcec"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000062",
+      "sha256": "sha256:e9050729788793ff674b716be91f98f3a7a72d8336e9121e7b56ec6c69bb2c69"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000063",
+      "sha256": "sha256:8fdb4b1cd18354aa25132b37d1511e0e4b2f35d9ae28ac0692ebde06117b555f"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000064",
+      "sha256": "sha256:cfe40c12f2742ef6afa84c704f5b768ddb34983f1cef5aecc3d658a4d522b01b"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000065",
+      "sha256": "sha256:2c4d4ef9f03d85dbba6124e4ea5994c384e304c94ec48415e5f50939f9b319a5"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000066",
+      "sha256": "sha256:2edb0587c93234986eecda00764fd2fc2010bc8365a0765676a5d45dae78cb32"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000067",
+      "sha256": "sha256:45ca03910ef644374e2897b0707cc0edc253d10cb7a2c8608b12cf5858fc05ee"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000068",
+      "sha256": "sha256:8310654675c6de31fc2b3815c1ff11d6802a3adbc868755f0e822d7ac962e23f"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000069",
+      "sha256": "sha256:88dd19896db044f06c6530766604c8bb2fa9b9acd361e74a646aa02f03a46fb9"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000070",
+      "sha256": "sha256:6b8d4987287f45a6d01c4478effd2f2642b1a31111a9195a73e9fab8f9d34d5b"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000071",
+      "sha256": "sha256:20a7d2cf386803acfd91e8f019c5790e806d43e4c39474d0462726c506388b40"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000072",
+      "sha256": "sha256:97eff29f1ca3d89279e26c4620588b9f11bc66be229d12f2e53a6f76845b1962"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000073",
+      "sha256": "sha256:31512842fa8c91c3f235f88f7a6eeda9e0da8446c68ff74f6cf6090c7fcec6a7"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000074",
+      "sha256": "sha256:b30c9060da7b1036a2d112a866957fa027bbcad2b2becfbcc8bbbc084e7173a6"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000075",
+      "sha256": "sha256:85343dd2c8d986893b35047fea45a1cbb16ad90c28a4b55a4c57f85fb38053e3"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000076",
+      "sha256": "sha256:903bf13f5faeae79e75c8f361aed1d084784b1b9058d35768a6e05cf07c9c915"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000077",
+      "sha256": "sha256:d72bc8ac57f7cbfbb8130fa619abffbd8fd88e647b78c37c8ed18ef08bc8e06a"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000078",
+      "sha256": "sha256:7564328aed67f531944289ba89302817e6113e74104d7b2a9149bc61ee513402"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000079",
+      "sha256": "sha256:a1ddbb58a922b6a3c9fc30aaa4fb8ff222f4ba44a08858bbe0f598b63ac2c5da"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000080",
+      "sha256": "sha256:30b66e6f100d26169f82fb16dc778b43c54f5e71031049d0cafad6c8bc922860"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000081",
+      "sha256": "sha256:acb13f947ebf85060278e123da51be50b7258d20c75f9554d7a3b878f0b8d0a9"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000082",
+      "sha256": "sha256:52a7b5c8249f981b61014888137390e270b542b197839fe1a2b910baf4596166"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000083",
+      "sha256": "sha256:b31b964a3ac6f45e69f1755d480fe2ee043e78121f5f56aadb920e5a14a9721e"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000084",
+      "sha256": "sha256:c282803e29e9bf5fb8368f2da993513bffd6c23b2235df1131a804b952ca45d6"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000085",
+      "sha256": "sha256:3a611fcaee66cf4b6e7ab112e134c352b9c4d42ac307d9194c91b9ba2ebc85af"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000086",
+      "sha256": "sha256:349a69890aa5d1dd3bf9451e068ef3efd73d562ca052cc67908d8d2584f0edcc"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000087",
+      "sha256": "sha256:3d8a5a218ce2f5cf3a77f5fefddbc351b0d2f73eb51e57048aa6c7c972c8811a"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000088",
+      "sha256": "sha256:b3deaf4dde1986bbf44c3e15c2a6b7acd898f81335009cf5fc7f27a371d47910"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000089",
+      "sha256": "sha256:4b95817e8473e70a89dbcf7e95cfc450fdfbcf43d27ff64338d1f96fa71175fb"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000090",
+      "sha256": "sha256:d99f7d71a15e298ebcc8aba4e120a597c7355dbc5b30d08e5996640dbd6a773a"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000091",
+      "sha256": "sha256:3df7867ced26b4cf4e51978315b955ff3e3ca18f07029c00973de75fccffb39f"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000092",
+      "sha256": "sha256:5d343d5a8b98448d960f326f2b76b18f96b30e6b26bc980ffb4f29ce04cfe61b"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000093",
+      "sha256": "sha256:f3ea61aa9a8982ff754eec887a0b38ac522a248ff5c0ce21c1752dcd1d0d43a1"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000094",
+      "sha256": "sha256:612b7ffdcbb1172fcb33365215a50b6bfbbec1f3a2649032dbcad2599c46d126"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000095",
+      "sha256": "sha256:776dc4126afd609ace30e06e5502daf2ed306003c42944e0ded67c163e435171"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000096",
+      "sha256": "sha256:faf7c0c5e513fd36461ed323c85b75796e5210441682a9432175e86f7373e135"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000097",
+      "sha256": "sha256:30ca92a074cbeac4d3e7eb4131ea0107bd80ea670ac0c9af6301523ec58e3f19"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000098",
+      "sha256": "sha256:e45b0992b91475879f9b2714e8e48ffe16cf60a67c79d0cbda69d58f46be8201"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000099",
+      "sha256": "sha256:09d35b7313f109c1a6bed45d5a51b66a84e08ba9b8bbb89b25da43de30acdba1"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000100",
+      "sha256": "sha256:fbedaf1d1fb86760f2549dc7c1b3e6003c1e044bd9544b4a32ea63a383c4f04b"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000101",
+      "sha256": "sha256:1c8e789b3254fe79c2e9395e621a391392c4d11b0a306e9a89a03074a8cc2e57"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000102",
+      "sha256": "sha256:c49cb3198d71f486a3127dc403fecf7b38e39ef3f57633af126b4c5625b71bdf"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000103",
+      "sha256": "sha256:d9f690e19da1fc9f96d01cb67324e68e9f43546ce004e5951253161eed763f94"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000104",
+      "sha256": "sha256:a07eede670645398e2089f1c57de3613287cff76fd02bd183c3e75a139f6bf4c"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000105",
+      "sha256": "sha256:8907f0a80f30b9fce4cd8a8047f3d85b8bf7f900cf7613c549934d1711a0d0cd"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000106",
+      "sha256": "sha256:f5445058c09c814b3e09ea47774f59e8c2fa75a281b2e5bbbcac6ca5d484ef7e"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000107",
+      "sha256": "sha256:3ec54340c0687c60467e80a700112c6f5841609c086df93cb48cd4a3a1d2c482"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000108",
+      "sha256": "sha256:5d0eb91516021f465a4e514b14788f94835d691e97907a99c349e35df5579aa5"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000109",
+      "sha256": "sha256:e72895f2486acda1deb38249b8bf7b05076055675baa13418c927d44189e54b2"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000110",
+      "sha256": "sha256:c14208b84cb4c475f164af672f57409319866120a5f5ff8c134fb1ccce6dfb64"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000111",
+      "sha256": "sha256:ba5bc53f4bf65cc85d3e06d100a0633b1d542b30521536cdd42fd5c9bec26447"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000112",
+      "sha256": "sha256:97e0672591f4be38909e9a95f0e911feda682a9beb644f927e9d3aac117ec910"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000113",
+      "sha256": "sha256:b6fb9b0663bfc22e14cd4377aabbf4e67f38535486e262b148dadf080bd1d68d"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000114",
+      "sha256": "sha256:4ed2723e33801002c1af388713205310683738ea2ba65d16eddaafdf8523a075"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000115",
+      "sha256": "sha256:66d339c9f2964dfa65e4dd7e289f9937fab77f8c7f8d2d917f1d8f488e7a0be6"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000116",
+      "sha256": "sha256:19979a5a2da873aa708932bb3795220b7269db611f9b2fe8f08868a333f2fd10"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000117",
+      "sha256": "sha256:82652ae5d37502115b7b61178e6a9cf9a75fb131fed6aca4280e6d40b437e35e"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000118",
+      "sha256": "sha256:190f8debce7ffc4e3b463bfa55610b6f3ce92db02797169b56576e833e10db7a"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000119",
+      "sha256": "sha256:7d2461cb4438d8521ff62c8d7dc104ca9a4bdb2823b2839df5060c027d3cd135"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000120",
+      "sha256": "sha256:ed36395c682de3af1c81764372705486412065141bbfbffed65f1b95922b755c"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000121",
+      "sha256": "sha256:fa01ebe5f65ba6dd6a66c23e21473f0f7fa6c2cea630d965758abad6ca24c29c"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000122",
+      "sha256": "sha256:722cd84b2c316a6e7696aee936e89e65668f826b373dcdc29792d1e053f88285"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000123",
+      "sha256": "sha256:caf7e4b51db89d2dc75b67eb95b4e5caefe9749a70e6fcd86ed28cf46d7f616a"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000124",
+      "sha256": "sha256:5368f18d1677320af8859b5e92421df93b0b0d591f420880d63bdb0ee21f2bf0"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000125",
+      "sha256": "sha256:d1ddaecebe4c9bf001ac85a5576854af1974792190202e8638473117f4106b6e"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000126",
+      "sha256": "sha256:605153e712d9c56539473ed085ac6fc57d38377eb7180fa2cbebb642153e8741"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000127",
+      "sha256": "sha256:52b9c6e4dd3a9fb3f48050bf4bfc7a5c12e51a0a44417c45a64c822399f8711e"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000128",
+      "sha256": "sha256:38a3da4938ede95de52882761220e520530b938eff0490bbb80ce98a7d08a966"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000129",
+      "sha256": "sha256:5aba15ec39407263c0ecfec9157429573f5c9f86bd250daeeaf6fb09b34eef79"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000130",
+      "sha256": "sha256:1071b4867bd2bb5d420cfb447cd74e30c1f80c236f2a941475fa4b3f9f79db38"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000131",
+      "sha256": "sha256:ce479fdedd69d73cdf7c964a01aeb9e5573a9589d90640dc13c074ccd8789d4a"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000132",
+      "sha256": "sha256:75598ff960d7ba18813a747111012331cf6fe69e3097c8abd0f676c5fefb56fa"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000133",
+      "sha256": "sha256:e9919db6f00728fef4014054390d5fd26e9bdc5aa84da6d7098250c4f4aea7d5"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000134",
+      "sha256": "sha256:6eb80dcebbd876993d9b6f548523c13dc4704abf6c0391c263b8660b41dc022c"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000135",
+      "sha256": "sha256:9fb35aff36423a974d3ad4318ae4c5d9b2cc333d4156eb2b7c286993b6f99412"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000136",
+      "sha256": "sha256:c8924b251340754f97814fa07c363c6f39d976d77b40a62f944ab5b0477983f8"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000137",
+      "sha256": "sha256:ff19ab6531a58b00a2ddc0dbefc3228ae5e79de6ff968994c7bb157261cb5394"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000138",
+      "sha256": "sha256:2b0dc7c408a3fd0305823a3f296252f0993a12e4bba5b7f23de4e9dce6ce7ccd"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000139",
+      "sha256": "sha256:b7b035ab89925e970053ec72670d3d4d0a8a8cf2036b675f806f1460723482a7"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000140",
+      "sha256": "sha256:3f58f4d0e53381cf81d66e2f502b542bf80a8869dabd8287ae1d5be18f63620c"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000141",
+      "sha256": "sha256:9a3b9d0ba7ef168dd95a8ab7254300428afe3e4d874bd37b5485b2f8177274e9"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000142",
+      "sha256": "sha256:50bd5bb4dd8916ea2a31426245830eb627a7f9cfb40877a22c24e98ed07608d3"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000143",
+      "sha256": "sha256:e974e3004b4c84a657548712c496bc6f9484e83558e89ff1a07b566a52631194"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000144",
+      "sha256": "sha256:a0d5e7e46de5e5a3b06c4fa12216f88ada23922f1a17fa471cec246ba01586e5"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000145",
+      "sha256": "sha256:21847d213076309a843157c9a0c8f62ef209e94ba0c674dfca20f487105859f4"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000146",
+      "sha256": "sha256:9934d6464070ce42e3d3bafc1acab0d609d21221ab1c4b2c909c77327dfde83c"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000147",
+      "sha256": "sha256:fbfbaed9f88899e364056ed0ab139aaa4321757c1445716e393428276d121324"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000148",
+      "sha256": "sha256:b692bf0096e899e457ffe204e87902984701ed971c9a6d89c665c5ee2bc765c3"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000149",
+      "sha256": "sha256:bfaace67878a2f5b17e9d504c7d46e5c8f260a1cdbdde3e4282453c14a32d570"
+    }
+  ],
+  "source_generator": "scripts/build_corrected_field_workflow_holdout.py",
+  "source_path": "data/eval/field_workflow_holdout_v1.jsonl",
+  "source_sha256": "e76a01938a29638747f52be24d6eab11de52108810933b44adfd275fc9e8e50f"
+}

data/eval/field_workflow_holdout_v1_manifest.json

@@ -0,0 +1,630 @@
+{
+  "category_counts": {
+    "asr_confirmed_text": 12,
+    "disaster_triage": 32,
+    "escalation_precision": 16,
+    "low_resource_constraints": 7,
+    "missing_observation_prioritization": 14,
+    "radio_handoff": 16,
+    "rural_clinic_intake": 36,
+    "sbar_handoff_usefulness": 10,
+    "source_card_discipline": 6,
+    "workflow_repair_seed": 1
+  },
+  "dataset_version": "field_workflow_holdout_v1",
+  "generated_at": "2026-06-09T11:59:43.307305+00:00",
+  "holdout_policy": {
+    "freeze_case_ids": true,
+    "never_copy_close_paraphrases_into_training": true,
+    "never_train_on_this_file": true,
+    "primary_v3_success_surface": true
+  },
+  "output_path": "data/eval/field_workflow_holdout_v1.jsonl",
+  "output_sha256": "e76a01938a29638747f52be24d6eab11de52108810933b44adfd275fc9e8e50f",
+  "row_count": 150,
+  "row_hashes": [
+    {
+      "case_id": "field_workflow_holdout_v1-000000",
+      "sha256": "sha256:1e136fbdbc3819e010cd9b569c17b21652f44be05ee0a7ff9a46eb25b591b883"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000001",
+      "sha256": "sha256:6168208747674b00590988477bcec928d96e8eb6a16d054a3e10bbe1f250d656"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000002",
+      "sha256": "sha256:040fc6fb290528455205aa1c1f2f1ec9d6c1965a66a8d2ba99a6a1963bcee7fe"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000003",
+      "sha256": "sha256:7dfff0baea6c1e448afabbb5c75fc5b964a608f304d2771f4e8dbaeba5378583"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000004",
+      "sha256": "sha256:626ba5ecf75e6e6d10cbcb5c105985e392355e5ce3b9653ba63c57896e52e14c"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000005",
+      "sha256": "sha256:2336a5eaa748e056eb98a3275be67ed934fa7ff03f01b2dabfb37ca606a65c61"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000006",
+      "sha256": "sha256:006e7f9e216f26c7d8948825950cf90fecb4cc941dcf7ebcf4bf211173707125"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000007",
+      "sha256": "sha256:1e44a73f20d00b095dc5612fe25d096f1c5dd8542b9eee89baaf919bc3d4b4ae"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000008",
+      "sha256": "sha256:716e21e11f9c30c6b8ac40a31185586a4667ac2df70b2e0e777816d46d6bddcd"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000009",
+      "sha256": "sha256:282a8667ca02579bf9213f52f79b86f4d61358d6b7686454deaf34a17d436a47"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000010",
+      "sha256": "sha256:bf389eac2b8aacc59c561df81c1dea2fcfeb156847e3af17826c22448bdd88a3"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000011",
+      "sha256": "sha256:46621e1e550154d74d487f48b24c7ddcfee63b226a0cd939effa386858b53aeb"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000012",
+      "sha256": "sha256:d5001fbd9e20d7d2965c2cf6e24c42e1a2e47fc2c2c6fdfca3a9cb3c3bb8d215"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000013",
+      "sha256": "sha256:b97637a87f296d0d8104cfa258081a09478082f4f88b04b36727c7b2c55daa4a"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000014",
+      "sha256": "sha256:e788c848d470cc48cae3ee8325b36821e4f0f8b17caa48c097b4f4bc2434f5d2"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000015",
+      "sha256": "sha256:e921adcaeedb5db898d1b5b30df35c8f0725ce60ad8f16b7db4ac179421f7411"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000016",
+      "sha256": "sha256:94c50384687dea7210e1084b229e443c5d7fa054e14a03a9aa553ab3669ddb7c"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000017",
+      "sha256": "sha256:d4e8c81265aad13d8acf1acf593c5a8d48e0dad119b4c38e2c8b1da4fbb5f777"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000018",
+      "sha256": "sha256:50241b9dcf1d660cd935a63fae4b37bec25166966abc14fd10b2f7e1943a1d27"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000019",
+      "sha256": "sha256:6812e84cc9ef5288c5eb0926e56471919e8c4321070a9277e52180ebce1bc602"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000020",
+      "sha256": "sha256:3ac5ff449918f53eb12a1038bb52a86864c446f231e5c30cd335bc8787df8ae8"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000021",
+      "sha256": "sha256:3450042f9bda28d48fceb73d73422bb3ac86a0d3ada3461d8dd4150449f79df6"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000022",
+      "sha256": "sha256:7ae44dca6abf9f533c0bf401d6e9dc4500513622d1d53af8ed07ad3b421c4232"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000023",
+      "sha256": "sha256:dad0b0dacb7621ceef58de62786d6bd3739196218591ce46cac1323d14690300"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000024",
+      "sha256": "sha256:02ec24f0f4e8543aaff4363ef30a08d8146c5a5a9f62cefe984e981ba51e7660"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000025",
+      "sha256": "sha256:5340b86f4490fadb0bdac3c7d8e6139db5a69ba748a726364b57789190487138"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000026",
+      "sha256": "sha256:64421af0d3f4c9c5bda42a64d13d870c2e1bb5b6ea56c9c7875c15a5773b5986"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000027",
+      "sha256": "sha256:b6fd0fff086ed48b4ae4a6000deecdad9b29ce0eea5738aac14feba2ef89f544"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000028",
+      "sha256": "sha256:a680e40cd97cd46a4700412469f10f96075808727d6bed04147c356e43edfade"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000029",
+      "sha256": "sha256:6bb675f16d9abe1c52e30ceebeff8f44693c617492b82352635d7477d6b3b0e9"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000030",
+      "sha256": "sha256:c0cfe1b03c31e99cc40bacd73c8a7fa9be3139d5bc133b15d3b7ec1d18743052"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000031",
+      "sha256": "sha256:60db6534fefc4c57746b783a4e144037ed801e1856eb17279a7c65998d193298"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000032",
+      "sha256": "sha256:a11024dbc3d4d6187affc169aa1f9c823c83522d98b5c378f8f5935e41de385f"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000033",
+      "sha256": "sha256:aa2463873626e7ce142413e29cf374365dfde500cce8df0e3c3b177646e7cb57"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000034",
+      "sha256": "sha256:508f1a186450be7f130a36bf6616111e9d4f46d09936e102cde0e7eea6d6db38"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000035",
+      "sha256": "sha256:80d4a379f2c033b8230dd53f0095305bfca6216e438a95367a6802f250075e5e"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000036",
+      "sha256": "sha256:fa65f6bb8be1816a01518176f700d1ab008a31fa0a3d095d7568d96b9b2657fb"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000037",
+      "sha256": "sha256:a262fc6d7f893202693bb587a851063b4e1a33bdd7072390e36ed73eaf932a54"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000038",
+      "sha256": "sha256:962ce120c28f03a2f55cd90f5d22f73b017b54579b04d7f0c07505b3b65cac17"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000039",
+      "sha256": "sha256:79b988bf3c102ad7b7d8511190899f6aa51ab6504699aeac57a9b744c30e85f9"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000040",
+      "sha256": "sha256:b64996f697382c7539a9720c4987182aa4aa81592cfcb629b06c4348f0c025a9"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000041",
+      "sha256": "sha256:9eb4f73110fce55a1ce57dd7ee29d836a48e86b6cbd1d11b7764fccca515dbb7"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000042",
+      "sha256": "sha256:80b98befe49d9f1f8a3e638c6c0a5e29a5e8a5e267a6cdca7b42c8b16d23c6e3"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000043",
+      "sha256": "sha256:0c896faf71116ef70f7b732f7062157c395c4df504a1bf7f104b351fc9683ada"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000044",
+      "sha256": "sha256:f6b75367707614f9eaf1a5041f08eb13a55602698b7373840cd44074ebd1049d"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000045",
+      "sha256": "sha256:5fadebe79ef42344d8ae78ddc2b9ed7e354b7b5ad83a844b459532fcd86108b5"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000046",
+      "sha256": "sha256:8e95d6b9193514fa9088db294ba1cfbe6acea6242f8cf325a23edc3b989b32e7"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000047",
+      "sha256": "sha256:786e4cd8bd0d75bb3c12550bb25ad584d89c7fa541180d1f4f5522aa8335d755"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000048",
+      "sha256": "sha256:27b1d6a1a1691d4c5ac6eebda8b3e13be0b65d4b9d3ada6d2341ed3761957f60"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000049",
+      "sha256": "sha256:ce6f2e6e69a40903187b936251e64705103daab9a78ccc0fc05bbd4d5bcb2b13"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000050",
+      "sha256": "sha256:18e52fb9e4f7136ca539dbeb39a79321598ef3c229c161d6a328314e3fe7c865"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000051",
+      "sha256": "sha256:6398f1de769f279bba659e1dec1be9cefb59fa84e109d8d158192a91dc917987"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000052",
+      "sha256": "sha256:e49457452cb92410ad7f495854a7c242535fbf051ad420c2c0c4464d98d39f39"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000053",
+      "sha256": "sha256:d168326d4370702d2f705657523caf169aa5203416c4c1025fa7e85541509d06"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000054",
+      "sha256": "sha256:78a95a4899027d7d575168fc5e6b257d36b5251c0a87c43bb9459cb960ea921d"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000055",
+      "sha256": "sha256:6654b56cadfa2b260d6518139701177cb574e0a51073290e72f1074f87886bf0"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000056",
+      "sha256": "sha256:6fd392cce2331a12be88764fd73b2376089e40e9f79dd9b997125e5ba2d04325"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000057",
+      "sha256": "sha256:2e798f045df58b69eaa0dc22ef1bf63710915b621d3f6ca321cf1605db68ff2a"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000058",
+      "sha256": "sha256:5b3399a2edeb89b1a30fdb53aeddc364935e274c69caf73bd982e546a53a66a2"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000059",
+      "sha256": "sha256:4d8d18c3e9f2479b8e49c3f1d0e625e179ca78968c6272eba3d13950b1e33d67"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000060",
+      "sha256": "sha256:db3ae5f9e041522d64df2200957d65667d999b27c2e6c58ab429455f18080e52"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000061",
+      "sha256": "sha256:8de790b141ef4b1a52760c7cc929511019c47e495ec6d18600739e162caccb37"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000062",
+      "sha256": "sha256:1ed1b927251f3b5889ad1946a57bbe2fd08be93b2d4216a7695e9612ce9fba7f"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000063",
+      "sha256": "sha256:9da8a285cf2259ce1f44a4fba4312cc4147409592a9af9e527342a1ec019908d"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000064",
+      "sha256": "sha256:a2045b99ed8d6586b08adf96ce8cbc07b3d4258bb148cb2b0e58cd2efb78e56a"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000065",
+      "sha256": "sha256:2468c022633b6950e145c7042a0a56cf791cbf36c6e7131b167a107bb8317c19"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000066",
+      "sha256": "sha256:3fb49147ce35bd660b0621432c122c5414a9d6b7db63f57c43ca163321ccefce"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000067",
+      "sha256": "sha256:b64763d718bd99fbc057ce918cc0bd7581ec6abb96e39277202225e71cfb03a4"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000068",
+      "sha256": "sha256:d5a8209a58f1e8c80d9369963df14c1d9f5428821ee89c9677d5669ab2a5eac6"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000069",
+      "sha256": "sha256:7164aa75b0d418abc5756512809befd8c543ecb550608a25e2a9a95b63dea732"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000070",
+      "sha256": "sha256:0336ec699e98573f6fe081fe0f8cbec69f025d2ee2feea698e5e04e9f77089bc"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000071",
+      "sha256": "sha256:06fc25b0f1fb65de6580a011d3bc69b1e791203407b8c3abb391a961de34c989"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000072",
+      "sha256": "sha256:75455266fb4d7601a9b62e5a9f6999d9a5a232842f872ea630729033523e1fd9"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000073",
+      "sha256": "sha256:33b564f061aac132677b613f59316709d2b96d8fd445f39ab29fc22d926ae5d8"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000074",
+      "sha256": "sha256:be16abebb24c9913cbde0f08795e6c37f653b0f68b6bdc074347126cf968b9dd"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000075",
+      "sha256": "sha256:e8c6dc484faf9cfb6f8ea0dbc209f2666cd18dd4c6cbb8efc09f5a38dad60e90"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000076",
+      "sha256": "sha256:012a9381e3f96344c0a03de78a2b20724df55f62e99fb544cf2c7b9711d805b8"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000077",
+      "sha256": "sha256:5d57e52fa543eeb70b1e4740c7267f46248ed2347f4743cb4854fbff95f6c01b"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000078",
+      "sha256": "sha256:0aca59efd79e1d1f5b1b6ded8c72ad7cbdbba2369393937f14b516278ebeb83f"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000079",
+      "sha256": "sha256:e86502c722764b93fb56ed10ca29aba2461a2a6c2d2cd2aeac6c2f2c1327c544"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000080",
+      "sha256": "sha256:53e4bc58f76f2b64e91c93daf6f1f9bd1f00939256f804667dc44d9402c664c8"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000081",
+      "sha256": "sha256:834aa3508a92f1733b254b386d304de278886da241e131cc772852846567e781"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000082",
+      "sha256": "sha256:5f4c132c86d501c81f9517e6369a0d7dd3c999a0c1206dafa64f4931cf425cbc"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000083",
+      "sha256": "sha256:e8fa85bfbac641d44c053309e1b7448abfa7c0629b6812eef8fd73a262a2f17c"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000084",
+      "sha256": "sha256:1524e1741f670418b7c373ad6b5517b56ad0472cdd20e611383973ae19fe1b38"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000085",
+      "sha256": "sha256:679c870d01f28e709800716a53449520f2ce19c837b73b2bb8af70c8ec7f2ea5"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000086",
+      "sha256": "sha256:a9dbbe9fef4c795ef8900069e72a6974ba8203f8e7dec136293d82ebcb8e6ef6"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000087",
+      "sha256": "sha256:257cb263a1f17b5827ac43f1db78ca6ecace1573b9be2cdc01ec696f6e436a17"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000088",
+      "sha256": "sha256:358f7ec51adcd4e5726bde47e845dabbce60a748de02f71d940b3e176b7cd64f"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000089",
+      "sha256": "sha256:c1d67db359755b25fa9e491f33a9f52330a81d0d0ffd0e1c024596236c1d513e"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000090",
+      "sha256": "sha256:1ff0a15fcfd7307b52f10a3c80e5e59a4ce4a14cbf7cf920a9ec2e6588e9086b"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000091",
+      "sha256": "sha256:40c292a31d1c80035ea498dc55a2f15722fc251feaff48463cec104b14e8ea3b"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000092",
+      "sha256": "sha256:cf69d0a6947b6cf7d6353f2ca80bf98cd09cfc86acdf432bf365add69690ba54"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000093",
+      "sha256": "sha256:c20a6f1345235a8709088a758bc253b0692336d84f4a6663775d299faec73ee9"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000094",
+      "sha256": "sha256:cd4e067c13018e16f2f1bfee6bd06fa7193e2a5498a7324e7ca5d0a926740bbb"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000095",
+      "sha256": "sha256:9714ec45d708bc18490ef649de90efa6c76b94fa3a3cfec53942d505749ee88c"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000096",
+      "sha256": "sha256:68912714e6e38800a549dcc7604832fca3aa5f88c6cfb670a5512f7c051629c3"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000097",
+      "sha256": "sha256:c97276d2ae06f8a493676fcc50abf04bbd9624f8af392310974bc70888d49443"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000098",
+      "sha256": "sha256:dc3198fc29923a0e458b6196471c3962720eb26533ec22c5be5a13faddaa768b"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000099",
+      "sha256": "sha256:ecba4a719da6d436e70c1d92c9fb803ede35aade2978a27c261ef69dfeb2739e"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000100",
+      "sha256": "sha256:8cbf64a75a1fde553e9b7044088a8e2719af055344e58faa364c16fe5aad26b7"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000101",
+      "sha256": "sha256:513160e156d9e7cd390b9760865bd6dc6c1a12f730b178d5fdca0e3b12949d20"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000102",
+      "sha256": "sha256:4c99739f18b6a352f9090b8612cb5100c08481ac455802b95de83160657485ff"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000103",
+      "sha256": "sha256:538d3207184ff519b29ca2b44946ab42a64760d858d8d34468f05ee9b9a9c0f5"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000104",
+      "sha256": "sha256:17de031493c60e4e14e5bd331469e27d5d6b9fa2033a9dc92790ff04e0ee5dba"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000105",
+      "sha256": "sha256:e4a13be95628cc92efb1f37d740095d67ac4cacbc7afd7db9d81214ad4596a15"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000106",
+      "sha256": "sha256:d60792758b10ae7068d3e73d4042c74cd0b09f44209ec5948d3ab64eb47951fe"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000107",
+      "sha256": "sha256:2e5d44580e6e3b85cc56f174e2033f1c941d16a6c8135f96929bed95ba0142fd"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000108",
+      "sha256": "sha256:5a79329287a57bc3c629064c2e5fee794fd192b5af3bcb60477e4bae676b3ec9"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000109",
+      "sha256": "sha256:54a374fbe2b1ca7b37529cc060254bd00c58b6055a537085e3e856664e2783b9"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000110",
+      "sha256": "sha256:e38c5565b1e3b4fac587d896544762fe866ef22b6aefa6a51e5173be7fb17c73"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000111",
+      "sha256": "sha256:54c15112354aee9a8bd5326756c44165dd835f9daf447f992d8c9a52d8e4dc84"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000112",
+      "sha256": "sha256:fe42ae0ec1d76690e717dbd6114fc25c7be2719ff06a0645fc0826b2e5105307"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000113",
+      "sha256": "sha256:3552e30bed12c21f173c45e0d4615f31274e3b974720aeb224ff4f1c0bfbcc63"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000114",
+      "sha256": "sha256:683dee5fb3322b705662001b86716c3f20c95fdaaef11729ae1d964ebb427baf"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000115",
+      "sha256": "sha256:bab47bbccf9bcb6c3604ddfb90ec5911c982796a0bb02395d45cff3805d8d188"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000116",
+      "sha256": "sha256:736fc4f850efb1af2636edb6ebe07dc13dcbb8991a6701a86219397bbb589c7c"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000117",
+      "sha256": "sha256:9d4492f84e7bbb17160391afe4b5ed5a0f4bba851a440780300bbc3030a40eb0"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000118",
+      "sha256": "sha256:5ab185fc0aa23b1f8e7aa87656ec5af2f07477b37b5dbc637648d77618e30649"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000119",
+      "sha256": "sha256:fa7a7ec934b8156e8ba6ac489d9e26302d92b4b07a55011fb4b2933dc10b287f"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000120",
+      "sha256": "sha256:10294425a1ddd7e1878aae26e33a4162a5a600b4eeaa31fe1a613cc9b0bc4254"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000121",
+      "sha256": "sha256:fd415e173f79efd62a2a06baa92da39bf55718c66d45f01095e4ed937871d22b"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000122",
+      "sha256": "sha256:8bf0570d275e3e83bf65dabf73cec408528c7d2ff90fc826eff6415c3c3041bd"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000123",
+      "sha256": "sha256:d596497c3e859254a951481372e24d16c6573e42f9fdd150e3001031792f358c"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000124",
+      "sha256": "sha256:dabcf1361056ccc546d3862f5f17dc93931fac7d08362d11b5e47ff7712ce471"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000125",
+      "sha256": "sha256:a15f122162031dc518e922653b02e051cd371ffe6fff383733ad8fd2588c50b8"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000126",
+      "sha256": "sha256:74d0285e052e1f254ed79fe5bd41137c06cf91e0cf3457bbae3bb33dbc33b936"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000127",
+      "sha256": "sha256:27c2cc77f714a87bd099132822778c2ac7229fd379d27239067f23edc29860aa"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000128",
+      "sha256": "sha256:592a8bc17ea8602472f4301c73ecee4a586b6a96843c31a60de77a5bbce0d8e5"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000129",
+      "sha256": "sha256:bbab1049e45b4f5568d9a46bbb24701e9fa5ec2d5b8c081b28c09d0ef7ebd820"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000130",
+      "sha256": "sha256:3cd330d6a4f9346c7425331ce50fdcb2b4031f23a20845a770ef1f99390a73c0"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000131",
+      "sha256": "sha256:c2b150d63b49e26b126cd31d339cb5bd4806de68fea89372baf2310062c802f7"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000132",
+      "sha256": "sha256:89ce9698dc9fca79136d43cb5f2e37e94a50d9972b85d53ebd8b85a90ec33c45"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000133",
+      "sha256": "sha256:4565f2f28a12d2b7b34bca8b5281cae1e5219db965f186c8c90549603fdca8f7"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000134",
+      "sha256": "sha256:53fff36514e7f5f93ac5ac86aa4c4d026cb614af6363a2fb756a4412eb32a171"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000135",
+      "sha256": "sha256:461c458998eb1bd8d514b62dd038f9a6778bfee93d1e16a90e93f2e5f1da1f05"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000136",
+      "sha256": "sha256:14c95a0286610c9ade8b5da3edc417c51f81d8f16fa5d5b46019f460836aa79a"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000137",
+      "sha256": "sha256:a2a1a7ce60824ce5a36e0df8423a0e12ea46d1f0d20a4f81269b9e82e4a421b2"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000138",
+      "sha256": "sha256:8fd3a15888d0ca51262cb4f9763f34ecdd98c146ddf8586624adcd7c3a98a9fa"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000139",
+      "sha256": "sha256:d98a69a184231f8ccba193ec60671cff36c120cfeaa81d6a711d75fa52a5cff8"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000140",
+      "sha256": "sha256:6dedb4339c51c86efaa297ab71739595082186bd68d0a4209c16a1e7d7b46af3"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000141",
+      "sha256": "sha256:5f54344d610b60325506f484603662648e81643188af17a3c11d7914c43f0c2d"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000142",
+      "sha256": "sha256:8f75880089609a4039a095b8d5b8c18f076ac0a2ddd0d97f85499f2baadc2969"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000143",
+      "sha256": "sha256:97ad437844511a1e16896116567b23bef5f41cd206a10c680611c15f64b6a095"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000144",
+      "sha256": "sha256:2218d9cb7b7685217e11a8d9adc258e20b466a43174dd1236c9aba2454f96bec"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000145",
+      "sha256": "sha256:e028e816002b57a6e391977a2a12192223a1c5d6053843447908c0d3e6925dec"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000146",
+      "sha256": "sha256:65915d239cfba35e70e18ebac56d43d7f838009b96f21f0c28b292339d3bac42"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000147",
+      "sha256": "sha256:fb3862de6a406e73b25142cbc00fb883fd2a82b345d1a730fdd3dbb4915869c6"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000148",
+      "sha256": "sha256:0d0d4374ed505e1cc4028a758c72e44b19c04195919d9c1eea6eb5b3053f2266"
+    },
+    {
+      "case_id": "field_workflow_holdout_v1-000149",
+      "sha256": "sha256:74f4995ed7dd79b7cf78ef0f111251b1c272adae6aa3711085c375a376fc45cd"
+    }
+  ],
+  "source_dataset_version": "figment_sft_v3_holdout_source",
+  "source_generator": "scripts/generate_field_workflow_holdout.py",
+  "source_generator_prompt_family": "figment_sft_v3_field_workflow"
+}

docs/figment-build-small-eval-loop-draft.md

@@ -0,0 +1,142 @@
+# The Eval Loop That Made Figment Better
+
+Draft status: rough technical follow-up draft for builders.
+
+The first Figment retrospective was about restraint: keep the model inside a narrow job, make rules deterministic where safety needs determinism, and separate app safety from model competence.
+
+Two days later, I would add a second lesson:
+
+Once restraint is real, the eval loop gets teeth.
+
+Figment improved because the harness got specific enough to make vague progress impossible. A run could no longer hide behind valid JSON. It could no longer borrow credit from deterministic fallback. It had to show which fields came from the raw model, which fields came from focused model repair, which fields came from deterministic patches, and which cases required full fallback.
+
+That made the later training loop uncomfortable in exactly the right way.
+
+## The Dangerous Shortcut Was Counting The App
+
+The easiest number to report is final validation.
+
+For Figment, that number answers a real question: did the app produce a schema-valid, grounded, safety-preserving navigator output? It matters. A user does not care whether a malformed model response had good intentions.
+
+But final validation is not the Build Small question. The Build Small question is whether the small model did useful bounded work.
+
+That is why v5 was so useful and so humbling. It reached `150/150` final validation and `150/150` expected labels on the 150-case holdout. If I had stopped there, I could have told a flattering story. But the configured model was competent on only `2/150` cases. Deterministic patches were doing most of the work.
+
+That run forced the metric split:
+
+- final validation: did the app stay inside the contract?
+- expected labels: did the final output preserve the case-level safety targets?
+- raw model competence: did the configured model produce a competent response before repair?
+- repair success: did a focused model repair fix a bounded field failure?
+- deterministic patches: did code patch over model output?
+- fallback: did the app abandon the model path and use deterministic output?
+
+Those numbers have different meanings. Combining them makes the project look smoother and teaches you less.
+
+## A Short Version History
+
+The later Figment loop looked roughly like this:
+
+| Run | Competence | Raw success | Repair success | Expected labels | Final validation | Fallback | Deterministic patches | Lesson |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |
+| v3 | 107/150 | 93/150 | 14/150 | 0/150 | 148/150 | 2 | 114 | Big jump, but weak handoff behavior |
+| v4 | 109/150 | 109/150 | 0/150 | 149/150 | 148/150 | 2 | 104 | Better raw output, still scaffold-heavy |
+| v5 | 2/150 | 2/150 | 0/150 | 150/150 | 150/150 | 0 | 302 | The app passed; the model did not |
+| v6 | 142/150 | 142/150 | 0/150 | 146/150 | 150/150 | 0 | 21 | Targeted replay and delta rows worked |
+| v7 corrected | 148/150 | 148/150 | 0/150 | 147/150 | 150/150 | 0 | 3 | Failures became inspectable |
+| v10 | 147/150 | 147/150 | 0/150 | 150/150 | 150/150 | 0 | 6 | Narrow misses resisted generic fixes |
+| v14p | 146/150 | 146/150 | 0/150 | 150/150 | 150/150 | 0 | 8 | Raw path plateaued |
+| v14p repair-union | 150/150 | 146/150 | 4/150 | 150/150 | 150/150 | 0 | 0 | Model repair closed the remaining cases |
+
+The v3 expected-label number is not apples-to-apples with the later corrected runs. I include it as an early baseline, not as a clean leaderboard row.
+
+The table is not a straight victory staircase. That is the point.
+
+Some runs looked safer because the app was helping more. Some prompt probes made the model worse. Some data deltas moved the target metric, then plateaued. The useful thing was not that every version improved. The useful thing was that the eval made it possible to tell what kind of change had happened.
+
+## V5 Was The Run That Prevented A Bad Blog Post
+
+V5 is the run I keep coming back to because it would have been easy to misunderstand.
+
+At the app level, v5 looked great: no fallback, all final outputs valid, all expected labels preserved. Under the hood, it was exactly the wrong kind of success. Deterministic patches were compensating for model failures across the output.
+
+That forced a decision. I could make the scaffold stronger and keep reporting final validation, or I could use v5 as evidence that the model-owned target needed more work.
+
+The second option made the rest of the project better.
+
+V6 used a targeted corpus: `1430` new delta rows plus `570` replay rows from earlier versions. It did not throw away the previous gains. It trained against the failure shape that v5 exposed. The result moved competence from `2/150` to `142/150` while keeping fallback at zero.
+
+V7 pushed further with `2800` total rows, including `800` new delta rows and replay from v3 through v6. It reached `148/150` competence. More importantly, it reduced the remaining problem to a small number of cases that could be read, argued with, and turned into concrete next actions.
+
+## The Best Fix Was Sometimes To Fix The Eval
+
+One of those concrete actions was not a training run.
+
+The holdout had a negation problem. Some logic treated phrases like "no chest pain reported" too much like positive chest pain evidence. That kind of bug is subtle because it can make an eval look stricter while making it less true.
+
+I fixed the rule behavior and created a corrected scoring view instead of silently rewriting the frozen holdout. The manifest says exactly what changed: six cases, with original and corrected hashes preserved.
+
+That was a turning point in how I thought about evaluation hygiene. A benchmark is not sacred because it is frozen. It is useful because it is inspectable, stable, and honest. If it is wrong, the answer is not to train a model to satisfy the wrong target. The answer is to create a corrected view with a receipt.
+
+## Prompt Contracts Were Not Free
+
+Another failure class involved required observation ownership, especially around postpartum fever cases. The tempting fix was to push more of the desired behavior into the prompt: make required observation IDs more explicit, make the policy more mandatory, make the model promise harder.
+
+That did not reliably help.
+
+One stricter prompt probe made the corrected holdout result worse. Other prompt variants shifted failure patterns without solving the underlying ownership problem. The lesson was not "prompts do not matter." Figment's earlier gains depended on prompt shape. The lesson was that prompt contracts are capacity tradeoffs. In a narrow 4B setup, extra instructions can compete with the actual task.
+
+The better loop was:
+
+- identify the exact field or case family,
+- decide whether the app, model, or scorer should own it,
+- add targeted data only when the model really should own it,
+- verify against the same holdout,
+- keep raw, repair, patch, and fallback counts separate.
+
+That loop is slower than prompt fiddling. It is also harder to fool.
+
+## What V14p Actually Proves
+
+The strongest current result is v14p repair-union on the corrected 150-case holdout:
+
+- `150/150` competence successes,
+- `150/150` expected-label successes,
+- `150/150` final-validation successes,
+- zero deterministic patches,
+- zero fallback uses,
+- `146/150` raw configured-model successes,
+- `4/150` cases resolved by focused model repair,
+- `1942` fields from raw model output,
+- `8` fields from model repair,
+- no unsupported facts counted in the handoff metrics.
+
+That is a strong result, but it has to be named precisely.
+
+It does not mean the raw model solved every case on the first pass. It does not mean the app is validated for real-world deployment. It does not mean the synthetic holdout is a substitute for field testing with actual users.
+
+It means the local 4B Figment system, with model-owned repair but without deterministic patching or fallback, can complete the corrected field-workflow holdout while preserving the prototype's safety and grounding contract.
+
+That is a narrow claim. I like it because it is narrow enough to be useful.
+
+## Publishing Receipts Changed The Work
+
+This loop also changed how I thought about publishing.
+
+If the only artifact is a demo, the audience has to trust your narrative. If the artifacts are public, the narrative becomes inspectable. For Figment, that meant publishing and verifying model artifacts, dataset cards, dataset configs, viewer schemas, and eval traces across versions.
+
+The Hugging Face dataset configs make the curriculum visible: v6 at `1800/200`, v7 at `2520/280`, then v8-v14p as targeted follow-on corpora with stable 47-column viewer schema. The model repo carries the versioned artifacts. The trace directories preserve how each run behaved.
+
+That matters because the most interesting part of this project is not the final number. It is the audit trail from failure to targeted data to rerun to new failure.
+
+## What I Would Do Next
+
+The next version of this work should not be another blind training run.
+
+I would first make the demo story faster to understand. The competitor scan made that obvious: projects like Dental SOAP are very clear at first glance. Figment has deeper receipts, but the demo has to communicate the Backyard scenario quickly.
+
+I would also add more user-facing evaluation around handoff usefulness. The current holdout measures radio/SBAR behavior, source support, unsupported facts, and validation, but real responders would expose different friction: wording, order, brevity, confidence, and whether the next observations are actually actionable under low-resource constraints.
+
+Finally, I would keep the model/app boundary visible. The best future Figment is not the one that hides all scaffolding and pretends the model is autonomous. It is the one that makes each responsibility legible: human confirmation, deterministic red-flag floors, retrieval, model reasoning within a narrow contract, model repair, validation, and trace.
+
+That is the main thing this eval loop taught me. Small models get better when the system around them is honest enough to make their failures specific.

docs/figment-build-small-lessons-draft.md

@@ -0,0 +1,205 @@
+# Building Figment For Build Small: What I Learned About Making Small Models Useful
+
+Draft status: rough public draft, refreshed June 13, 2026 after the v5-v14p training/eval loop.
+
+I started Figment with a simple idea: what if a responder in a rural clinic, mobile unit, shelter, or disaster site had a protocol binder that could talk back?
+
+Not an AI doctor. Not a system that decides whether someone should go home, receive treatment, or ignore a symptom. The version I wanted to build for the Hugging Face Build Small Hackathon was narrower than that: a protocol navigator that could take messy field intake, surface red flags, retrieve relevant protocol cards, ask for missing observations, and help draft a grounded SBAR handoff.
+
+The Build Small constraint made that idea more interesting. A lot of AI demos become persuasive by making the model bigger or the problem blurrier. Figment had to move the other way. The useful question was not "can a model answer medical questions?" It was "can a small-enough model do bounded, visible work inside a system that refuses to let it improvise?"
+
+After the first few days, I thought the lesson was restraint. After the next two days of evals, failed prompts, corrected scoring, H100 runs, and public artifact publishing, the lesson got sharper:
+
+Restraint is not just a safety pattern. It is an iteration engine.
+
+Once the model's job is small enough to inspect, every failure can become a decision. Sometimes the decision is "train on this." Sometimes it is "fix the harness." Sometimes it is "the benchmark is wrong." Sometimes it is "the app already knows this deterministically, so stop asking the model to invent it."
+
+That changed how I understand small-model product work.
+
+## Audio Should Draft, Not Decide
+
+One of the first design decisions that stuck was that audio intake should be a draft layer, not a decision layer.
+
+In the field, speech is natural. A responder may not have the time, lighting, or hand freedom to fill out a perfect form. It is tempting to treat audio as magic: record the note, send it to a model, and let the app continue.
+
+Figment does not do that.
+
+Audio-derived text is treated as provisional. It can suggest fields like age, symptoms, vitals, allergies, medications, supplies, and free-text notes. But a human has to confirm or edit the intake before deterministic red-flag rules or navigator output run. Unconfirmed audio is not allowed to trigger final red flags, clear red flags, or drive the handoff.
+
+That may sound like a small UX detail, but it is one of the most load-bearing safety choices in the app. ASR errors are not rare edge cases. A dropped negation or malformed field can change the meaning of a case. The safer product shape is not "voice in, answer out." It is "voice in, editable draft, confirmed facts, then navigation."
+
+The same lesson applied to the demo. I originally had audio upload and demo clips working, but the better primary workflow was live audio ingest, with upload as a backup. That made the demo closer to the actual setting Figment is meant for: a responder speaking into the tool, then correcting the draft before using it.
+
+## Deterministic Safety Rules Are The Floor
+
+The second thing I learned is that deterministic safety logic should not be treated as an embarrassing fallback. In Figment, it is the floor.
+
+The app has deterministic red-flag rules for things like pediatric dehydration, respiratory distress, pregnancy danger signs, stroke signs, wound infection cues, and other prototype protocol-card categories. If those rules fire, the model cannot lower the urgency. The model can add useful structure around the case, but it does not get to reinterpret away the safety floor.
+
+That sounds obvious when written down. In practice, it changes how you evaluate the model. A safe final output does not necessarily mean the model performed well. It may mean the deterministic layer caught the case, retrieval supplied the relevant cards, validators rejected unsafe output, and fallback kept the app inside the contract.
+
+For a while, that made the project feel less impressive. Then I realized it made the project more honest.
+
+A medical-adjacent prototype should not try to prove that a model is safe by letting it be dangerous and hoping it behaves. It should make the model's job small enough that success and failure are both visible. Figment's rules, protocol cards, validators, traces, and fallback paths are not there because I do not believe in small models. They are there because I want to know exactly where the model helped and exactly where it did not.
+
+## App Safety And Model Competence Are Different Numbers
+
+This became the most important evaluation lesson of the project.
+
+Early on, it would have been easy to report only final validation. The app could often produce a valid final navigator output because deterministic fallback was strong. But that would have hidden the real question for Build Small: was the model actually doing load-bearing work?
+
+So I split the metrics.
+
+In the first 50-case hosted Omni eval, final validation passed `50/50`, but hosted model competence was only `28/50`. That distinction mattered. The app stayed inside its safety envelope, but the model was not carrying all of the work. Some cases needed deterministic fallback after hosted output failed validation or grounding checks.
+
+After adding a more constrained prompt contract, field-level provenance, and focused repair, the hosted follow-up improved. Whole-output competence moved to `31/50`, full deterministic fallback dropped to `8/50`, and the field-level metric showed `480/650` model-retained fields, with `170/650` deterministic patches.
+
+That was the moment the eval started to feel honest. Instead of saying "the model passed" or "the app passed," I could say something more precise:
+
+The application produced safe final outputs on the eval. The hosted model carried many bounded fields. Deterministic logic patched the rest. Full fallback still existed, and it was counted separately.
+
+That distinction became even more important later. One local run looked perfect if I only counted final validation: v5 reached `150/150` final validation and `150/150` expected labels on the 150-case holdout. But the configured model was only competent on `2/150` cases, and deterministic patches were doing the heavy lifting.
+
+That was not a victory lap. It was a smoke alarm.
+
+If your app has fallback, validators, retrieval, and deterministic rules, do not collapse everything into one success number. A model-competence score and an app-safety score answer different questions.
+
+## Field-Level Provenance Changed My Relationship With Fallback
+
+The first version of Figment treated model output mostly as all-or-nothing JSON. If one important field failed validation, the app could fall back to deterministic output. That was safe, but it also threw away useful model work.
+
+The better pattern was field-level provenance.
+
+Instead of asking "did the whole model response pass?", Figment started asking:
+
+- Which fields came from the raw model?
+- Which fields were repaired by a focused model call?
+- Which fields were deterministically patched?
+- Which cases required full fallback?
+
+That changed the project. A model might select the right protocol pathway, ask useful missing-observation questions, and draft a reasonable checklist, while still failing one SBAR grounding rule. Field-level provenance lets the app keep the validated parts and patch the failed parts without pretending the whole output was model-generated.
+
+It also makes the trace more useful. The Trace tab is not just a debugging feature; it is the project's honesty surface. It shows input, rules, retrieval, prompt context, model output, validation, repair, fallback, and provenance. For a hackathon project, that might seem like a lot of plumbing. For a small-model project, it became the main way to show that the model was doing bounded work rather than being credited for deterministic scaffolding.
+
+Fallback is not one thing. There is a big difference between:
+
+- the model succeeded raw,
+- the model succeeded after focused repair,
+- the model contributed some fields,
+- the model failed and deterministic fallback produced the result.
+
+Those distinctions matter if you want to make credible claims about small models.
+
+## Fine-Tuning Only Helped After The Eval Got Honest
+
+The local 4B path was where the project got the most interesting and the most humbling.
+
+The target was `nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16`, served through a llama.cpp-compatible route after LoRA fine-tuning and GGUF conversion. The goal was not to make a general medical assistant. The goal was to teach a small local model the narrow Figment behavior: protocol-card discipline, red-flag preservation, missing-observation planning, safe handoff drafting, and schema-valid navigator JSON.
+
+The first fine-tuning pilot was valuable because it proved the full loop: generate teacher data, train on Modal, merge the adapter, convert to GGUF, serve locally, and run the eval harness without cloud inference. But the result was not a clean win. The pilot made the model better at shape and field retention, but it regressed competence to `11/50` on the locked 50-case eval.
+
+That failure was useful. It showed that training loss and JSON validity were not enough. The dataset had taught format more than judgment. It had too few examples for some failure modes. Some rows were not aligned tightly enough to the real harness. And the eval was punishing behaviors that looked safe in prose but violated the exact scorer or product contract.
+
+The v2 dataset was a better answer. It used a stronger teacher model to generate synthetic, validated rows aligned to the actual Figment prompt and repair tasks. It kept locked eval cases out of training. It added more repair rows and failure-class coverage. The v2 local model improved to `33/50` on the locked 50-case eval, with `50/50` final validation.
+
+Then v3 changed the question again.
+
+Rather than only optimizing the locked 50-case eval, I created a 150-case field-workflow holdout. That holdout asked whether Figment helped the real workflow: rural clinic intake, disaster triage, ASR-like confirmed text, low-resource constraints, radio handoff, SBAR usefulness, and source-card discipline.
+
+On that holdout, v3 reached `107/150` competence, with `93/150` raw local-model successes, `14/150` focused repair successes, `2/150` full fallbacks, and `148/150` final validation.
+
+That sounded good, and in many ways it was. But the failure distribution mattered more than the headline score. The handoff layer was weak: radio handoff and SBAR usefulness were exactly where the model still needed help.
+
+That is where the next two days of work changed the project.
+
+## Sometimes The Benchmark Is Wrong
+
+The most valuable bug I found was not in the model. It was in the scoring and deterministic rule path.
+
+The old holdout treated some negated phrases too bluntly. A sentence like "no chest pain reported" could still trigger a chest-pain-related signal because the matcher saw the words and missed the negation. That is the kind of bug that can make a benchmark look tougher while actually making it less faithful.
+
+I did not mutate the original frozen holdout. Instead, I created a corrected scoring view with a manifest. It changed exactly six cases and kept the original and corrected hashes visible. That mattered because the point of an eval is trust. If the target moves, readers should be able to see how and why.
+
+This became a new rule for the project: do not train your way around a bad benchmark. Fix the benchmark, leave a receipt, and rerun the model.
+
+## Sometimes Prompting Harder Makes Things Worse
+
+I also tried the obvious prompt fixes.
+
+Some failures involved missing required observation ownership. So I tested stricter prompt contracts that made required observation IDs more explicit and more mandatory. In theory, that should have helped. In practice, one mandatory-observation prompt probe made the run worse.
+
+That was a useful embarrassment.
+
+It reminded me that a prompt is not a magic policy layer. If the model is already near the edge of a narrow behavior, adding more contract language can crowd the task, make it overfit the wrong cue, or shift attention away from the actual field workflow. Some failures needed better data. Some needed a clearer scaffold. Some needed the app to stop asking the model for things the app already knew. "Prompt harder" was not a general solution.
+
+## V5 Through V14p Became A Curriculum Loop
+
+The later local models were less like one big training run and more like an eval-driven curriculum.
+
+V5 proved that the harness could keep the app safe even when the model was not carrying the work. It reached `150/150` final validation, but only `2/150` model competence. That result forced the right question: what would it take for the configured model, not the scaffolding, to own the fields?
+
+V6 was the first big answer. Instead of regenerating everything from scratch, I built a corpus with targeted deltas plus replay rows from earlier versions. It reached `142/150` competence, `150/150` final validation, zero fallback, and far fewer deterministic patches.
+
+V7 improved again: `148/150` competence, zero fallback, and only a handful of deterministic patches. On the corrected scoring view, it still had real misses, especially around postpartum-fever observation ownership and field-specific required-observation behavior. But now the failures were small enough to inspect case by case.
+
+The v8-v14p loop kept narrowing those misses. The data was not generic "more medical examples." It was targeted rows for multi-rule observation ownership, postpartum-fever required observations, source/support cards, visible-field closure, and focused repair behavior. Some versions improved the exact metric. Some did not. A few looked nearly identical. That was frustrating, but it was also evidence that the eval had become specific enough to resist hand-wavy progress stories.
+
+The strongest current run is v14p repair-union: `150/150` competence, `150/150` expected labels, and `150/150` final validation on the corrected 150-case holdout, with zero deterministic patches and zero fallback. The nuance matters: raw configured-model success is still `146/150`; four cases are resolved by focused model repair, and eight fields are marked as model-repaired rather than model-raw.
+
+That is a much better result than the early local runs, but it is not the same claim as "the raw model passed everything." It proves something narrower and more useful:
+
+The local 4B Figment system can complete this corrected field-workflow eval with model-owned output and model repair, without deterministic patching or full fallback, while preserving red flags, source discipline, and handoff constraints.
+
+That is the kind of claim I can actually defend.
+
+## The Product Surface Had To Catch Up
+
+Another thing I learned: evidence is not enough if the product surface does not make the evidence legible.
+
+Figment started as a fairly functional Gradio app. It had the pieces: intake, rules, retrieval, navigator output, trace. But it felt more like a harness than a field tool.
+
+The later UI work moved it to a custom Gradio Server surface with a "Field Kit Workbench" feel. The important part was not just that it looked better. The important part was that the user workflow became clearer without changing the model harness contract. The named API endpoints, intake/risk/retrieval/navigator/trace data shape, demo-case loader, and eval harness stayed stable.
+
+That taught me a product lesson I wish I had internalized earlier. You can make a prototype more delightful without hiding the machinery that makes it trustworthy. The right UI did not bury the trace; it made the workflow easier to understand so the trace could matter more.
+
+## Public Receipts Matter
+
+The hackathon also changed how I think about artifact publishing.
+
+It is one thing to say "I trained a local model." It is another thing to publish model artifacts, dataset cards, configs, eval traces, and schema-stable dataset viewers that let someone inspect the path. By the end of the later loop, the Hugging Face repos had public artifacts and dataset configs for v5 through v14p, with the v8-v14p corpora published and verified.
+
+That matters for a small-model project because the interesting claim is rarely just the final score. The claim is the path: which rows were added, which cases were excluded, which eval was frozen, which scoring view was corrected, which artifacts were served, and which fallback paths were counted separately.
+
+The competitor scan made that even clearer. Some Build Small projects had very legible demos. Dental SOAP, for example, is a strong direct comparison: guided intake, small Qwen model roles, deterministic safety sentinel, and printable handoff. ScrubData is technically strong in another direction. Figment's edge is not that it is the simplest demo. Its edge is the depth of the evidence trail: model versions, datasets, traces, failure accounting, and an app surface that shows how the answer was made.
+
+That is also the risk. A deep evidence trail only helps if judges and users can understand it quickly. The demo still has to make the Backyard story obvious: here is the messy field intake, here are the red flags, here is what the small local model contributed, here is what the app refused to let it decide, and here is the handoff you can use.
+
+## What I Would Tell Another Build Small Team
+
+If I were giving advice to someone building with a small model in a high-stakes-ish workflow, I would say:
+
+Start with the boundary, not the model. Decide what the model is allowed to own, what the app owns deterministically, and what a human must confirm.
+
+Measure app safety and model competence separately. If the final app output passes because a scaffold saved it, count that as scaffold success, not model success.
+
+Make provenance a product feature. Users and judges should be able to see which fields came from the model, repair, rules, retrieval, or fallback.
+
+Let failures become curriculum, but only after checking whether the eval is fair. Some misses deserve training rows. Some deserve harness fixes. Some deserve a corrected benchmark.
+
+Do not assume a stricter prompt is a better contract. Verify it with the same eval, and be willing to throw it away.
+
+Publish the receipts. Scores are more credible when the artifacts, datasets, traces, and manifests exist outside your laptop.
+
+## The Lesson I Am Taking From Build Small
+
+Before this project, I would have described small-model product work mostly in terms of parameter count, latency, hardware, and model quality. Those still matter. But Figment made me think about "small" differently.
+
+Small is also a design discipline.
+
+It means narrowing the model's job until it can be checked. It means using deterministic rules where determinism is safer. It means making retrieval explicit. It means refusing to count fallback as model competence. It means keeping traces detailed enough that a judge, user, or future builder can see what happened. It means letting the model contribute where language and prioritization matter, while keeping safety-critical floors outside the model's control.
+
+The first version of this post ended with "make the next eval sharper." After v5 through v14p, I would say it a little differently:
+
+Make the next failure smaller, clearer, and harder to hide.
+
+That is the thing Figment taught me. Useful small-model apps are not small because they ask less ambitious questions. They are small because they are honest about where the model belongs, and because that honesty gives you a way to improve.

docs/figment-build-small-posts-index.md

@@ -0,0 +1,43 @@
+# Figment Build Small Blog Drafts
+
+Draft status: working index for the Build Small writing set.
+
+## 1. Building Figment For Build Small: What I Learned About Making Small Models Useful
+
+File: `docs/figment-build-small-lessons-draft.md`
+
+Audience: judges, builders, and public readers who want the high-level story.
+
+Core argument: useful small-model apps are systems of restraint, and that restraint turns failures into an iteration engine.
+
+Best current evidence to preserve in revisions:
+
+- hosted Omni split between app safety and model competence,
+- local v3 field-workflow holdout jump to `107/150`,
+- v5 false comfort: `150/150` final validation but only `2/150` model competence,
+- corrected scoring view with six changed cases,
+- v14p repair-union: `150/150` competence/final validation/expected labels, zero deterministic patches, zero fallback, with `146/150` raw success and `4/150` model-repair cases,
+- Field Kit Workbench UI as a clearer product surface without changing the harness contract,
+- public Hugging Face artifacts and trace receipts.
+
+## 2. The Eval Loop That Made Figment Better
+
+File: `docs/figment-build-small-eval-loop-draft.md`
+
+Audience: builders who want the technical learning loop.
+
+Core argument: final app validation is not model competence; the later Figment loop worked because raw success, repair success, deterministic patches, and fallback were measured separately.
+
+Best current evidence to preserve in revisions:
+
+- v5 as the run that prevented a misleading success story,
+- v6/v7 targeted replay-and-delta curriculum,
+- corrected benchmark hygiene instead of training around scorer bugs,
+- prompt-policy probes that made results worse,
+- v14p repair-union as a narrow but defensible system claim.
+
+## Likely Follow-Up Angles
+
+- A demo-focused post or submission section comparing Figment's evidence depth against clearer Backyard demos.
+- A short "what the trace shows" post with screenshots or annotated examples.
+- A public artifact guide that points readers to the model repo, dataset configs, and selected eval traces.

docs/local_4b_finetuning_plan.md

@@ -0,0 +1,550 @@
+# Local 4B Fine-Tuning Plan
+
+Date: 2026-06-08
+
+This note captures the fine-tuning strategy I would use after the prompting and scaffolding fixes in `docs/local_4b_prompting_scaffolding_fixes.md`. The goal is a small, local, full-weight Figment navigator that is more load-bearing on the 50-case eval without weakening deterministic safety gates.
+
+## Training Hypothesis
+
+The local 4B model is probably big enough for this task if the task is framed as bounded protocol navigation instead of open-ended clinical reasoning.
+
+The evidence points toward trainable rubric-following gaps:
+
+- Urgency floors are already reliable: `min_urgency_met` passed 50/50.
+- After the prompting/scaffolding pass, expected-label success improved from 2/50 to 13/50 and local competence improved from 18/50 to 26/50.
+- The biggest remaining miss is still exact required-observation cue coverage: 35/50 failures.
+- Forbidden behavior is nearly solved but not fully solved: 1/50 still included a forbidden medication mention.
+- Negated red-flag handling is a trainable false-positive problem: all 7 red-flag mismatches are unexpected red flags on negated/safety-boundary cases.
+- Source-card, candidate-pathway, target-card selection, and SBAR failures are still format and grounding behaviors that SFT can teach.
+- The model is less load-bearing than the final validation score suggests: every record needed at least one deterministic scaffold patch, `red_flags` needed deterministic patching in 45/50 records, and `missing_info_to_collect` / `next_observations_to_collect` each needed deterministic patching in 37/50 records.
+
+I would not jump to a larger model unless Figment must rely on raw model output without scaffolding, field repair, or deterministic fallback. That is not the current architecture.
+
+## Locked Test Set
+
+Do not train on the current 50-case eval.
+
+Keep this as the locked regression test:
+
+- `data/eval/initial_handwritten_cases.jsonl`
+- `data/eval/adversarial_strict_cases.jsonl`
+- `data/eval/comprehensive_hosted_cases.jsonl`
+- Pre-scaffold baseline: `traces/local_4b_evidence_20260607T231248Z/`
+- Current post-scaffold baseline: `traces/local_4b_evidence_20260608T015209Z/`
+
+Use those cases only for checkpoint comparison and final evidence.
+
+## Failure Analysis From The Current Trace
+
+Current trace: `traces/local_4b_evidence_20260608T015209Z/`.
+
+Headline comparison against `traces/local_4b_evidence_20260607T231248Z/`:
+
+- `competence_successes`: 18 -> 26.
+- `repair_successes`: 5 -> 26.
+- `fallback_uses`: 9 -> 6.
+- `expected_label_successes`: 2 -> 13.
+- `expected_label_failures`: 48 -> 37.
+- `final_validation_successes`: 50 -> 50.
+- `raw_configured_model_successes`: 13 -> 0, because the stricter post-scaffold scorer only counts raw success when no deterministic scaffold field is patched.
+
+Failure breakdown in the current trace:
+
+- `missing_observation_cues_present`: 35 failures.
+- `expected_source_cards_present`: 11 failures.
+- `expected_candidate_pathways_present`: 8 failures.
+- `target_card_in_candidate_pathways`: 8 failures.
+- `red_flags_match`: 7 failures.
+- `target_card_in_source_cards`: 2 failures.
+- `forbidden_behavior_absent`: 1 failure.
+- `min_urgency_met`: 0 failures.
+
+The failures are not evenly distributed. The finetune should target these clusters:
+
+- Required-observation cue lexicalization. The model often includes a nearby observation but misses the evaluator-recognized cue. The most common missing cues are `symptom trend`, `complete vital signs`, `confirmed intake status`, `deterministic rule results`, `retrieved protocol card IDs`, `hydration status`, `source card IDs`, `source protocol card IDs`, `time since last urine`, and `fluid retention`.
+- Negation and safety-boundary target selection. The 7 red-flag mismatches are all unexpected red flags where the expected rule set is empty: negated chest pain, AMS, respiratory distress, stroke, pediatric dehydration, wound infection, or fever-like cases should land on `SAFETY-BOUNDARIES-v1` / `REFERRAL-SBAR-v1`, not the condition red-flag pathway.
+- Source-card omissions. The model still drops `REFERRAL-SBAR-v1` and/or `SAFETY-BOUNDARIES-v1` in pregnancy, AMS, respiratory-negated, pediatric, fever-infant, wound, and negated-safety cases.
+- SBAR target-card behavior. SBAR-focused cases sometimes include `REFERRAL-SBAR-v1` in source cards but fail to put it in `candidate_protocol_pathways` as the target pathway.
+- Canned fallback cases. Six cases still needed full canned fallback: respiratory gasping, safety-boundary injection, wound red streaking, pediatric lethargy, pregnancy SBAR handoff, and safety-ignore-cards.
+
+Interpretation:
+
+- This is a good SFT target, not a pure model-size blocker. The model can generally preserve schema, urgency floors, and safety phrasing, but it needs repeated examples of exact target-field coverage.
+- The training objective should not reward prettier prose. It should reward exact field membership, exact observation-cue coverage, and correct abstention from red flags when facts are negated.
+- Continue to measure final validation separately from model competence. Final validation is protected by deterministic scaffolding; the finetune should reduce deterministic patches and fallbacks.
+
+## Teacher Model For Synthetic Data
+
+Do not rely on handmaking the training data. Use a significantly larger teacher model to generate and critique synthetic SFT rows:
+
+- Teacher model: `nvidia/nemotron-3-ultra-550b-a55b`.
+- Use the same OpenAI-compatible hosted route Figment already uses for the hosted model.
+- Endpoint config: use `OMNI_ENDPOINT_URL` or `HF_ENDPOINT_URL` if that is the active hosted endpoint, otherwise use `NVIDIA_BASE_URL` with the default `https://integrate.api.nvidia.com/v1`.
+- API key: use the existing `NVIDIA_API_KEY` secret. Never write the key into datasets, logs, model cards, manifests, or Modal artifacts.
+- Keep teacher selection separate from app runtime selection. Use a dedicated generation variable such as `SFT_TEACHER_MODEL_ID=nvidia/nemotron-3-ultra-550b-a55b`; do not overwrite the production `NVIDIA_MODEL_ID` just to generate data.
+
+The teacher model is only for data generation, repair suggestions, and critique. The target artifact remains the full-weight local 4B route: `nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16` plus a Figment adapter merged back into BF16 weights.
+
+The teacher is not trusted blindly. Every accepted training row must pass deterministic schema validation, product-contract validation, safety validation, card-id validation, cue-coverage validation, and a rubric check before it is admitted to `data/finetune/figment_sft_v1.jsonl`.
+
+## NVIDIA Training-Data Priors
+
+Use NVIDIA's published Nemotron datasets as recipe priors for the Figment SFT set. The Hugging Face dataset pages list `nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16` among models trained or fine-tuned on these datasets, so the plan should match the kind of supervision the base model already knows.
+
+Authenticated Hugging Face CLI probe on 2026-06-08:
+
+- `hf datasets info nvidia/Nemotron-Post-Training-Dataset-v2` showed a gated dataset with parquet shards for `chat`, `code`, `math`, and multilingual splits.
+- `hf datasets sql` over the Post-Training `chat` parquet showed row columns `uuid`, `license`, `generator`, `version`, `category`, `reasoning`, and `messages`.
+- Downloading the small Post-Training multilingual shard showed the same columns plus a JSON `metadata` field with values such as `sub_category`, `dataset_name`, `source_file`, and `lang_id`.
+- `hf download nvidia/Nemotron-RL-Agentic-Conversational-Tool-Use-Pivot-v1 train.jsonl` produced rows with top-level keys `trajectory_id`, `responses_create_params`, `expected_action`, `scenario`, `num_unique_actions`, `meta_info`, `qwen_235b_info`, `agent_ref`, `pass_rate`, `pass_rate_total`, and `pass_rate_passed`. In the first 1000 rows, `expected_action` used `type`, `content`, `name`, and `arguments`.
+- `hf datasets list nvidia/Nemotron-CC-v2 -R` and authenticated README/LICENSE downloads exposed the CC-v2 file layout and source recipe, but `hf download ... Diverse-QA/part_000000.parquet --dry-run` returned access denied because the repo still requires approval for data-file access in this token. Until that approval clears, treat CC-v2 as README/file-layout-informed rather than row-schema-confirmed.
+
+Dataset-specific lessons:
+
+- `nvidia/Nemotron-Post-Training-Dataset-v2`: SFT/RL-style post-training data with public/open or synthetic prompts, synthetic responses from larger public/open models, quality and complexity filtering, and multiple response modes. Figment should copy the shape: synthetic prompts, teacher-generated gold, explicit validation, and mode discipline. The target mode is "final navigator JSON only"; any teacher reasoning or critique stays in metadata and never becomes assistant output.
+- `nvidia/Nemotron-RL-Agentic-Conversational-Tool-Use-Pivot-v1`: structured conversational tool-use trajectories where each assistant step is treated as a behavior-cloning problem with an `expected_action`, pass-rate fields, and reward metadata. Figment should copy this more than generic chat SFT: each row should have verifiable expected actions for card selection, red-flag membership, required observations, SBAR fields, and refusal boundaries.
+- `nvidia/Nemotron-CC-v2`: broad pretraining data with high-value math/code preservation, synthetic rephrasing, multilingual QA, filtering, and global deduplication. Figment should copy the data hygiene, not the raw corpus: diverse paraphrases, hard examples, dedupe hashes, source metadata, and synthetic-only/de-identified healthcare scenarios.
+
+Do not ingest NVIDIA dataset rows directly into the Figment fine-tune unless license and redistribution terms are reviewed for this project. Use them to set the data-generation recipe:
+
+- Generate synthetic public-style prompts and synthetic domain cases instead of handmaking one-off examples.
+- Use multiple generation prompts and temperatures with the Ultra teacher to create diversity, then filter heavily.
+- Reject easy cases that do not exercise a real failure mode, just as the post-training dataset filtered easy-to-guess or low-quality prompts.
+- Store structured row fields inspired by Post-Training: `uuid`, `license`, `generator`, `version`, `category`, `reasoning`, `messages`, and `metadata`.
+- Store structured metadata inspired by the RL dataset: `expected_action`, `reward_components`, `pass_rate_total`, `pass_rate_passed`, `teacher_model_id`, `critic_model_id`, `generation_prompt_id`, `dedupe_hash`, and `recipe_sources`.
+- Globally deduplicate by normalized intake text, protocol-card set, expected target card, and embedding similarity so the synthetic set does not become 1500 near-copies of the locked eval.
+- Keep legal and ethical metadata explicit: source is synthetic, no PHI, no copied clinical transcript, no NVIDIA row copied, license review status recorded.
+- Reject assistant outputs containing `<think>`, hidden reasoning tags, or visible teacher critique. The local model should learn the final navigator artifact, not the teacher's reasoning trace.
+
+## Dataset
+
+Create `data/finetune/figment_sft_v1.jsonl` with 500 to 1500 synthetic sibling cases generated by the Ultra teacher from existing protocol cards, synthetic case specs, and failure-class templates.
+
+Recommended distribution:
+
+- 40% required-observation exactness and cue lexicalization.
+- 20% negation, denied symptoms, routine near-miss cases, and safety-boundary target selection.
+- 15% source-card, target-card, candidate-pathway, and citation repair cases.
+- 15% grounded SBAR slot filling, including SBAR-as-target-pathway cases.
+- 5% forbidden clinical instruction avoidance.
+- 5% repair-and-fallback-rescue cases based on the six current canned-fallback failure shapes.
+
+Each example should use the exact production prompt shape, including:
+
+- confirmed structured intake,
+- deterministic red flags,
+- urgency floor,
+- retrieved protocol cards,
+- allowed facts inventory,
+- required observation targets,
+- fact ledger,
+- required JSON skeleton.
+
+The target output should be ideal navigator JSON, not a cleaned-up local-model sample. Generate the gold JSON with the Ultra teacher, then accept it only after deterministic validation and, where useful, a second teacher critique pass.
+
+Generation pipeline:
+
+1. Generate a synthetic case spec from protocol cards and a failure class such as `missing_observation_cues`, `negated_red_flag`, `sbar_target_pathway`, or `source_card_coverage`.
+2. Avoid copying locked eval cases or near-paraphrasing their free-text intake. The locked eval can define failure classes, not training examples.
+3. Run the same deterministic retrieval, red-flag rules, urgency floors, required-observation extraction, fact ledger construction, and JSON skeleton construction used by production.
+4. Ask `nvidia/nemotron-3-ultra-550b-a55b` to produce concise, observation-only semantic notes over a bounded JSON contract. The accepted SFT row still pairs the final assistant label with the exact production prompt shape.
+5. Assemble the navigator JSON from the teacher-authored notes plus deterministic fixed fields for urgency floors, red flags, source cards, and candidate pathway ids, then validate the output with Figment's deterministic validators and eval-label scorer.
+6. Reject, repair, or regenerate rows that miss required cues, cite unavailable cards, add unsupported red flags, lower urgency below the floor, leak unsafe clinical instruction, or omit SBAR grounding.
+7. Optionally run a second Ultra pass as a critic that checks field membership against the rubric. Do not use the critic's approval as a replacement for deterministic validators.
+8. Generate 4 to 8 candidate outputs for ordinary cases and 16 to 32 candidate outputs for high-risk failure classes such as negated red flags, forbidden behavior, and fallback-rescue cases. Score candidates with reward components, keep the best passing row, and store the candidate pass rate. This mirrors NVIDIA's agentic dataset pattern without requiring a full RL environment for the first run.
+9. Deduplicate globally by case text, required facts, expected target card, and embedding similarity.
+10. Write only accepted rows, with metadata for `teacher_model_id`, `teacher_label_mode`, endpoint variable names, prompt hash, protocol card ids, failure class, validation result, pass-rate metadata, dedupe hash, recipe-source links, and generation timestamp. Do not store secrets.
+
+Current implementation note, 2026-06-08:
+
+- `scripts/generate_finetune_data.py` uses streamed teacher calls to `nvidia/nemotron-3-ultra-550b-a55b` with `reasoning_effort="none"`.
+- Non-streaming full-output teacher calls were not reliable for this prompt family; they produced long waits/hangs. The working route asks the teacher for concise semantic notes and rejects malformed/missing note payloads.
+- The stream call runs in a child process with a parent-enforced timeout, so wedged teacher requests become `teacher_backend_error` rejections instead of blocking the dataset run.
+- The first live artifact is a 50-row validated seed set at `data/finetune/figment_sft_v1.jsonl` with manifest `data/finetune/figment_sft_v1_manifest.json`; it is `dry_run=false`, generated by `nvidia/nemotron-3-ultra-550b-a55b`, and can be extended with `--resume`.
+- The seed set is aligned to the actual local 4B harness: each row uses the same single user-message prompt shape that `ModelClient(... model_backend="llama_cpp")` sends to `/v1/chat/completions`, and each prompt is built with plain `search_protocol_cards(query_from_intake(...), limit=6)` retrieval rather than teacher-only forced cards.
+- The seed set now covers both local-4B chat-completion tasks in the harness:
+  - `navigator_full`: the primary production navigator prompt sent by `ModelClient.generate_json(prompt, context)`.
+  - `focused_repair`: the field-repair prompts produced by `build_focused_repair_prompts(...)` when deterministic validation fails.
+- `scripts/augment_finetune_repair_rows.py` adds repair-task rows from teacher-gold navigator outputs by corrupting a previous output in the same ways the harness sees, rebuilding the exact focused repair prompt, and using only the relevant teacher-gold fields as the assistant target.
+- `scripts/verify_finetune_harness_alignment.py` verifies this contract by rebuilding every navigator prompt from `figment.prompt_builder.build_prompt`, rebuilding every focused repair prompt from `build_focused_repair_prompts`, validating navigator assistant JSON against the same retrieved cards, checking expected-label success, and rejecting teacher-facing artifacts such as `teacher_note` sources or teacher-specific pathway reasons.
+- Audio intake is not currently a separate 4B chat-completion task in this harness. The local audio path is Parakeet/provider payload plus deterministic draft-field extraction, with the 4B route recorded as the field-fill model id but not called through `ModelClient.generate_json` for audio drafting.
+
+Create a cue alias table before generating examples. The teacher can propose aliases, but deterministic code should canonicalize and filter them. Each required observation target should have:
+
+- canonical cue text from the protocol card,
+- 3 to 6 allowed natural-language variants,
+- one short responder-facing phrase that should appear in `missing_info_to_collect` or `next_observations_to_collect`,
+- a negative example where the cue is absent or contradicted.
+
+For the first SFT dataset, oversample examples whose gold outputs cover every required observation cue in both the structured observation fields and the SBAR handoff. Then add ablations where the same intake should not trigger a condition pathway because the symptom is denied or historical.
+
+## Data Split
+
+Use a deterministic split by case id:
+
+- 80% train.
+- 10% validation during training.
+- 10% synthetic holdout.
+
+Also keep the current 50-case eval as a separate locked test set. It should never be mixed into train or validation.
+
+## Gold Output Rules
+
+Gold outputs must preserve Figment's product contract:
+
+- `protocol_urgency` never below deterministic floor.
+- `red_flags` only from deterministic fired rules or confirmed present facts.
+- When deterministic red flags are empty, `red_flags` must be empty and the target pathway should usually be `SAFETY-BOUNDARIES-v1` or `REFERRAL-SBAR-v1`, not the condition card for the denied symptom.
+- `source_cards` must cite every fired rule card and every candidate pathway card.
+- `candidate_protocol_pathways` may only use allowed/retrieved card ids.
+- In SBAR-focused tasks, `REFERRAL-SBAR-v1` must be present in both `source_cards` and `candidate_protocol_pathways`.
+- `missing_info_to_collect` and `next_observations_to_collect` must cover required observation target ids.
+- Observation coverage must use evaluator-recognizable cue wording, especially for `symptom trend`, `complete vital signs`, `confirmed intake status`, `deterministic rule results`, `retrieved protocol card IDs`, and `source card IDs`.
+- SBAR must be grounded in confirmed intake, deterministic rules, and allowed slot sources.
+- No diagnosis, prescribing, dosing, discharge, autonomous routing, or unsafe treatment instructions.
+- Audio-derived facts must only appear when accepted or edited by the responder.
+
+## Training Format
+
+Use supervised fine-tuning where each row contains:
+
+```json
+{
+  "case_id": "figment-sft-v1-000123",
+  "uuid": "figment-sft-v1-000123",
+  "license": "synthetic internal training data",
+  "generator": "nvidia/nemotron-3-ultra-550b-a55b",
+  "version": "figment_sft_v1",
+  "category": "missing_observation_cues",
+  "reasoning": "off",
+  "messages": [
+    {"role": "system", "content": "Figment system prompt..."},
+    {"role": "user", "content": "CONTEXT JSON..."},
+    {"role": "assistant", "content": "{...ideal navigator JSON...}"}
+  ],
+  "tags": ["missing_observations", "negation"],
+  "metadata": {
+    "teacher_model_id": "nvidia/nemotron-3-ultra-550b-a55b",
+    "critic_model_id": "nvidia/nemotron-3-ultra-550b-a55b",
+    "teacher_base_url_env": "NVIDIA_BASE_URL",
+    "teacher_api_key_env": "NVIDIA_API_KEY",
+    "failure_class": "missing_observation_cues",
+    "expected_action": {
+      "target_card": "SAFETY-BOUNDARIES-v1",
+      "required_observation_cues": ["complete vital signs", "symptom trend"]
+    },
+    "reward_components": {
+      "schema_valid": 1,
+      "source_cards_present": 1,
+      "required_observation_cues_present": 1,
+      "red_flags_match": 1,
+      "forbidden_behavior_absent": 1
+    },
+    "pass_rate_total": 8,
+    "pass_rate_passed": 6,
+    "dedupe_hash": "sha256:...",
+    "recipe_sources": [
+      "nvidia/Nemotron-Post-Training-Dataset-v2",
+      "nvidia/Nemotron-RL-Agentic-Conversational-Tool-Use-Pivot-v1",
+      "nvidia/Nemotron-CC-v2"
+    ],
+    "validator_passed": true,
+    "license_review": "synthetic_figment_row_no_nvidia_rows_copied"
+  }
+}
+```
+
+If the training stack expects a single `text` column, serialize the same chat template into one string and keep `case_id`, `tags`, and non-secret teacher metadata as metadata.
+
+Disable sequence packing for the first run. Exact prompt-to-output boundaries matter more than throughput for this task.
+
+## Modal Job Shape
+
+Use Modal for data generation and the training run because it gives code-defined images, GPU selection, secrets, and persistent volumes. Data generation is API-bound and can run on CPU; training needs GPU.
+
+Reference docs:
+
+- Modal guide: https://modal.com/docs/guide
+- GPUs: https://modal.com/docs/guide/gpu
+- Volumes: https://modal.com/docs/guide/volumes
+- Secrets: https://modal.com/docs/guide/secrets
+- Unsloth fine-tuning example: https://modal.com/docs/examples/unsloth_finetune
+- LLM fine-tuning example: https://modal.com/docs/examples/llm-finetuning
+
+Create a generation script such as `modal/generate_figment_sft_data.py` or `scripts/generate_finetune_data.py` that calls the same hosted endpoint shape as `figment/model_client.py`:
+
+```python
+import os
+import modal
+
+app = modal.App("figment-sft-data-generation")
+
+image = (
+    modal.Image.debian_slim(python_version="3.11")
+    .uv_pip_install("openai", "pydantic")
+)
+
+data_volume = modal.Volume.from_name("figment-sft-data", create_if_missing=True)
+
+
+@app.function(
+    image=image,
+    volumes={"/data": data_volume},
+    secrets=[modal.Secret.from_name("nvidia-api")],
+    timeout=6 * 60 * 60,
+)
+def generate(config: dict) -> dict:
+    teacher_model_id = os.environ.get(
+        "SFT_TEACHER_MODEL_ID",
+        "nvidia/nemotron-3-ultra-550b-a55b",
+    )
+    base_url = (
+        os.environ.get("OMNI_ENDPOINT_URL")
+        or os.environ.get("HF_ENDPOINT_URL")
+        or os.environ.get("NVIDIA_BASE_URL")
+        or "https://integrate.api.nvidia.com/v1"
+    )
+    api_key = os.environ["NVIDIA_API_KEY"]
+    ...
+```
+
+The generation job should record `teacher_model_id` and the endpoint environment variable name that was used, but it should never record `api_key` or the resolved secret value.
+
+The generation job should also write a manifest inspired by NVIDIA's dataset cards:
+
+- row counts by failure class,
+- candidate counts and pass rates,
+- rejection counts by validator,
+- dedupe counts,
+- teacher and critic model ids,
+- source recipe links,
+- license/PHI assertions,
+- prompt-template hashes.
+
+Create a new script such as `modal/finetune_figment_nemotron.py`:
+
+```python
+import modal
+
+app = modal.App("figment-nemotron-lora")
+
+image = (
+    modal.Image.debian_slim(python_version="3.11")
+    .uv_pip_install(
+        "accelerate",
+        "datasets",
+        "peft",
+        "sentencepiece",
+        "torch",
+        "transformers",
+        "trl",
+        "wandb",
+    )
+)
+
+model_cache = modal.Volume.from_name("figment-model-cache", create_if_missing=True)
+data_volume = modal.Volume.from_name("figment-sft-data", create_if_missing=True)
+checkpoint_volume = modal.Volume.from_name("figment-checkpoints", create_if_missing=True)
+
+
+@app.function(
+    image=image,
+    gpu="L40S",
+    volumes={
+        "/model_cache": model_cache,
+        "/data": data_volume,
+        "/checkpoints": checkpoint_volume,
+    },
+    secrets=[
+        modal.Secret.from_name("huggingface-token"),
+        modal.Secret.from_name("wandb-secret"),
+    ],
+    timeout=12 * 60 * 60,
+)
+def train(config: dict) -> dict:
+    ...
+```
+
+Start with `gpu="L40S"` for LoRA if 12k to 16k context fits with gradient checkpointing and batch size 1. Move to `gpu="H100"` or `gpu="A100-80GB"` if the long-context run is memory-bound. Do not make the final local model quantized just because training used memory-saving tricks.
+
+Suggested setup commands:
+
+```bash
+modal volume create figment-model-cache
+modal volume create figment-sft-data
+modal volume create figment-checkpoints
+modal secret create nvidia-api NVIDIA_API_KEY=... NVIDIA_BASE_URL=https://integrate.api.nvidia.com/v1 SFT_TEACHER_MODEL_ID=nvidia/nemotron-3-ultra-550b-a55b
+modal secret create huggingface-token HF_TOKEN=...
+modal secret create wandb-secret WANDB_API_KEY=...
+modal run modal/generate_figment_sft_data.py --dataset-version figment_sft_v1
+modal run modal/finetune_figment_nemotron.py --dataset-version figment_sft_v1
+```
+
+If the hosted model is currently using `OMNI_ENDPOINT_URL` or `HF_ENDPOINT_URL` instead of `NVIDIA_BASE_URL`, put that same endpoint variable in the `nvidia-api` Modal secret as well. The point is to reuse the hosted route, not to create a second provider configuration for the teacher.
+
+## Base Model And Adapter
+
+Base model:
+
+- `nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16`
+- Use the full-weight BF16 base.
+- Train a LoRA adapter.
+- Merge the adapter back into BF16 weights for the final local route.
+- Convert the merged BF16 weights to BF16 GGUF for `llama.cpp`.
+
+Training parameters for the first run:
+
+- LoRA rank: 16.
+- LoRA alpha: 32.
+- LoRA dropout: 0.05.
+- Learning rate: `1e-4`.
+- Warmup ratio: 0.05.
+- Epochs: 2 to 3.
+- Effective batch size: 8 to 16 via gradient accumulation.
+- Per-device batch size: 1.
+- Max sequence length: 16384 if memory allows. The post-scaffold prompts reached roughly 12k to 14.5k tokens before completion, so a 12000-token first run would teach the wrong truncated task.
+- Packing: false.
+- Precision: BF16.
+- Gradient checkpointing: true.
+- Save every 50 steps.
+- Evaluate every 25 to 50 steps.
+- Early stop on validation loss plus task metrics, not loss alone.
+
+For target modules, start with PEFT all-linear targeting if supported by the installed stack. If not, inspect the base model's `named_modules()` and include attention and MLP projection linears such as `q_proj`, `k_proj`, `v_proj`, `o_proj`, `gate_proj`, `up_proj`, and `down_proj`, plus any Nemotron hybrid projection modules exposed as linear layers. Do not guess silently; log the matched trainable modules in the Modal run artifact.
+
+If 16k context is memory-bound on `L40S`, prefer moving to `A100-80GB` or `H100` over silently truncating examples. If a smaller pilot is needed, build a separate short-context ablation dataset and label it as such; do not compare it directly to the locked 50-case eval.
+
+Use oversampling before introducing more complex objectives:
+
+- 3x oversample required-observation cue examples.
+- 3x oversample negated-red-flag false-positive examples.
+- 2x oversample SBAR target-pathway and source-card omission examples.
+- Keep forbidden-behavior examples present but do not let them dominate, since the current failure rate is 1/50.
+
+If SFT reduces loss but red-flag false positives persist, add a second small preference-tuning pass using paired outputs: one output that incorrectly fires the condition red flag from a negated fact, and one output that stays on the safety-boundary pathway with empty `red_flags`.
+
+If SFT improves field imitation but still leaves tool-like decisions brittle, add a small RLVR-style or preference-tuning stage after SFT. Keep it simple on Modal first:
+
+- Build paired outputs from the same synthetic cases.
+- Score each pair with deterministic reward components.
+- Train with DPO/ORPO in TRL if it is enough.
+- Move to a NeMo Gym-style environment only if simple preference tuning cannot reduce deterministic patch counts.
+
+The reward should be verifiable and product-shaped, not subjective: exact card ids, exact required observation cues, empty red flags when negated, no forbidden instruction, and SBAR groundedness.
+
+## Training Metrics
+
+During each checkpoint, run a lightweight held-out scorer that measures:
+
+- exact JSON parse rate,
+- required schema pass rate,
+- required-observation target coverage,
+- required-observation cue lexical coverage,
+- source-card coverage,
+- candidate pathway coverage,
+- target-card-in-source and target-card-in-candidate rates,
+- negation/red-flag match,
+- unexpected-red-flag rate on negated cases,
+- deterministic patch counts by field,
+- full canned fallback count,
+- model-visible-fields-retained,
+- forbidden clinical language rate,
+- SBAR grounding violations.
+
+After promising checkpoints, run the full local 50-case eval against the checkpoint through the same `llama.cpp` route used for evidence.
+
+## Acceptance Targets
+
+Use the 2026-06-08 post-scaffold local 4B trace as the baseline:
+
+- Expected-label successes: improve from 13/50 to at least 35/50.
+- Missing-observation failures: reduce from 35 to 5 or fewer.
+- Unexpected red flags on negated cases: reduce from 7 to 0.
+- Source-card failures: reduce from 11 to 2 or fewer.
+- Candidate-pathway target failures: reduce from 8 to 2 or fewer.
+- Forbidden-behavior failures: reduce from 1 to 0.
+- Full canned fallback uses: reduce from 6 to 2 or fewer.
+- Deterministic scaffold patches: reduce from 206 patched fields to 80 or fewer, with `red_flags` deterministic patches below 10 and observation-field deterministic patches below 10 each.
+- Model-visible-fields-retained: improve from 0.683 to at least 0.85.
+- Raw configured-model successes under the strict no-patch definition: improve from 0/50 to at least 20/50.
+- Final validation: remain 50/50.
+- No-cloud local proof: remain true.
+
+If the model improves loss but fails those task metrics, discard the checkpoint. Figment needs protocol-rubric behavior, not prettier prose.
+
+## Modal Pilot Run
+
+Status as of 2026-06-08:
+
+- Modal app: `figment-nemotron-4b-lora`.
+- Dataset staged in Modal volume `figment-sft-data` at `/data/figment_sft_v1`.
+- Dataset split: 99 train rows, 11 validation rows.
+- Training image: PyTorch 2.7.1 CUDA 12.6 devel base, PEFT LoRA, Transformers, `mamba-ssm==2.2.6.post3`, and `causal-conv1d==1.6.2.post1`.
+- Smoke run: `initial-smoke`, 1 step at 2048 context, passed and saved adapter.
+- Full-context smoke: `16k-context-smoke`, 1 step at 16384 context, passed and saved adapter.
+- Pilot run: Modal run `ap-rocN1bpDhAMYPBLVE9Dfrg`, adapter path `/checkpoints/figment_sft_v1/pilot-20260608`.
+- Pilot config: full BF16 base `nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16`, LoRA rank 16, alpha 32, dropout 0.05, learning rate `1e-4`, max sequence length 16384, gradient accumulation 8, 40 optimizer steps.
+- Pilot result: 99 train rows and 11 validation rows tokenized; train runtime 1297.0099 seconds; train loss 3.4215212553739547; checkpoint artifacts include `adapter_model.safetensors`, `adapter_config.json`, tokenizer files, and `figment_training_manifest.json`.
+- Local manifest copy: `data/finetune/modal/figment_sft_v1/pilot-20260608-training_manifest.json`.
+- Local adapter copy: `artifacts/modal_checkpoints/pilot-20260608/`.
+- Modal merge run: `ap-D2ySCj6r9jRo8zQwUGBc6b`, saved merged BF16 Hugging Face weights to `/checkpoints/figment_sft_v1/pilot-20260608-merged-bf16`.
+- Local merged BF16 copy: `artifacts/modal_checkpoints/pilot-20260608-merged-bf16/`.
+- Local BF16 GGUF: `artifacts/modal_checkpoints/pilot-20260608-merged-bf16.gguf`, SHA-256 `85d92bc721a6f6d04c8f656ea3d32ff7c2714eef500f6cd0b8227e53268fc6d2`.
+- GGUF conversion used `tools/llama.cpp/convert_hf_to_gguf.py`; the local `NemotronHModel` converter needed a repo-local patch so dense 4B configs are detected from raw `config.json` rather than the `AutoConfig` MoE defaults.
+- Local server route was proved through `llama-server` on `http://127.0.0.1:8001/v1`, advertising `nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16` with `n_params=3973556832`, `n_ctx=16384`, and `n_ctx_train=1048576`.
+
+Pilot eval evidence:
+
+- Fine-tuned trace: `traces/local_4b_finetuned_evidence_20260608T151555Z/`.
+- Baseline trace: `traces/local_4b_evidence_20260608T015209Z/`.
+- Final validation remained 50/50 and no-cloud local route proof remained true.
+- Raw configured-model successes improved from 0/50 to 10/50.
+- Full canned fallback uses improved from 6 to 2.
+- Deterministic scaffold patches improved from 206 fields to 104 fields.
+- Model-visible fields retained improved from 0.683 to 0.84.
+- Expected-label successes did not improve: 13/50 before, 13/50 after.
+- Competence successes regressed from 26/50 to 11/50 because the previous focused-repair path collapsed from 26 repair successes to 1.
+- The main remaining patched fields were `missing_info_to_collect` and `next_observations_to_collect`, 36 deterministic patches each.
+- The pilot also produced a wound-case timeout and a follow-on HTTP 500 from `llama-server`; the eval recovered through canned fallback, but this is evidence that stop discipline and runtime token caps still need work.
+
+This pilot checkpoint is not accepted as the final local model. It is useful because it proves the Modal train, merge, BF16 GGUF conversion, local serve, and no-cloud eval loop, and because it identifies the next training target: preserve the raw JSON/schema gains while restoring focused-repair behavior and improving expected-label cue coverage.
+
+Next training iteration:
+
+- Increase focused-repair rows substantially. The adapter must learn the exact `build_focused_repair_prompts(...)` task, including returning only the requested field subset and preserving valid existing fields.
+- Add hard negative and ablation examples where the model must not drift from repair into full navigator output.
+- Oversample accepted rows for `missing_info_to_collect` and `next_observations_to_collect` until every required observation target is expressed in evaluator-recognizable language.
+- Add explicit wound, source-card, and target-card coverage rows because the local timeout happened in the wound cluster and source/candidate coverage remains a task metric.
+- Add length-control examples and reject any training output with hidden reasoning tags, analysis prose, repeated JSON, or completion text outside the requested JSON object.
+- Add an eval-time runtime guard for the local route, such as a lower `max_tokens` for primary and repair calls plus stop sequences where supported, so a single runaway generation cannot consume the full 240-second timeout.
+- Keep the locked 50-case eval out of training data. Use these results only to define failure classes and acceptance metrics.
+
+## Artifact Flow
+
+1. Train LoRA adapter on Modal.
+2. Save adapter, training config, data manifest, metrics, and module-match log to the checkpoint volume.
+3. Pull the chosen adapter locally.
+4. Merge adapter into the full BF16 base weights.
+5. Save merged Hugging Face weights with a model card.
+6. Convert merged safetensors to BF16 GGUF with the same `llama.cpp` conversion path used for the current local model.
+7. Start local `llama-server` with the merged BF16 GGUF.
+8. Rerun:
+
+```bash
+PYTHON_DOTENV_DISABLED=true FIGMENT_MODEL_TIMEOUT_SECONDS=180 .venv/bin/python scripts/run_local_4b_evidence.py --base-url http://127.0.0.1:8001/v1 --timeout-seconds 180 --force-eval
+```
+
+9. Update evidence docs only if the new run passes the gates. The `pilot-20260608` run does not pass the task-quality gates even though it passes final validation and no-cloud route proof.
+
+## What Not To Do
+
+- Do not train on the locked 50-case eval.
+- Do not copy rows from NVIDIA's published datasets into Figment SFT without a deliberate license and redistribution review.
+- Do not handwave teacher output into the dataset without deterministic validation and rejection.
+- Do not store `NVIDIA_API_KEY` or any resolved hosted endpoint secret in training artifacts.
+- Do not change the production hosted model config to Ultra 550B just to generate SFT data; use a dedicated teacher variable.
+- Do not replace deterministic safety gates with model judgment.
+- Do not use final validation success as model competence.
+- Do not count deterministic fallback or deterministic target fills as model output.
+- Do not publish a Well-Tuned claim until the adapter is trained, published or otherwise evidence-linked, loaded by the app, and verified by eval traces.
+- Do not describe the final local route as quantized if the artifact is BF16/full-weight.

docs/local_4b_prompting_scaffolding_fixes.md

@@ -0,0 +1,191 @@
+# Local 4B Prompting And Scaffolding Fixes
+
+Date: 2026-06-07
+
+This note captures the prompt and controller changes I would make before reaching for a larger model. The goal is to make `nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16` more load-bearing on the local `llama.cpp` route while keeping Figment's deterministic validators and safety boundaries intact.
+
+## Evidence Snapshot
+
+Primary trace:
+
+- `traces/local_4b_evidence_20260607T231248Z/`
+- `traces/local_4b_evidence_20260607T231248Z/local_4b_eval.jsonl`
+- `traces/local_4b_evidence_20260607T231248Z/eval_summary.json`
+
+Observed local 4B behavior:
+
+- 50/50 final validation successes.
+- 18/50 competence successes.
+- 13/50 raw configured-model successes.
+- 5/50 repair successes.
+- 9/50 full deterministic fallbacks.
+- 499/650 visible fields retained from model output, or 76.8%.
+- Expected-label full success was 2/50.
+
+The failure pattern looks fixable with better scaffolding and fine-tuning. The model is not generally missing urgency: `min_urgency_met` passed 50/50. It is mostly missing exact required-observation cues, candidate pathway/source-card rubric details, negation discipline, and grounded SBAR phrasing.
+
+Expected-label failure counts:
+
+- `missing_observation_cues_present`: 47 failures.
+- `target_card_in_candidate_pathways` / `expected_candidate_pathways_present`: 15 failures.
+- `expected_source_cards_present`: 12 failures.
+- `red_flags_match`: 7 failures.
+- `forbidden_behavior_absent`: 4 failures.
+- `target_card_in_source_cards`: 1 failure.
+- `min_urgency_met`: 0 failures.
+
+## Current Anchors
+
+Current prompt scaffolding already exists but is too advisory:
+
+- `figment/prompt_builder.py` builds `allowed_facts_inventory`, `required_observations_inventory`, `routine_or_negated_case_guidance`, and a required JSON skeleton.
+- `figment/validators.py` enforces source-card constraints, urgency floors, missing-observation grounding, SBAR grounding, and forbidden clinical language.
+- `figment/focused_repair.py` already has a `missing_observations` repair scope, but the repair prompt still asks the model to infer the exact observation language.
+- `figment/eval_metrics.py` scores expected labels separately from safety validation, which is the right separation.
+
+## Fix 1: Promote Required Observations From Context To Targets
+
+Problem: `required_observations_inventory` is present, but the 4B model treats it like optional supporting context.
+
+Change:
+
+- Add a compact `required_observation_targets` payload to the prompt context.
+- Give each target a stable id, card id, normalized cue tokens, and display text.
+- Tell the model that `missing_info_to_collect` and `next_observations_to_collect` must include at least one target cue for every cited non-exempt card that has required observations.
+- After model output, deterministically patch missing target cues into those two fields before falling back.
+- Trace each patch as `deterministic_required_observation_fill`, not model competence.
+
+The key distinction is that the LLM can phrase the responder-facing sentence, but the app owns the checklist target. This should attack the largest failure class directly without weakening the validator.
+
+Done when:
+
+- `missing_observation_cues_present` improves from 3/50 passing to at least 45/50 passing.
+- The trace shows whether each required-observation cue was model-written, repaired, or deterministically filled.
+- Whole-output competence and field-retention metrics do not count deterministic fills as raw model success.
+
+## Fix 2: Pre-Fill Non-Creative Control Fields
+
+Problem: the model is being asked to regenerate facts the deterministic system already knows.
+
+Change:
+
+- Pre-fill `protocol_urgency` from the deterministic urgency floor.
+- Pre-fill fired `red_flags` from deterministic rule results.
+- Pre-fill mandatory `source_cards` from fired rule card ids plus the retrieved/selected card ids.
+- Pre-fill candidate pathway options from retrieval before asking the model to write reasons.
+- Ask the model to write bounded text for `reason_relevant`, `responder_checklist`, `do_not_do`, SBAR slots, plain language, and uncertainty handling.
+
+This narrows the 4B model's job from "reconstruct the whole navigation state" to "explain and operationalize already-bounded state." That is a better match for a small model.
+
+Done when:
+
+- Source-card and candidate-pathway expected-label failures drop sharply.
+- Fallbacks caused by sparse or malformed source-card fields become rare.
+- The model is still visibly load-bearing on prose, checklists, and pathway reasons.
+
+## Fix 3: Add A Negation Ledger
+
+Problem: routine or denied-symptom cases still sometimes inherit nearby emergency-card language.
+
+Change:
+
+- Add a `case_fact_ledger` to the prompt with three explicit buckets: `present`, `absent_or_denied`, and `unclear`.
+- Add `must_not_fire_rule_ids` when a rule's trigger terms are absent or denied.
+- Put the ledger before the protocol cards in the prompt.
+- Require any red flag to cite a `present` fact or deterministic rule result.
+- For SBAR and checklist text, forbid copying high-risk card language unless supported by `present` facts or deterministic red flags.
+
+Done when:
+
+- `red_flags_match` failures drop from 7/50 to 1/50 or less.
+- Routine negated cases stay routine when the deterministic urgency floor is routine.
+
+## Fix 4: Make SBAR A Filled Template
+
+Problem: SBAR failures are not mainly creative-writing failures. They are slot-grounding failures.
+
+Change:
+
+- Build an SBAR draft template with fixed slots:
+  - `situation`: chief concern plus setting, if confirmed.
+  - `background`: confirmed age, pregnancy status, and relevant context only.
+  - `assessment_observations_only`: confirmed symptoms, vitals, and deterministic red flags only.
+  - `handoff_request`: local protocol, supervisor, clinician, or emergency pathway request only.
+- Ask the model to lightly rewrite the template, not invent SBAR content.
+- Keep unsupported high-risk facts out of the template before the model sees it.
+
+Done when:
+
+- SBAR grounding failures do not force full fallback when the rest of the model output is usable.
+- Unsupported high-risk SBAR terms such as pregnancy, oxygen, or pressure only appear when present in confirmed intake, deterministic rules, or card text that is allowed for that slot.
+
+## Fix 5: Separate Observations From Clinical Actions
+
+Problem: the expected-label scorer and validators need sharper language around terms that can be either safe observations or unsafe instructions.
+
+Change:
+
+- Split observation cues from intervention cues.
+- Treat `oxygen saturation`, `SpO2`, and `room-air saturation` as observation language.
+- Treat `administer oxygen`, `start oxygen`, oxygen-flow settings, dosing, and medication instructions as intervention language.
+- Update expected-label forbidden checks so observation requests do not get penalized as unsafe oxygen instructions.
+- Keep forbidden clinical action patterns strict.
+
+Done when:
+
+- The model can ask for oxygen saturation as missing information without being pushed toward oxygen administration language.
+- `forbidden_behavior_absent` failures are true safety failures, not observation/action ambiguity.
+
+## Fix 6: Make Focused Repair Deterministic-Target-Aware
+
+Problem: the current focused repair scope for missing observations asks the model to repair only the two observation arrays, but it does not force exact target coverage.
+
+Change:
+
+- For the `missing_observations` repair scope, include only:
+  - the failed card ids,
+  - the missing target cue ids,
+  - the allowed display text,
+  - the previous two observation arrays.
+- Require the repair output to return exactly `missing_info_to_collect` and `next_observations_to_collect`.
+- Reject repairs that omit target cue ids.
+- If repair still omits a cue, patch deterministically rather than asking for another broad repair.
+
+Done when:
+
+- Missing-observation repairs become short, low-latency, and predictable.
+- Repaired observation fields count as `model_repaired`; deterministic fills count separately.
+
+## Fix 7: Add A Structured Output Contract For Target Coverage
+
+Problem: natural-language arrays are hard to audit for exact expected-label coverage.
+
+Change:
+
+- Add an internal-only field during model generation, such as `selected_required_observation_ids`.
+- Strip it from the user-facing navigator output after validation.
+- Validate that every cited required-observation card has at least one selected observation id.
+- Use the selected ids to prove why a natural-language observation sentence satisfies the target.
+
+Done when:
+
+- Expected-label scoring can distinguish "model selected the right cue but phrased it differently" from "model missed the cue."
+- The visible output remains clean while trace evidence becomes more exact.
+
+## Suggested Implementation Order
+
+1. Add `required_observation_targets` and `case_fact_ledger` in `figment/prompt_builder.py`.
+2. Add deterministic target-fill helpers shared by navigator finalization and tests.
+3. Update focused repair for the `missing_observations` scope to use target ids.
+4. Pre-fill or lock control fields before the LLM call.
+5. Convert SBAR generation to a slot template plus bounded rewrite.
+6. Split observation/action forbidden-language scoring for oxygen-like terms.
+7. Rerun `scripts/run_local_4b_evidence.py` and compare against the 2026-06-07 trace.
+
+## Non-Negotiables
+
+- Do not weaken deterministic red-flag rules.
+- Do not loosen validators to inflate model competence.
+- Do not count deterministic target fills as raw model output.
+- Do not claim Parakeet ASR proof from typed transcripts or local artifact presence.
+- Keep the final local artifact full-weight/BF16 for the local route; no quantized local model claim.

docs/local_4b_v2_training_data_plan.md

@@ -0,0 +1,229 @@
+# Local 4B V2 Training Data Plan
+
+Date: 2026-06-08
+
+This note turns the `pilot-20260608` failure analysis into the next training-data target. The goal is `figment_sft_v2`: a larger, cleaner, harness-aligned dataset for the full-weight local `nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16` route.
+
+## Verdict
+
+Use a combination of additional data and training-technique changes.
+
+More data is necessary, but more of the current `figment_sft_v1` recipe is not enough. The next round should fix data quality and rubric alignment first, then train with task-balanced sampling or a small curriculum so the adapter is optimized against Figment's actual eval metrics, not only JSON validity or loss.
+
+Do not jump to a bigger local model yet. The pilot proved that the 4B can learn structure and retain more fields; the failures point mostly to underspecified or contradictory supervision plus a repair-trigger mismatch.
+
+## Evidence Sources
+
+- Fine-tuned pilot trace: `traces/local_4b_finetuned_evidence_20260608T151555Z/`.
+- Post-scaffold baseline trace: `traces/local_4b_evidence_20260608T015209Z/`.
+- Current training set: `data/finetune/figment_sft_v1.jsonl`.
+- Current harness-alignment verifier: `scripts/verify_finetune_harness_alignment.py`.
+- Current generation script: `scripts/generate_finetune_data.py`.
+- Current repair augmentation script: `scripts/augment_finetune_repair_rows.py`.
+
+## What Improved
+
+The pilot adapter learned output shape and field retention:
+
+- Raw configured-model success improved from `0/50` to `10/50`.
+- Full canned fallback uses improved from `6` to `2`.
+- Deterministic scaffold patches dropped from `206` fields to `104` fields.
+- Model-visible fields retained improved from `0.683` to `0.84`.
+- Final validation remained `50/50`.
+- No-cloud local route proof remained true.
+
+## What Failed
+
+The pilot did not learn the task rubric:
+
+- Expected-label success stayed flat at `13/50`.
+- Competence regressed from `26/50` to `11/50`.
+- Repair successes collapsed from `26` to `1`.
+- Missing-observation cue failures stayed high at `34/50`.
+- Red-flag mismatches stayed at `7`.
+- Candidate target failures worsened from `8` to `11`.
+- Forbidden-behavior failures worsened from `1` to `6`.
+
+The main remaining deterministic patches were:
+
+- `missing_info_to_collect`: `38` deterministic fallback fields.
+- `next_observations_to_collect`: `38` deterministic fallback fields.
+
+## Root Causes
+
+### 1. The dataset taught schema more than rubric
+
+`figment_sft_v1` has only `50` full navigator rows and `60` focused-repair rows. That was enough to make the model emit more valid-looking JSON, but not enough to teach exact card selection, negation behavior, observation-cue coverage, or eval-safe lexical choices.
+
+### 2. The negation data is too small and partly contradictory
+
+There are only `8` `negation_safety_boundary` rows. Several of those rows still contain red flags in the assistant target even though the category should teach the opposite behavior. This can teach the model to include `SAFETY-BOUNDARIES-v1` while keeping condition red flags around, which is exactly what appears in the eval failures.
+
+For `figment_sft_v2`, category invariants should reject any `negation_safety_boundary` row unless:
+
+- `red_flags` is empty,
+- `protocol_urgency` is not raised by a denied symptom,
+- `candidate_protocol_pathways` targets `SAFETY-BOUNDARIES-v1` or `REFERRAL-SBAR-v1`,
+- condition cards appear only when justified as source context, not as active red flags.
+
+### 3. The dataset includes eval-forbidden lexical patterns
+
+The locked expected-label scorer flags tokens such as `medication`. In `figment_sft_v1`, many assistant targets include strings such as "Do not prescribe, dose, administer, or start medication." That is semantically safe, but it trains the exact lexical pattern the scorer later punishes.
+
+For `figment_sft_v2`, either the evaluator should distinguish prohibited advice from safe boundary language, or the dataset should avoid the scorer's forbidden lexical tokens entirely. If the eval remains locked, prefer scorer-safe phrasing such as:
+
+- "Do not give treatment instructions."
+- "Do not provide dosing or clinical orders."
+- "Do not tell the responder to start, stop, or administer anything."
+
+Avoid `medication`, `prescribe`, `dose`, `antibiotics`, and similar tripwire terms in assistant outputs unless the scorer is fixed first.
+
+### 4. Repair mostly stopped being invoked
+
+After fine-tuning, the model often emits schema-valid JSON. That means the harness accepts raw output and marks weak fields as deterministic patches, rather than invoking focused repair. The repair path currently runs after strict validation failures, not after expected-label/rubric failures or scaffold patches.
+
+This means data alone will not fully recover repair behavior. The next round should combine better data with at least one harness/objective change:
+
+- Option A: train the full navigator output to satisfy rubric checks directly, reducing the need for repair.
+- Option B: add a rubric-repair phase that can repair expected-label misses and scaffold-patched fields, not only strict validation failures.
+- Option C: both, which is the recommended route.
+
+### 5. SBAR handoff grounding is undertrained
+
+All actual repair attempts in the pilot were SBAR/handoff grounding failures. There are only `10` `focused_repair:handoff_note_sbar` rows in `v1`.
+
+`figment_sft_v2` needs many more SBAR repair rows, especially examples where the model must remove unsupported high-risk facts from `handoff_note_sbar` without changing valid fields.
+
+### 6. Runtime length control needs training and serving guards
+
+The pilot produced a wound-case timeout and a follow-on HTTP 500 from `llama-server`. This should be addressed two ways:
+
+- Dataset: reject outputs with repeated JSON, hidden reasoning tags, analysis prose, or excessive list growth.
+- Runtime: use lower `max_tokens` for primary and repair calls, plus stop sequences where supported.
+
+## V2 Dataset Target
+
+Create `data/finetune/figment_sft_v2.jsonl` with `1000` to `1500` accepted rows after validation and rejection.
+
+Use `nvidia/nemotron-3-ultra-550b-a55b` as the teacher through the same hosted OpenAI-compatible endpoint and `NVIDIA_API_KEY` secret. Do not store secrets or resolved endpoint credentials in data, manifests, traces, or model cards.
+
+Recommended accepted-row mix:
+
+- `400` to `500` full navigator rows focused on exact required-observation cue coverage.
+- `250` to `300` negation and safety-boundary rows.
+- `200` to `250` source-card, candidate-target, and SBAR-as-target rows.
+- `150` to `200` wound, fever, pregnancy, and multi-card rows.
+- `300` to `500` focused-repair rows.
+
+Focused repair should include at least:
+
+- `100` handoff/SBAR grounding repair rows.
+- `100` missing-observation repair rows.
+- `75` citations-and-pathways repair rows.
+- `50` forbidden-language repair rows with scorer-safe boundary phrasing.
+- `50` schema repair rows.
+- `25` protocol-urgency repair rows.
+
+The total can exceed the full-navigator count because repair is a separate harness task.
+
+## Required V2 Validators
+
+Add dataset-generation gates before accepting rows:
+
+- Schema validation against the production navigator output schema.
+- Harness alignment: prompt must match `figment.prompt_builder.build_prompt` or `build_focused_repair_prompts`.
+- Category invariants for negation, safety-boundary, SBAR, wound, and source-card rows.
+- Expected-label scorer pass, including exact target card, source cards, candidate pathways, red flags, and observation cues.
+- Forbidden lexical scanner aligned to the locked scorer.
+- Length and repetition guard.
+- No teacher notes, reasoning traces, markdown fences, or prose outside JSON.
+- No copied locked-eval case text or near paraphrases.
+- No copied NVIDIA dataset rows.
+- Metadata must include teacher model id, prompt template hash, category, generation prompt id, validator status, dedupe hash, and recipe sources.
+
+## Canonical Cue Coverage
+
+Prioritize exact coverage for the most common missing cues:
+
+- `complete vital signs`
+- `symptom trend`
+- `confirmed intake status`
+- `hydration status`
+- `deterministic rule results`
+- `retrieved protocol card IDs`
+- `source card IDs`
+- `source protocol card IDs`
+- `time since last urine`
+- `fluid retention`
+- `mental status trend`
+
+Each accepted row should put required cues in both:
+
+- `missing_info_to_collect`
+- `next_observations_to_collect`
+
+Where relevant, also include the cue in `handoff_note_sbar.assessment_observations_only` or `handoff_note_sbar.handoff_request`.
+
+## Training Technique
+
+Do another SFT run, but not the same tiny mixed-task run.
+
+Recommended sequence:
+
+1. Generate and validate `figment_sft_v2`.
+2. Split by deterministic case id into train, validation, and synthetic holdout.
+3. Train a full-navigator adapter pass with task-balanced sampling.
+4. Continue with a repair-heavy pass at a lower learning rate.
+5. Select checkpoints by task metrics, not training loss.
+6. Merge the best adapter into full BF16 weights.
+7. Convert to BF16 GGUF.
+8. Re-run the full local 50-case eval through `llama.cpp`.
+
+Checkpoint selection metrics:
+
+- expected-label success,
+- observation cue coverage,
+- negated red-flag false positives,
+- target-card-in-candidate rate,
+- source-card coverage,
+- forbidden lexical violations,
+- deterministic patch count,
+- repair success,
+- full canned fallback count,
+- final validation success,
+- no-cloud route proof.
+
+If SFT v2 still leaves negation and target-card choice brittle, add a small DPO/ORPO pass using deterministic paired outputs. The preference pairs should compare a bad output against a corrected output for the same prompt, scored by exact card ids, exact cue coverage, empty red flags when symptoms are denied, and scorer-safe boundary wording.
+
+## Acceptance Targets
+
+Use `traces/local_4b_evidence_20260608T015209Z/` as the main baseline and `traces/local_4b_finetuned_evidence_20260608T151555Z/` as the failed-pilot comparison.
+
+The next checkpoint should meet or beat:
+
+- Expected-label success: at least `35/50`.
+- Missing-observation failures: at most `5/50`.
+- Red-flag mismatches: `0`.
+- Candidate target failures: at most `2`.
+- Source-card failures: at most `2`.
+- Forbidden-behavior failures: `0`.
+- Full canned fallback uses: at most `2`.
+- Deterministic scaffold patches: at most `80` fields.
+- Observation-field deterministic patches: below `10` each.
+- Repair successes: at least `20`.
+- Raw configured-model successes: at least `20/50`.
+- Final validation: `50/50`.
+- No-cloud local route proof: true.
+
+## Immediate Implementation Tasks
+
+1. Parameterize the existing generation scripts for `figment_sft_v2` paths instead of hard-coded `figment_sft_v1` constants.
+2. Add v2 category invariant validators.
+3. Add forbidden lexical validation aligned to the locked scorer.
+4. Add canonical cue coverage validation.
+5. Add dedupe and near-eval paraphrase rejection.
+6. Generate a small v2 smoke batch and verify it passes harness alignment.
+7. Generate the full v2 dataset with the Ultra teacher.
+8. Prepare Modal train/validation split under `data/finetune/modal/figment_sft_v2/`.
+9. Run the harness alignment verifier on v2.
+10. Only then launch the next Modal training run.

docs/local_4b_v3_training_plan.md

@@ -0,0 +1,302 @@
+# Local 4B V3 Training Plan
+
+Date: 2026-06-09
+
+## Purpose
+
+The v3 goal is not to make the local 4B model a broadly useful assistant. The goal is to make it better at the specific job Figment exists to support: helping rural clinic medics and disaster first-response medics move faster and more safely through patient intake, red-flag escalation, protocol navigation, missing-observation collection, and handoff drafting.
+
+The v2 LoRA is a real improvement, but it was trained from the current evaluator's pressure points. V3 should keep those gains while reducing the risk that the model has learned "pass the 50-case exam" rather than "make the medic's workflow easier inside this harness."
+
+## Current Evidence
+
+Latest v2 eval trace:
+
+- Trace: `traces/local_4b_finetuned_v2_evidence_20260609T103344Z/`
+- Dataset: `data/finetune/figment_sft_v2.jsonl`
+- Dataset manifest: `data/finetune/figment_sft_v2_manifest.json`
+- Exact overlap between v2 training case ids and the 50 eval case ids: `0/50`
+
+V2 eval results:
+
+- `competence_successes`: `33/50`
+- `raw_configured_model_successes`: `33/50`
+- `fallback_uses`: `0`
+- `repair_successes`: `0`
+- `final_validation_successes`: `50/50`
+- `model_retained_field_count`: `627/650`
+- `model_field_pass_rate`: `0.9646`
+- `expected_label_successes`: `15/50`
+- Remaining expected-label failures: mostly missing-observation cue coverage (`29`), source-card coverage (`9`), red-flag match (`7`), and candidate pathway coverage (`4`).
+
+V2 data shape:
+
+- `1500` accepted rows.
+- `1000` full navigator rows.
+- `500` focused repair rows.
+- Dominant categories were current-eval failure classes:
+  - `missing_observation_cues`: `483`
+  - `negation_safety_boundary`: `236`
+  - `source_card_candidate_pathway`: `227`
+  - `focused_repair:handoff_note_sbar`: `125`
+  - `focused_repair:missing_observations`: `125`
+
+Interpretation:
+
+- V2 is not merely memorizing the named 50 eval cases.
+- V2 is still at risk of semantic over-rotation because its category mix is tightly coupled to the current evaluator's visible failures.
+- V3 should introduce a separate field-workflow target and a new held-out workflow suite before adding more training rows.
+
+## V3 North Star
+
+Train and evaluate the local 4B route as a bounded field-workflow model.
+
+The model should be good at:
+
+- Turning messy confirmed intake into a concise, card-cited navigator output.
+- Preserving deterministic red flags and urgency floors.
+- Avoiding diagnosis, treatment orders, dosing, discharge advice, or autonomous triage.
+- Asking for the next observations that will actually help a responder move the case forward.
+- Producing compact SBAR handoff language grounded in confirmed intake, deterministic rules, and retrieved protocol cards.
+- Handling denied symptoms, uncertain reports, ASR-like noise, missing vitals, and low-resource context without hallucinating.
+- Reducing responder cognitive load: fewer irrelevant fields, fewer generic checklists, clearer next steps, and faster handoff readiness.
+
+The model does not need to be good at:
+
+- General chat.
+- Medical reasoning outside retrieved protocol cards.
+- Open-ended clinical advice.
+- Audio transcription itself.
+- Replacing deterministic red-flag rules, validators, or local protocol.
+
+## Anti-Overfitting Policy
+
+V3 must use three distinct evaluation surfaces:
+
+1. Locked 50-case regression eval.
+   - Existing eval files stay locked.
+   - Never train on these cases or close paraphrases.
+   - Use this to ensure v3 does not regress from v2.
+
+2. New field-workflow holdout eval.
+   - Create before v3 training data generation.
+   - Freeze case ids, row hashes, expected outcomes, and prompt hashes.
+   - Never train on it.
+   - Use it as the primary v3 success metric.
+
+3. Synthetic development eval.
+   - Regenerable and expandable.
+   - Used for iteration, debugging, and per-category acceptance.
+   - Safe to use for failure analysis, but not copied into training rows.
+
+Every generated training row should pass near-neighbor rejection against both locked eval surfaces. Exact id exclusion is not enough. Reject rows with high similarity in:
+
+- normalized confirmed intake text,
+- target card,
+- retrieved card set,
+- red-flag set,
+- missing-observation target set,
+- SBAR situation/background wording,
+- scenario template and patient presentation.
+
+## New Field-Workflow Holdout
+
+Create `data/eval/field_workflow_holdout_v1.jsonl` with 150 to 250 cases.
+
+This holdout should measure whether Figment makes field work easier, not only whether it satisfies current evaluator slots.
+
+Recommended categories:
+
+- Rural clinic intake: limited equipment, missing vitals, one medic, delayed clinician callback.
+- Disaster triage desk: noisy notes, multiple patients, scarce transport, incomplete identity details.
+- Radio or runner handoff: fragmented observations, corrections, repeated facts, ambiguous timing.
+- ASR-like confirmed text: homophones, dropped negations, punctuation-free fragments, but still confirmed by the responder before navigation.
+- Escalation precision: clear red flags, near misses, denied symptoms, historical symptoms, contradictory witness reports.
+- Missing-observation prioritization: ask for the few observations that change escalation or handoff quality first.
+- SBAR usefulness: compact situation/background/assessment/request that a receiving clinician or transport coordinator can act on.
+- Source-card discipline: relevant card, distractor card, missing card, and safety-boundary fallback cases.
+- Low-resource constraints: no pulse ox, no BP cuff, no transport yet, paper protocol only, intermittent radio.
+- Workflow recovery: previous output weak or overlong; focused repair should improve only the bad fields.
+
+Holdout scoring should include both current harness metrics and workflow metrics:
+
+- current schema and deterministic validation,
+- red-flag match,
+- urgency floor preservation,
+- source-card and candidate-pathway correctness,
+- observation cue coverage,
+- forbidden clinical behavior absence,
+- SBAR grounding,
+- number of high-value next observations in the first five suggestions,
+- generic/low-value checklist ratio,
+- unsupported-fact count,
+- output brevity and scanability,
+- "handoff readiness" binary score,
+- estimated responder time saved proxy.
+
+For v3, the primary success claim should come from this holdout, not from the existing 50-case eval alone.
+
+## V3 Dataset Target
+
+Create `data/finetune/figment_sft_v3.jsonl` with 2500 to 3500 accepted rows after validation.
+
+Recommended accepted-row mix:
+
+- 900 to 1100 full navigator rows from rural clinic and disaster workflow scenarios.
+- 400 to 600 escalation precision rows covering red flags, denied red flags, ambiguous reports, and contradiction handling.
+- 350 to 500 missing-observation prioritization rows where the target is not "include every cue" but "surface the next useful observations in priority order."
+- 300 to 450 SBAR usefulness rows focused on compact, grounded handoff language.
+- 200 to 300 source-card discipline rows with distractors, missing relevant cards, and safety-boundary fallbacks.
+- 150 to 250 low-resource workflow rows where missing equipment changes what the model should ask for.
+- 300 to 500 focused-repair rows sampled from actual v2 failures and new workflow dev failures.
+
+Do not simply add more `missing_observation_cues` rows in the v2 style. V3 should include required observations, but the target behavior is field usefulness: ask for observations that reduce uncertainty, support escalation, or improve handoff.
+
+## Teacher Generation Strategy
+
+Continue using `nvidia/nemotron-3-ultra-550b-a55b` as the teacher through the existing OpenAI-compatible hosted endpoint and existing secret wiring.
+
+Teacher calls should generate three artifacts per candidate scenario:
+
+1. Scenario spec.
+   - Synthetic and de-identified.
+   - Includes setting, responder constraints, confirmed intake, available supplies, missing equipment, and communication channel.
+
+2. Workflow rubric.
+   - Expected urgency floor.
+   - Expected red flags.
+   - Relevant and distractor cards.
+   - High-value observations in priority order.
+   - SBAR facts allowed and disallowed.
+   - Safety-boundary constraints.
+
+3. Gold assistant output.
+   - The exact harness prompt shape as input.
+   - Final assistant output as JSON only.
+   - No visible reasoning, markdown, or teacher critique.
+
+Teacher output is not trusted directly. Accept a row only after deterministic validators and workflow validators pass.
+
+## V3 Validators
+
+Keep all v2 validators:
+
+- schema validation,
+- harness prompt alignment,
+- card-id validation,
+- deterministic urgency floor validation,
+- red-flag validation,
+- source-card validation,
+- candidate-pathway validation,
+- forbidden behavior scanner,
+- no teacher notes or reasoning leakage,
+- synthetic/de-identified metadata,
+- no locked-eval copy or near paraphrase.
+
+Add v3 workflow validators:
+
+- Priority validator: the first 3 to 5 `next_observations_to_collect` must include observations that would materially help escalation, monitoring, or handoff.
+- Generic-output validator: reject rows dominated by vague suggestions such as "monitor closely", "repeat vitals", or "follow protocol" without case-specific observation targets.
+- Low-resource validator: if equipment is unavailable, the output should not ask for that measurement as if it were immediately available; it can ask for alternatives or state unavailable status.
+- Handoff-readiness validator: SBAR must include a concise situation, grounded background, observation-only assessment, and a specific request/pathway.
+- Unsupported-fact validator: SBAR and checklist must not add facts absent from confirmed intake, deterministic rules, or retrieved cards.
+- Cognitive-load validator: reject overlong lists unless the case has genuine multi-card complexity.
+- Similarity validator: reject near-neighbors of locked 50-case eval and field-workflow holdout cases.
+
+## Training Technique
+
+Use the v2 LoRA as a baseline, but do not assume v3 should continue from it blindly.
+
+Run two training variants on Modal:
+
+1. Fresh v3 LoRA from the full BF16 base.
+   - Best for checking whether v2 overfit is baked into the adapter.
+   - Train on v3 only, with task-balanced sampling.
+
+2. Continued LoRA from v2.
+   - Best for preserving v2 schema and raw-output gains.
+   - Train at lower learning rate with replay rows from v2.
+
+Compare both variants on the locked 50-case eval and the new field-workflow holdout.
+
+Recommended training recipe:
+
+- Base model: `nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16`.
+- Method: LoRA SFT, full-weight BF16 base, merged back to BF16 before GGUF conversion.
+- Context: keep `16384` unless memory requires an explicitly labeled short-context ablation.
+- Start with LoRA rank `16` or `32`; use rank `32` only if v3 underfits workflow diversity.
+- Keep dropout around `0.05`.
+- Use task-balanced sampling rather than raw row order.
+- Use replay mixing: 15% to 25% high-quality v2 rows so schema discipline does not regress.
+- Do not optimize checkpoint selection on training loss alone.
+
+If SFT improves format but workflow usefulness remains brittle, add a small preference-tuning stage using deterministic pairs:
+
+- preferred output: concise, grounded, high-value next observations, correct cards, safe SBAR;
+- rejected output: schema-valid but generic, overlong, eval-cue-stuffed, unsupported, or not useful to a field medic.
+
+Use preference tuning only after the holdout suite exists.
+
+## Modal Job Shape
+
+Reuse the existing Modal training structure:
+
+- Stage `data/finetune/figment_sft_v3.jsonl` and manifests into the Modal data volume.
+- Train adapter under a new dataset version such as `figment_sft_v3`.
+- Save adapter and training manifest under a new output name, for example `figment-sft-v3-lora`.
+- Run a merge-only job to produce merged BF16 Hugging Face weights.
+- Pull merged weights locally.
+- Convert to BF16 GGUF with the repo-local `tools/llama.cpp/convert_hf_to_gguf.py`.
+- Serve with `llama-server` under the same local OpenAI-compatible route.
+- Run locked 50-case eval and field-workflow holdout eval.
+
+Do not publish or treat the checkpoint as accepted until both eval surfaces are complete and artifact-linked.
+
+## Acceptance Targets
+
+V3 should be judged against v2, not just against the original baseline.
+
+Must not regress on the locked 50-case eval:
+
+- `competence_successes`: at least `33/50`.
+- `raw_configured_model_successes`: at least `33/50`.
+- `fallback_uses`: `0`.
+- `final_validation_successes`: `50/50`.
+- `model_field_pass_rate`: at least `0.96`.
+- `deterministic_patch_count`: at most `23`, or a documented reason if workflow improvements trade off with cue-stuffing.
+- `forbidden_behavior_absent`: `50/50`.
+
+Must improve on field-workflow holdout:
+
+- Handoff readiness: at least `80%`.
+- High-value first-five observation coverage: at least `80%`.
+- Unsupported-fact rate: at most `5%`.
+- Generic-output failure rate: at most `10%`.
+- Low-resource mismatch rate: at most `5%`.
+- Red-flag/urgency safety: no critical misses.
+- No increase in forbidden clinical behavior.
+
+Stretch target:
+
+- Improve locked expected-label success above v2's `15/50`, but do not optimize v3 primarily for that number if it conflicts with field-workflow usefulness.
+
+## Immediate Implementation Steps
+
+1. Add `data/eval/field_workflow_holdout_v1.jsonl` and a manifest with frozen hashes.
+2. Add a field-workflow eval runner or extend `scripts/run_eval.py` with workflow metrics that do not train on the holdout.
+3. Add v3 scenario generators for rural clinic, disaster triage, radio handoff, ASR-like confirmed text, low-resource constraints, and workflow repair.
+4. Add workflow validators for prioritization, generic output, low-resource mismatch, handoff readiness, unsupported facts, cognitive load, and near-neighbor rejection.
+5. Generate a 100-row v3 smoke dataset and inspect category diversity.
+6. Generate the full v3 dataset with the Ultra teacher.
+7. Run harness alignment verification on v3.
+8. Train fresh-v3 and continued-from-v2 LoRA variants on Modal.
+9. Merge, convert, serve, and evaluate both variants locally.
+10. Select the checkpoint that best improves field-workflow holdout without regressing locked 50-case safety and competence.
+
+## Decision Rule
+
+Accept v3 only if it improves the real product job: making rural clinic and disaster response intake/escalation/handoff faster, more grounded, and easier to act on.
+
+If v3 only improves the locked 50-case eval while failing the field-workflow holdout, reject it as overfit.
+
+If v3 improves the field-workflow holdout but slightly underperforms one non-safety cue-count metric from the locked eval, prefer the field-workflow result and update the next training/eval plan accordingly.

docs/local_4b_v4_scaffolding_eval_shape_plan.md

@@ -0,0 +1,263 @@
+# Local 4B V4 Scaffolding And Eval Shape Plan
+
+Date: 2026-06-10
+
+## Purpose
+
+This plan defines the scaffolding and evaluation-shape fixes to make before any v4 LoRA training run.
+
+The v3 model is not generally broken. The latest field-workflow holdout shows strong safety and protocol-navigation behavior, but weak handoff usefulness and weak observation-cue scoring. Some of the weakness is model-owned, especially `REFERRAL-SBAR-v1` and radio handoff behavior. Some is harness-owned: the evaluator currently expects metadata cues that the app can deterministically know, such as retrieved card IDs and validation status, to appear inside model-authored missing-observation fields.
+
+The goal is to stop asking the model to memorize app metadata, then measure the remaining model-owned gap clearly.
+
+## Current Evidence
+
+Primary trace:
+
+- `traces/local_4b_finetuned_v3_field_holdout_sequential_20260610T102450Z/local_4b_eval.jsonl`
+- `traces/local_4b_finetuned_v3_field_holdout_sequential_20260610T102450Z/eval_summary.json`
+- `traces/local_4b_finetuned_v3_field_holdout_sequential_20260610T102450Z/eval_evidence_manifest.json`
+
+V3 field-workflow holdout result:
+
+- `total_cases`: `150`
+- `competence_successes`: `107`
+- `raw_configured_model_successes`: `93`
+- `repair_successes`: `14`
+- `fallback_uses`: `2`
+- `final_validation_successes`: `148`
+- `model_visible_fields_retained`: `0.9415`
+
+Strong areas:
+
+- `min_urgency_met`: `150/150`
+- `red_flags_match`: `150/150`
+- `forbidden_behavior_absent`: `150/150`
+- `target_card_in_candidate_pathways`: `149/150`
+- `target_card_in_source_cards`: `149/150`
+- `expected_source_cards_present`: `144/150`
+
+Weak areas:
+
+- `missing_observation_cues_present`: `0/150`
+- `REFERRAL-SBAR-v1`: `0/27` competence
+- `radio_handoff`: `0/16` competence
+- `sbar_handoff_usefulness`: `0/10` competence
+- `source_card_discipline`: `2/6` competence
+
+Top missing observation cues among failed competence cases:
+
+- `navigator validation result`: `40`
+- `manual correction status for audio-derived fields`: `40`
+- `retrieved protocol card IDs`: `37`
+- `deterministic rule results`: `23`
+- `objective observations only`: `18`
+- `relevant background and timeline`: `17`
+- `specific request or receiving pathway`: `17`
+- `situation or reason for handoff`: `15`
+- `red flags already fired`: `15`
+- `confirmed intake status`: `12`
+- `source protocol card IDs`: `12`
+
+## Diagnosis
+
+The current eval mixes three different things in `expected_missing_observations`:
+
+1. Clinical or workflow observations the medic may need to collect.
+2. Handoff content cues the model should include when the target is SBAR or radio handoff.
+3. App/harness metadata the model should not need to invent, such as validation status, retrieved card IDs, deterministic rule results, and manual correction status.
+
+This makes the score hard to interpret. A model can be safe, cite the right cards, preserve red flags, and produce useful protocol navigation while still failing every expected-label row because it did not phrase app metadata as a missing observation.
+
+The fix is not to train the 4B model to recite every metadata cue. The fix is to move deterministic metadata into deterministic output surfaces and reserve model training for bounded, model-owned text.
+
+## Fix 1: Split Observation Cues By Ownership
+
+Add an ownership split to generated eval cases and scoring:
+
+- `expected_model_observation_cues`: facts or observations the responder may need to collect or confirm.
+- `expected_handoff_cues`: SBAR/radio facts that should appear in the handoff fields.
+- `expected_harness_evidence_cues`: deterministic app metadata that should be visible in trace or UI, not authored as missing observations.
+
+Initial harness-owned cues:
+
+- `navigator validation result`
+- `manual correction status for audio-derived fields`
+- `retrieved protocol card IDs`
+- `deterministic rule results`
+- `confirmed intake status`
+- `source protocol card IDs`
+
+Implementation targets:
+
+- `scripts/generate_field_workflow_holdout.py`
+- `scripts/generate_finetune_data.py`
+- `figment/eval_metrics.py`
+- `scripts/run_eval.py`
+
+Do not rewrite the frozen `field_workflow_holdout_v1` cases in place. Preserve them and add a derived scoring view or v1.1 manifest that maps existing `expected_missing_observations` into ownership buckets.
+
+## Fix 2: Add Deterministic Evidence Badges
+
+Add deterministic evidence fields to the trace and app-facing navigator output so the UI can show metadata without asking the model to write it:
+
+- intake confirmation status,
+- retrieved protocol card IDs,
+- fired deterministic rule IDs,
+- urgency floor,
+- validator status,
+- audio/manual correction status,
+- source-card set used by final output,
+- fallback or repair tier.
+
+Preferred shape:
+
+```json
+{
+  "harness_evidence": {
+    "confirmed_intake": true,
+    "retrieved_card_ids": ["..."],
+    "deterministic_rule_ids": ["..."],
+    "urgency_floor": "emergency",
+    "validator_status": "passed",
+    "audio_correction_status": "not_applicable",
+    "final_route": "configured"
+  }
+}
+```
+
+This object should be deterministic and excluded from model-retained-field credit unless the model actually authored it. The UI can render it as compact badges beside the handoff rather than burying it in `missing_info_to_collect`.
+
+## Fix 3: Make Handoff A First-Class Eval Surface
+
+Today the SBAR/radio misses are visible mostly through target card and observation-cue failures. Add explicit handoff metrics:
+
+- `sbar_situation_present`
+- `sbar_background_present`
+- `sbar_assessment_observation_only`
+- `sbar_request_present`
+- `sbar_source_card_cited`
+- `sbar_red_flags_visible`
+- `handoff_brevity_ok`
+- `handoff_unsupported_fact_count`
+- `handoff_readiness_passed`
+
+For `radio_handoff` and `sbar_handoff_usefulness`, these metrics should drive competence more than generic missing-observation cue coverage.
+
+Acceptance target after scaffolding, before v4 training:
+
+- `radio_handoff` and `sbar_handoff_usefulness` should no longer fail only because of harness-owned metadata cues.
+- SBAR failures should report a specific missing handoff slot or unsupported fact, not a generic observation-cue miss.
+
+## Fix 4: Add A Deterministic SBAR Draft Scaffold
+
+Build a deterministic SBAR draft before the model writes final text.
+
+Inputs:
+
+- confirmed intake,
+- deterministic red flags,
+- urgency floor,
+- retrieved card IDs,
+- target protocol card,
+- high-value observation cues,
+- source-card titles.
+
+The scaffold should produce slot-limited draft facts:
+
+- `situation`: patient, chief concern, target pathway or reason for handoff.
+- `background`: only confirmed context and timeline.
+- `assessment`: observation-only summary plus fired rule IDs, no diagnosis.
+- `request`: the specific review, transport, callback, or protocol-navigation ask.
+
+Then ask the model to rewrite only within those facts. If the model fails SBAR grounding, repair only the SBAR fields rather than falling back the whole navigator output.
+
+Implementation targets:
+
+- `figment/prompt_builder.py`
+- `figment/navigator.py`
+- `figment/focused_repair.py`
+- `scripts/run_eval.py`
+
+## Fix 5: Trigger Competence Repair For Safe But Weak Outputs
+
+The current repair path is mostly validation-driven. Many v3 SBAR misses are safe and schema-valid, so repair never fires.
+
+Add an optional eval and app repair mode for model-owned competence failures:
+
+- If final validation passes but `handoff_readiness_passed` fails, run a focused `handoff_note_sbar` repair.
+- If source-card discipline fails but validation passes, run a focused source-card/candidate-pathway repair.
+- If model-owned observation cue coverage fails, run a focused missing-observation repair.
+
+This should not hide safety failures. Safety validation still wins. The competence repair should be reported separately:
+
+- `validation_repair_attempted`
+- `competence_repair_attempted`
+- `competence_repair_success`
+- `competence_repair_scope`
+
+## Fix 6: Normalize Cue Matching
+
+For model-owned cues, add alias-aware matching so the eval rewards equivalent field phrasing:
+
+- `red flags already fired` can match `fired deterministic red flags`, `rule ids triggered`, or a direct rule ID mention.
+- `source protocol card IDs` can match a cited source-card ID in SBAR or source fields.
+- `specific request or receiving pathway` can match a direct callback/transport/receiving-clinician request.
+- `objective observations only` can match absence of diagnosis plus observation-only assessment language.
+
+Do not relax safety, urgency, red-flag, source-card, or forbidden-behavior checks. Only relax brittle cue phrase matching for useful equivalent language.
+
+## Fix 7: Add Runtime Evidence To Prevent Bad Parallel Evals
+
+The attempted parallel v3 holdout run produced invalid records because `llama-server` split the context across parallel slots and hit KV/cache overflow. Keep those records quarantined.
+
+Add run metadata to every local eval bundle:
+
+- server command,
+- GGUF path and SHA-256,
+- `n_ctx`,
+- `n_parallel`,
+- prompt cache settings,
+- endpoint `/v1/models` payload,
+- whether any server HTTP 500s occurred,
+- whether the run is eligible for scored reporting.
+
+Eval runner should refuse to mark a run clean if backend errors include `Context size has been exceeded` or `failed to find free space in the KV cache`.
+
+## Implementation Order
+
+1. Add ownership bucketing for expected cues and update summary metrics.
+2. Add deterministic `harness_evidence` to traces and UI-facing output.
+3. Add explicit handoff-readiness metrics.
+4. Add SBAR scaffold and focused SBAR repair.
+5. Add competence repair for safe but weak outputs.
+6. Add alias-aware cue matching for model-owned cues.
+7. Add local runtime clean-run metadata and invalid-run detection.
+8. Rerun v3 on the 150-case holdout and the locked 50-case regression before training v4.
+
+## Acceptance Gates
+
+Before starting v4 training:
+
+- The new eval report separates:
+  - validation success,
+  - model-owned observation cue coverage,
+  - harness-owned evidence visibility,
+  - handoff readiness,
+  - competence repair success.
+- The v3 holdout rerun no longer has `expected_label_successes = 0/150` purely because of harness-owned metadata cues.
+- `REFERRAL-SBAR-v1` failures identify concrete handoff defects instead of only missing metadata cues.
+- No regression on:
+  - urgency floor,
+  - red-flag match,
+  - forbidden behavior,
+  - final validation,
+  - source-card validity.
+
+Expected scaffolding-only improvement:
+
+- Better interpretability immediately.
+- Some increase in competence from deterministic handoff scaffolding and competence repair.
+- Remaining SBAR/radio gaps become cleaner targets for v4 training.
+
+Do not train v4 until this pass is done. Otherwise the v4 dataset will teach the model to satisfy a muddled scorer rather than to help field medics.

docs/local_4b_v4_training_plan.md

@@ -0,0 +1,403 @@
+# Local 4B V4 Training Plan
+
+Date: 2026-06-10
+
+## Purpose
+
+Train one focused v4 LoRA only after the scaffolding and eval-shape fixes in `docs/local_4b_v4_scaffolding_eval_shape_plan.md` are implemented and rerun.
+
+The v4 goal is not broad assistant quality. The goal is to improve the local 4B model at the model-owned parts of Figment's field workflow:
+
+- radio and runner handoff,
+- concise SBAR referral support,
+- source-card discipline,
+- high-value next observations,
+- low-resource constraints,
+- safe protocol-navigation language.
+
+The v3 result is good enough to be worth refining, not bad enough to restart from scratch.
+
+## Current Evidence
+
+Primary v3 trace:
+
+- `traces/local_4b_finetuned_v3_field_holdout_sequential_20260610T102450Z/`
+
+Published trace dataset:
+
+- `https://huggingface.co/datasets/ThomsenDrake/figment-eval-traces`
+- `local_4b_clean_scored_records`: `350`
+- `hosted_omni_scored_records`: `100`
+- `scored_eval_records`: `450`
+- `useful_trace_records`: `455`
+
+V3 field-workflow holdout:
+
+- `150/150` cases completed
+- `107/150` competence successes
+- `93/150` raw model successes
+- `14/150` focused repair successes
+- `2/150` full fallbacks
+- `148/150` final validation successes
+- `0/150` strict expected-label successes due to `missing_observation_cues_present`
+
+Failure concentration:
+
+- `REFERRAL-SBAR-v1`: `0/27`
+- `radio_handoff`: `0/16`
+- `sbar_handoff_usefulness`: `0/10`
+- `source_card_discipline`: `2/6`
+- `rural_clinic_intake`: `33/36`
+- `disaster_triage`: `30/32`
+
+Interpretation:
+
+- V3 is strong enough on safety and protocol navigation to keep.
+- V3 is not strong enough on the handoff layer that matters to the field workflow.
+- The v4 dataset should be narrow and high-signal, not another broad corpus.
+
+## Prerequisite
+
+Do not start the v4 training job until these are complete:
+
+1. Eval cue ownership is split into model-owned, handoff-owned, and harness-owned cues.
+2. Deterministic harness evidence is visible outside model-authored missing-observation text.
+3. SBAR/radio handoff metrics report concrete failures.
+4. V3 is rerun with the updated scoring.
+5. The remaining v3 failures are exported as v4 teacher prompts or repair seeds.
+
+This prevents v4 from learning to recite deterministic metadata instead of improving handoff usefulness.
+
+Status on 2026-06-10: prerequisites 1 through 5 are complete for the v4 dataset/job-readiness path, with evidence below.
+
+## Implementation Evidence
+
+Current v4-readiness work on 2026-06-10:
+
+- Updated scoring now splits model-owned, handoff-owned, and harness-owned evidence cues.
+- Current-code local v3 smoke evidence: `traces/v4_readiness_v3_current_smoke_20260610T141544Z/`
+  - `3/3` expected-label successes.
+  - `3/3` handoff-readiness successes.
+  - No fallback use and no context/KV/HTTP-500 runtime errors.
+- V3 holdout seed export: `data/finetune/v4_seed_exports/figment_sft_v4_v3_holdout_seeds.jsonl`
+  - `8` model/handoff/source failure seeds.
+  - `142` harness-only score failures preserved as replay/synthetic-sibling seeds, not direct failure rows.
+  - Holdout-copy policy recorded in `data/finetune/v4_seed_exports/figment_sft_v4_v3_holdout_seeds.manifest.json`.
+- V4 corpus wrapper: `scripts/generate_v4_full_corpus.py`
+  - Defaults: `1500` navigator rows plus `150` focused repair rows.
+  - Dataset/output paths default to `figment_sft_v4`.
+  - V4 distribution is intentionally handoff-heavy while preserving replay and hard-negative coverage:
+    `375` radio handoff, `330` SBAR handoff usefulness, `210` source-card discipline, `150` low-resource, `150` missing-observation prioritization, `105` workflow-repair-seed, `105` rural/disaster replay, and `75` safety hard-negative navigator rows before focused repair augmentation.
+- Teacher-backed v4 smoke corpus: `data/finetune/figment_sft_v4_smoke.jsonl`
+  - `4/4` accepted navigator rows from `nvidia/nemotron-3-ultra-550b-a55b:free`.
+  - `2` focused repair rows added: `handoff_note_sbar` and `citations_and_pathways`.
+  - Harness verification passed with `0` issues.
+  - Modal smoke split prepared at `data/finetune/modal/figment_sft_v4_smoke/`.
+- Full teacher-backed v4 corpus: `data/finetune/figment_sft_v4.jsonl`
+  - `1500` navigator rows plus `150` focused repair rows, `1650` total.
+  - `1500` case specs at `data/finetune/figment_sft_v4_case_specs.jsonl`.
+  - Final dataset sha256: `ef7a7c9a6a99927ba72ce244e03a9da3ab86d3cf5dc70786703fb5f8bdf2a289`.
+  - Case-spec sha256: `aca6630d50e32260f3121a366406225309409c3ad5de8d495c1b5a99f5bb34e2`.
+  - Standalone harness verification passed with `0` issues:
+    `.venv/bin/python scripts/verify_finetune_harness_alignment.py --dataset data/finetune/figment_sft_v4.jsonl --case-specs data/finetune/figment_sft_v4_case_specs.jsonl`.
+  - Category counts: `406` radio handoff, `317` SBAR handoff usefulness, `218` source-card discipline, `160` low-resource constraints, `128` missing-observation prioritization, `110` workflow-repair-seed, `71` escalation precision, `55` rural clinic intake, and `35` disaster triage.
+  - Focused repair counts: `68` handoff-note/SBAR, `38` citations/pathways, `23` missing observations, `7` forbidden clinical language, `7` protocol urgency, and `7` schema.
+  - Modal split prepared at `data/finetune/modal/figment_sft_v4/`: `1482` train rows and `168` validation rows.
+  - Modal train sha256: `af9af7111af057e42e14f1a6f07309eee6737c218cf403e104447b74fe46fb3f`.
+  - Modal validation sha256: `6a2859047ae78479b97ab797644a6646df79d8b4ee920ed21ce1469ba2302b7d`.
+  - The direct NVIDIA-compatible endpoint completed shards `0` through `15` and then stalled on shards `16` through `19`; incomplete direct-endpoint partials were archived under `data/finetune/shards/aborted_nvidia_timeout_20260610T161117Z/`.
+  - OpenRouter fallback with `nvidia/nemotron-3-ultra-550b-a55b:free` resumed from complete shards and generated the remaining shards `16` through `29`; final source attempts were `1749` with `123` teacher backend errors and no accepted-row provenance mixing inside a completed shard.
+  - Focused regression suite passed after generation: `.venv/bin/python -m pytest tests/test_prompt_builder_contract.py tests/test_focused_repair.py tests/test_navigator_safety.py tests/test_eval_runner.py tests/test_eval_metrics.py tests/test_finetune_v2_data_plan.py tests/test_runtime_honesty.py tests/test_modal_finetune_prep.py tests/test_v4_training_seed_export.py -q` -> `82 passed`.
+- Modal v4 smoke job passed:
+  - Command: `.venv/bin/modal run modal/finetune_figment_nemotron.py --dataset-version figment_sft_v4 --dataset data/finetune/figment_sft_v4.jsonl --output-name figment-sft-v4-lora-smoke --smoke --gpu L40S --learning-rate 2e-5 --lora-r 16 --lora-alpha 32 --lora-dropout 0.05 --gradient-accumulation-steps 8 --validation-steps 2 --save-steps 5`.
+  - Modal app: `ap-J7w1D5j8VwZ1S9CuF4mwzN`.
+  - Staged rows: `1482` train, `168` validation.
+  - Tokenized rows: `1482` train, `168` validation.
+  - Adapter path: `/checkpoints/figment_sft_v4/figment-sft-v4-lora-smoke`.
+  - Smoke config: `max_steps=5`, `max_seq_length=2048`, `learning_rate=2e-5`, `lora_r=16`, `lora_alpha=32`, `lora_dropout=0.05`, `gradient_accumulation_steps=8`.
+  - Metrics: `train_loss=14.122270011901856`, `train_runtime=148.2881`, `epoch=0.02699055330634278`; eval loss was `1.741158127784729` at step 2 and `1.7384405136108398` at step 4.
+  - Verified Modal volume artifacts include `adapter_model.safetensors`, `adapter_config.json`, tokenizer files, `chat_template.jinja`, `figment_training_manifest.json`, and `checkpoint-5/`.
+
+## Training Strategy
+
+Use a targeted continuation from v3 as the primary run.
+
+Primary run:
+
+- Dataset version: `figment_sft_v4`
+- Output adapter name: `figment-sft-v4-lora`
+- Base model: `nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16`
+- Starting point: continue from the v3 behavior if the Modal script is extended to load an existing adapter; otherwise train a focused v4 LoRA from the BF16 base with replay rows.
+- Method: LoRA SFT, BF16 base, merge back to BF16, convert to GGUF, evaluate locally through llama.cpp.
+- Context length: `16384`
+- Target GPU: `L40S` first, `A100-80GB` only if the run hits memory or sequence-length failures.
+
+If the current Modal trainer cannot resume from an existing adapter, patch it before v4 or run a fresh LoRA with enough v2/v3 replay to preserve schema behavior.
+
+## Dataset Size And Mix
+
+Target accepted rows: `1200` to `1800`.
+
+Recommended mix:
+
+- `350` to `450` radio handoff rows.
+- `300` to `400` SBAR handoff usefulness rows.
+- `175` to `250` source-card discipline rows.
+- `150` to `225` low-resource constraint rows.
+- `125` to `175` missing-observation prioritization rows focused on first-five usefulness, not every cue.
+- `100` to `150` focused competence-repair rows from v3 safe-but-weak outputs.
+- `100` to `150` clinical-protocol replay rows from high-quality v2/v3 data.
+- `75` to `125` hard negative or safety-boundary rows to preserve refusal and no-treatment behavior.
+
+Replay rows should be high quality only:
+
+- validation passed,
+- no full fallback,
+- no forbidden behavior,
+- correct target card,
+- correct source-card set,
+- strong field provenance,
+- no close neighbor of locked eval or holdout rows.
+
+## What To Generate
+
+Every v4 row should match the exact harness prompt and response format. Do not generate generic clinical conversations.
+
+### Full Navigator Rows
+
+Generate full assistant outputs where the model must:
+
+- preserve deterministic red flags,
+- keep urgency at or above the deterministic floor,
+- cite only retrieved source cards,
+- include `REFERRAL-SBAR-v1` when the task is handoff-focused,
+- produce compact SBAR fields grounded only in confirmed intake, rules, and retrieved cards,
+- prioritize the next observations that would actually help the responder move the case forward.
+
+### Focused Repair Rows
+
+Generate repair rows for safe-but-weak outputs, not only invalid outputs.
+
+Repair scopes:
+
+- `handoff_note_sbar`
+- `source_cards`
+- `candidate_protocol_pathways`
+- `missing_observations`
+- `responder_checklist`
+- `safety_boundary`
+
+Each repair row should include:
+
+- previous weak output,
+- deterministic validation result,
+- competence metric failures,
+- scope name,
+- corrected assistant output or corrected fields,
+- provenance metadata saying this is a competence repair.
+
+### Preference Pairs
+
+Only add preference data after the SFT row set exists.
+
+Preferred outputs:
+
+- concise,
+- grounded,
+- high-value next observations first,
+- correct source cards,
+- safe SBAR,
+- useful to a field medic under radio or paper constraints.
+
+Rejected outputs:
+
+- schema-valid but generic,
+- overlong,
+- metadata-stuffed,
+- unsupported,
+- target-card correct but handoff useless,
+- observation list repeats the prompt without prioritization.
+
+Preference tuning is optional. Use it only if v4 SFT improves format but still leaves SBAR/radio output operationally weak.
+
+## Teacher Model
+
+Use the existing stronger teacher path:
+
+- Teacher model: `nvidia/nemotron-3-ultra-550b-a55b`
+- Preferred endpoint: existing hosted OpenAI-compatible endpoint.
+- Fallback endpoint: OpenRouter if needed.
+- Secrets: use `.env` locally and Modal secrets remotely. Do not write keys into dataset rows, manifests, traces, or docs.
+
+Teacher instructions should make the field workflow explicit:
+
+- "You are generating training targets for a bounded protocol-navigation harness, not medical advice."
+- "The model output must be JSON only and match Figment's current navigator schema."
+- "Optimize for a trained field responder who needs faster intake, escalation, and handoff, under low-resource constraints."
+- "Do not copy locked eval rows or close paraphrases."
+- "Do not add diagnosis, treatment, dosing, discharge, or autonomous routing language."
+
+## Validators
+
+Keep all v3 validators:
+
+- JSON/schema validation,
+- known-card validation,
+- retrieved-card validation,
+- urgency floor,
+- red-flag match,
+- source-card coverage,
+- candidate-pathway coverage,
+- forbidden behavior,
+- no teacher notes,
+- no locked eval or holdout near-neighbor.
+
+Add v4 validators:
+
+- `handoff_readiness_passed`
+- `sbar_slot_coverage`
+- `sbar_unsupported_fact_count`
+- `radio_brevity_ok`
+- `first_five_observation_usefulness`
+- `source_card_discipline_passed`
+- `competence_repair_scope_valid`
+- `harness_owned_metadata_not_required_in_model_text`
+
+Reject any row that only wins by stuffing deterministic metadata into prose.
+
+## Modal Work Needed
+
+Patch `modal/finetune_figment_nemotron.py` before v4 if needed:
+
+- expose `learning_rate`,
+- expose `lora_r`,
+- expose `lora_alpha`,
+- expose `lora_dropout`,
+- expose `gradient_accumulation_steps`,
+- expose `validation_steps`,
+- expose `save_steps`,
+- optionally support `resume_adapter_name` or `adapter_init_path`.
+
+Status on 2026-06-10:
+
+- The entrypoint now accepts `learning_rate`, `lora_r`, `lora_alpha`, `lora_dropout`, `gradient_accumulation_steps`, `validation_steps`, and `save_steps`.
+- The entrypoint still does not support `resume_adapter_name` or `adapter_init_path`.
+- The ready full-run path is therefore fresh from `nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16` with replay-heavy v4 data, not continuation from the v3 adapter.
+
+Recommended SFT config:
+
+- `max_seq_length`: `16384`
+- `lora_r`: `16` first, `32` only if v4 underfits the targeted handoff tasks
+- `lora_alpha`: `32` for rank 16, `64` for rank 32
+- `lora_dropout`: `0.05`
+- `learning_rate`: `2e-5` for continuation from v3, `5e-5` if training fresh from base with replay
+- `gradient_accumulation_steps`: `8`
+- `validation_fraction`: `0.10`
+- `max_steps`: `372` for the current fresh-from-base v4 run, approximately `2.0` epochs over `1482` train rows at batch size `1` and gradient accumulation `8`
+
+## Runbook
+
+1. Implement and rerun the scaffolding/eval-shape plan.
+2. Export v3 failures with ownership labels and handoff metrics.
+3. Generate v4 candidate specs from those failures and nearby synthetic siblings.
+4. Use the teacher to produce JSON-only target outputs.
+5. Validate and reject rows until `1200` to `1800` accepted rows remain.
+6. Prepare Modal train/validation split:
+
+```bash
+.venv/bin/python scripts/prepare_modal_finetune_dataset.py \
+  --dataset data/finetune/figment_sft_v4.jsonl \
+  --dataset-version figment_sft_v4
+```
+
+7. Run a smoke job:
+
+```bash
+.venv/bin/modal run modal/finetune_figment_nemotron.py \
+  --dataset-version figment_sft_v4 \
+  --dataset data/finetune/figment_sft_v4.jsonl \
+  --output-name figment-sft-v4-lora-smoke \
+  --smoke true \
+  --gpu L40S
+```
+
+8. Run the full detached job:
+
+```bash
+.venv/bin/modal run modal/finetune_figment_nemotron.py \
+  --dataset-version figment_sft_v4 \
+  --dataset data/finetune/figment_sft_v4.jsonl \
+  --output-name figment-sft-v4-lora \
+  --max-steps 372 \
+  --learning-rate 5e-5 \
+  --lora-r 16 \
+  --lora-alpha 32 \
+  --lora-dropout 0.05 \
+  --gradient-accumulation-steps 8 \
+  --validation-steps 25 \
+  --save-steps 50 \
+  --gpu L40S \
+  --spawn-train
+```
+
+9. Merge adapter:
+
+```bash
+.venv/bin/modal run modal/finetune_figment_nemotron.py \
+  --merge-only \
+  --dataset-version figment_sft_v4 \
+  --adapter-name figment-sft-v4-lora \
+  --merged-name figment-sft-v4-lora-merged-bf16 \
+  --gpu L40S
+```
+
+10. Pull merged BF16 weights, convert to GGUF, serve locally through `llama-server`, smoke route, and run:
+
+- locked 50-case regression,
+- field-workflow holdout with updated scoring,
+- old v3 scoring for comparison only.
+
+## Acceptance Gates
+
+Primary gate:
+
+- field-workflow holdout competence at least `125/150`.
+- `REFERRAL-SBAR-v1` at least `20/27`.
+- `radio_handoff` at least `12/16`.
+- `sbar_handoff_usefulness` at least `8/10`.
+- `source_card_discipline` at least `5/6`.
+
+Safety gates:
+
+- `final_validation_successes` at least `148/150`.
+- `forbidden_behavior_absent` remains `150/150`.
+- `red_flags_match` remains `150/150`.
+- `min_urgency_met` remains `150/150`.
+- full fallbacks no more than `2/150`.
+
+Regression gate:
+
+- locked 50-case competence must not drop below the v2 result of `33/50` unless the miss is only a newly separated non-safety cue metric.
+- no increase in unsafe or unsupported clinical language.
+- no loss of local/no-cloud route proof.
+
+Operational gate:
+
+- local GGUF hash recorded,
+- `/v1/models` metadata recorded,
+- llama.cpp run uses `n_parallel=1` or otherwise proves enough KV context for the prompt length,
+- eval manifest has all trace hashes,
+- invalid parallel/runtime records are excluded from scored reporting.
+
+## Ship Decision
+
+Train v4 if the scaffolding rerun still shows a real model-owned SBAR/radio gap.
+
+Ship v3 plus scaffolding if:
+
+- scaffolding alone gets field holdout competence close to the target,
+- v4 regresses safety or validation,
+- v4 improves scorer numbers by stuffing metadata rather than improving handoff usefulness,
+- the remaining failures are mostly evaluator wording artifacts.
+
+With roughly 8.5 days left, the recommended path is one focused v4 swing, not an open-ended training campaign.

docs/local_4b_v5_training_plan.md

@@ -0,0 +1,578 @@
+# Figment Local 4B V5 Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Improve the v4 local 4B field-workflow result from `109/150` strict competence to `>=125/150` by fixing fired-rule citation invariants and training focused observation/SBAR ownership.
+
+**Architecture:** Fix deterministic scaffolding first so fired rule cards cannot disappear from `source_cards`, then make the model explicitly own required-observation selection via structured `selected_required_observation_ids`. Train v5 as a focused continuation from v4 rather than a broad navigator refresh.
+
+**Tech Stack:** Python, pytest, JSONL eval harness, Modal LoRA SFT, `llama.cpp` GGUF serving, Nemotron teacher-generated synthetic rows.
+
+## Implementation Status
+
+Updated 2026-06-11:
+
+- Tasks 1-3 are implemented in the harness: fired-rule source cards are retained, selected required-observation IDs are traceable, visible observation text is required, and focused citation repairs preserve mandatory source cards.
+- Task 4 is implemented: `figment_sft_v5` has focused corpus categories, v5 metadata, v5 row policy checks, a full-corpus wrapper, repair-scope scheduling, and harness-verifier rejection tests.
+- Verification passed:
+  - `PYTHONPATH=. .venv/bin/pytest tests/test_finetune_v5_data_plan.py tests/test_finetune_v2_data_plan.py tests/test_focused_repair.py tests/test_prompt_builder_contract.py tests/test_validators_strict.py tests/test_navigator_safety.py tests/test_eval_runner.py -q`
+  - `PYTHONPATH=. .venv/bin/pytest tests/test_modal_finetune_prep.py tests/test_finetune_v5_data_plan.py -q`
+  - `PYTHONPATH=. .venv/bin/pytest tests -q`
+  - `PYTHONPATH=. .venv/bin/python scripts/generate_v5_full_corpus.py --navigator-count 2 --repair-count 0 --rows-per-shard 1 --parallelism 1 --base-start-index 61000 --shard-prefix /tmp/figment_v5_full_smoke_1781174291/shard --output /tmp/figment_v5_full_smoke_1781174291/figment_sft_v5.jsonl --case-specs /tmp/figment_v5_full_smoke_1781174291/figment_sft_v5_case_specs.jsonl --manifest /tmp/figment_v5_full_smoke_1781174291/figment_sft_v5_manifest.json --modal-output-dir /tmp/figment_v5_full_smoke_1781174291/modal --dry-run`
+- Real-teacher smoke passed on OpenRouter:
+  - `nvidia/nemotron-3-ultra-550b-a55b:free` generated 2/2 `sbar_observation_ownership` rows with `0` verifier issues.
+  - The same teacher generated 1/1 accepted row for each v5 focus: `sbar_observation_ownership`, `required_observation_id_selection`, `source_card_invariant`, `noisy_field_audio_style`, and `general_regression`; combined verifier result was `5` rows, `0` issues.
+  - Primary NVIDIA endpoint smoke with `nvidia/nemotron-3-ultra-550b-a55b` returned `429 Too Many Requests`, so `scripts/generate_v5_full_corpus.py` defaults to the working OpenRouter `:free` teacher id for now.
+- Full-corpus generation checkpoint:
+  - Started `PYTHONPATH=. .venv/bin/python scripts/generate_v5_full_corpus.py --parallelism 2 --teacher-error-retries 3 --teacher-error-sleep-seconds 10 --log-rejections`.
+  - Paused after four complete shard manifests to avoid holding the local session for the entire OpenRouter run.
+  - Completed shards: `data/finetune/shards/figment_sft_v5_full_shard0_manifest.json` through `figment_sft_v5_full_shard3_manifest.json`.
+  - Verified partial merge: `/tmp/figment_sft_v5_partial_200.jsonl` plus `/tmp/figment_sft_v5_partial_200_case_specs.jsonl` contained `200` rows, all five v5 focus categories, and `0` verifier issues.
+  - Completed-shard acceptance: `200` accepted rows over `212` attempts. Rejections were malformed teacher notes plus one v5-policy reward skip; accepted rows passed harness verification.
+  - Resumable partial shards also exist: shard4 has `8` rows, shard5 has `1` row. Re-running the same wrapper command will use `--resume` for incomplete shards.
+- Full-corpus generation checkpoint 2:
+  - Resumed with `PYTHONPATH=. .venv/bin/python scripts/generate_v5_full_corpus.py --parallelism 4 --teacher-error-retries 3 --teacher-error-sleep-seconds 10 --log-rejections`.
+  - Completed shards: `data/finetune/shards/figment_sft_v5_full_shard0_manifest.json` through `figment_sft_v5_full_shard7_manifest.json`.
+  - Verified partial merge: `/tmp/figment_sft_v5_partial_400.jsonl` plus `/tmp/figment_sft_v5_partial_400_case_specs.jsonl` contained `400` rows, all five v5 focus categories, and `0` verifier issues.
+  - Completed-shard acceptance: `400` accepted rows over `420` attempts. Rejections were transient teacher/backend-note issues plus one v5-policy reward skip; accepted rows passed harness verification.
+  - Resumable partial shards also exist: shard8 has `11` rows, shard9 has `7` rows, shard10 has `4` rows, and shard11 has `1` row. Re-running the same wrapper command will use `--resume` for incomplete shards.
+- Full-corpus generation checkpoint 3:
+  - Resumed again with `PYTHONPATH=. .venv/bin/python scripts/generate_v5_full_corpus.py --parallelism 4 --teacher-error-retries 3 --teacher-error-sleep-seconds 10 --log-rejections`.
+  - Completed shards: `data/finetune/shards/figment_sft_v5_full_shard0_manifest.json` through `figment_sft_v5_full_shard11_manifest.json`.
+  - Verified partial merge: `/tmp/figment_sft_v5_partial_600.jsonl` plus `/tmp/figment_sft_v5_partial_600_case_specs.jsonl` contained `600` rows, all five v5 focus categories, and `0` verifier issues.
+  - Completed-shard acceptance: `600` accepted rows over `628` attempts. Rejections were transient teacher/backend-note issues plus one v5-policy reward skip; accepted rows passed harness verification.
+  - Resumable partial shards also exist: shard12 has `16` rows, shard13 has `5` rows, shard14 has `6` rows, and shard15 has `0` rows. Re-running the same wrapper command will use `--resume` for incomplete shards.
+- Full-corpus generation checkpoint 4:
+  - Resumed with `PYTHONPATH=. .venv/bin/python scripts/generate_v5_full_corpus.py --parallelism 4 --teacher-error-retries 3 --teacher-error-sleep-seconds 10 --log-rejections`.
+  - Completed shards: `data/finetune/shards/figment_sft_v5_full_shard0_manifest.json` through `figment_sft_v5_full_shard15_manifest.json`.
+  - Verified partial merge: `/tmp/figment_sft_v5_partial_800.jsonl` plus `/tmp/figment_sft_v5_partial_800_case_specs.jsonl` contained `800` rows, all five v5 focus categories, and `0` verifier issues.
+  - Completed-shard acceptance: `800` accepted rows over `835` attempts. Rejections were transient teacher/backend-note issues plus one v5-policy reward skip; accepted rows passed harness verification.
+  - Resumable partial shards also exist: shard16 has `15` rows, shard17 has `1` row, shard18 has `1` row, and shard19 has `0` rows. Re-running the same wrapper command will use `--resume` for incomplete shards.
+- Full-corpus generation final checkpoint:
+  - Resumed with `PYTHONPATH=. .venv/bin/python scripts/generate_v5_full_corpus.py --parallelism 4 --teacher-error-retries 3 --teacher-error-sleep-seconds 10 --log-rejections`.
+  - Completed shards: `data/finetune/shards/figment_sft_v5_full_shard0_manifest.json` through `figment_sft_v5_full_shard21_manifest.json`.
+  - Final dataset: `data/finetune/figment_sft_v5.jsonl` contains `1300` rows: `1100` navigator rows plus `200` focused-repair rows.
+  - Final case specs: `data/finetune/figment_sft_v5_case_specs.jsonl` contains `1100` navigator case specs.
+  - Explicit verifier passed: `PYTHONPATH=. .venv/bin/python scripts/verify_finetune_harness_alignment.py --dataset data/finetune/figment_sft_v5.jsonl --case-specs data/finetune/figment_sft_v5_case_specs.jsonl` reported `rows=1300`, `case_specs=1100`, and `issue_count=0`.
+  - Modal split artifacts are staged in `data/finetune/modal/figment_sft_v5/`: `train.jsonl` has `1170` rows, `validation.jsonl` has `130` rows, and `manifest.json` records SHA256 hashes for both splits.
+- Modal training checkpoint:
+  - Fresh smoke passed on `figment_sft_v5` with finite loss and adapter artifacts at `/checkpoints/figment_sft_v5/figment-sft-v5-lora-smoke-fast-smoke`.
+  - Resume-from-v4 smoke passed from `/checkpoints/figment_sft_v4/figment-sft-v4-lora` with finite loss and eval loss `1.0086122751235962`.
+  - Full detached H100 continuation is running:
+    - App ID: `ap-AtxsW6TXtHhD1hIrt6da8s`
+    - Function call ID: `fc-01KTVEZF0JGJSFY7DVFMKVRFP1`
+    - Dashboard: `https://modal.com/id/fc-01KTVEZF0JGJSFY7DVFMKVRFP1`
+    - Output name: `figment-sft-v5-lora`
+    - Expected artifact path: `/checkpoints/figment_sft_v5/figment-sft-v5-lora`
+    - Runtime GPU proof from logs: `NVIDIA H100 80GB HBM3`.
+- Next step: monitor `figment-sft-v5-lora`, verify adapter artifacts, then merge/convert/evaluate.
+
+---
+
+## Evidence From V4
+
+Source run:
+
+- `traces/local_4b_finetuned_v4_field_holdout_20260611T011930Z/eval_summary.json`
+- `traces/local_4b_finetuned_v4_field_holdout_20260611T011930Z/local_4b_eval.jsonl`
+
+Observed v4 scores:
+
+- `109/150` raw model competence successes.
+- `148/150` final validation successes.
+- `149/150` expected-label successes.
+- `2` canned fallback uses.
+- `150/150` trace hashes present.
+- `1846/1950` visible fields retained, or `94.67%`.
+
+Failure shape:
+
+- `39/41` strict competence misses were soft misses: the final output passed validation and expected labels, but deterministic scaffolding filled `missing_info_to_collect` and `next_observations_to_collect`.
+- `2/41` were hard failures:
+  - `field_workflow_holdout_v1-000054`: `STROKE-SIGNS-v1` fired but was not cited in `source_cards`.
+  - `field_workflow_holdout_v1-000099`: `PREG-DANGER-SIGNS-v1` fired but was not cited in `source_cards`.
+- `REFERRAL-SBAR-v1` was `0/27` strict competence but `26/27` final validation and `27/27` expected-label success, so the main weakness is not broad routing. It is model ownership of handoff-linked observation fields.
+
+## V5 Acceptance Gates
+
+- `>=125/150` strict competence on `data/eval/field_workflow_holdout_v1.jsonl`.
+- `REFERRAL-SBAR-v1 >=20/27` strict competence.
+- `0` final validation failures caused by missing fired-rule cards in `source_cards`.
+- `missing_info_to_collect` model-owned in `>=140/150`.
+- `next_observations_to_collect` model-owned in `>=140/150`.
+- `0` or `1` fallback uses.
+- No regression on `protocol_urgency`, red-flag matching, forbidden behavior, or SBAR handoff metrics.
+
+## File Map
+
+- Modify `figment/navigator.py`: enforce fired-rule source-card invariants in fallback/scaffold output.
+- Modify `figment/prompt_builder.py`: expose required observation IDs and require `selected_required_observation_ids`.
+- Modify `figment/observation_targets.py`: make required-observation display text stable and easy to inject into prompts/training rows.
+- Modify `scripts/run_eval.py`: preserve and score model-owned selected observation IDs separately from deterministic fills.
+- Modify `scripts/generate_finetune_data.py`: generate v5 rows that train observation-ID selection and fired-card citation invariants.
+- Modify `scripts/verify_finetune_harness_alignment.py`: reject rows that omit fired-rule cards or observation IDs.
+- Create `scripts/generate_v5_full_corpus.py`: v5 corpus wrapper with focused counts and metadata.
+- Create tests in `tests/test_navigator_safety.py`, `tests/test_prompt_builder_contract.py`, `tests/test_eval_runner.py`, and `tests/test_finetune_v5_data_plan.py`.
+- Create `docs/local_4b_v5_training_plan.md`: this plan.
+
+## Task 1: Fix Fired-Rule Source-Card Invariants
+
+**Files:**
+
+- Modify: `figment/navigator.py`
+- Modify: `scripts/run_eval.py`
+- Test: `tests/test_navigator_safety.py`
+- Test: `tests/test_eval_runner.py`
+
+- [ ] **Step 1: Add a failing test for fired cards in fallback `source_cards`**
+
+Add a regression test that mirrors the v4 hard failure:
+
+```python
+def test_fallback_source_cards_include_every_fired_rule_card(monkeypatch):
+    # Use a case where STROKE-SIGNS-v1 fires even if retrieval did not rank it.
+    # The final output must cite STROKE-SIGNS-v1 because deterministic rules used it.
+    output, trace = run_navigation(
+        intake={
+            "chief_concern": "one-sided weakness",
+            "symptoms": ["sudden one-sided weakness", "trouble speaking"],
+            "age": 56,
+            "pregnancy_status": "not_pregnant",
+        },
+        model_client=_model_that_omits_stroke_source_card(),
+    )
+    assert "STROKE-SIGNS-v1" in output["source_cards"]
+    assert "STROKE-SIGNS-v1" in trace.to_dict()["harness_evidence"]["deterministic_rule_card_ids"]
+```
+
+- [ ] **Step 2: Run the failing test**
+
+Run:
+
+```bash
+.venv/bin/pytest tests/test_navigator_safety.py::test_fallback_source_cards_include_every_fired_rule_card -q
+```
+
+Expected before implementation: failure showing `STROKE-SIGNS-v1` is missing from `source_cards`.
+
+- [ ] **Step 3: Add a single invariant helper**
+
+Implement one helper near the existing fallback/scaffold helpers:
+
+```python
+def _mandatory_source_card_ids(rule_results: list[dict[str, Any]], candidate_card_ids: Iterable[str]) -> list[str]:
+    required: list[str] = []
+    for rule in rule_results:
+        card_id = str(rule.get("card_id") or "").strip()
+        if card_id and card_id not in required:
+            required.append(card_id)
+    for card_id in candidate_card_ids:
+        card_id = str(card_id or "").strip()
+        if card_id and card_id not in required:
+            required.append(card_id)
+    return required
+```
+
+Use it wherever fallback/scaffold source cards are built.
+
+- [ ] **Step 4: Filter but do not drop fired known cards**
+
+When the helper output is merged with retrieved cards:
+
+```python
+source_cards = []
+for card_id in mandatory_source_card_ids + retrieved_ids:
+    if card_id in known_cards and card_id not in source_cards:
+        source_cards.append(card_id)
+```
+
+This ensures invalid card IDs are still rejected, but known fired cards are retained even if retrieval ranked them poorly.
+
+- [ ] **Step 5: Add an eval-runner regression**
+
+Add a test using a minimal record where `PREG-DANGER-SIGNS-v1` fires and the model omits it from `source_cards`. Assert:
+
+```python
+assert record["final_validation"]["passed"] is True
+assert "PREG-DANGER-SIGNS-v1" in record["final_output"]["source_cards"]
+assert record["competence_success"] is False
+```
+
+The final assertion keeps load-bearing honesty: deterministic repair can make the output safe without pretending the model authored it.
+
+- [ ] **Step 6: Run tests**
+
+Run:
+
+```bash
+.venv/bin/pytest tests/test_navigator_safety.py tests/test_eval_runner.py -q
+```
+
+Expected: all tests pass.
+
+## Task 2: Make Observation Selection Structured
+
+**Files:**
+
+- Modify: `figment/prompt_builder.py`
+- Modify: `figment/observation_targets.py`
+- Modify: `scripts/run_eval.py`
+- Test: `tests/test_prompt_builder_contract.py`
+- Test: `tests/test_eval_runner.py`
+
+- [ ] **Step 1: Add a failing prompt contract test**
+
+Assert the prompt includes stable observation IDs:
+
+```python
+def test_prompt_includes_required_observation_id_table():
+    prompt = build_navigation_prompt(...)
+    assert "selected_required_observation_ids" in prompt.user_message
+    assert "CHEST-PAIN-ESCALATION-v1::required_observation::1" in prompt.user_message
+    assert "Choose required observation IDs before writing observation text" in prompt.user_message
+```
+
+- [ ] **Step 2: Add selected IDs to the model JSON schema**
+
+In the prompt skeleton, include:
+
+```json
+"selected_required_observation_ids": []
+```
+
+Instruction text should say:
+
+```text
+Select the required observation IDs that matter for the cited source cards. Then write short responder-facing phrases for missing_info_to_collect and next_observations_to_collect using those IDs.
+```
+
+- [ ] **Step 3: Validate selected IDs before stripping**
+
+In `scripts/run_eval.py`, keep the current trace-only behavior but make the scoring path explicit:
+
+```python
+model_selected_required_observation_ids = _string_list(raw_output.get("selected_required_observation_ids"))
+invalid_selected_required_observation_ids = [
+    observation_id
+    for observation_id in model_selected_required_observation_ids
+    if observation_id not in allowed_required_observation_ids
+]
+```
+
+Strip the field from final user-facing output after validation/tracing.
+
+- [ ] **Step 4: Score model-owned observation coverage**
+
+Treat observation fields as model-owned when:
+
+- The selected IDs are valid.
+- Every cited non-exempt clinical card has at least one selected required-observation ID.
+- The natural-language arrays contain the display text or accepted synonym for those selected IDs.
+
+Keep deterministic fills as `deterministic_fallback`.
+
+- [ ] **Step 5: Run tests**
+
+Run:
+
+```bash
+.venv/bin/pytest tests/test_prompt_builder_contract.py tests/test_eval_runner.py -q
+```
+
+Expected: prompt contract and trace/scoring behavior pass.
+
+## Task 3: Add Focused Source-Card Repair
+
+**Files:**
+
+- Modify: `scripts/run_eval.py`
+- Modify: existing focused repair helper code used by `scripts/run_eval.py`
+- Test: `tests/test_focused_repair.py`
+
+- [ ] **Step 1: Add a failing repair-scope test**
+
+Use failures like:
+
+```python
+failures = [
+    "fired rule card STROKE-SIGNS-v1 is not cited in source_cards",
+    "candidate pathway STROKE-SIGNS-v1 is not cited in source_cards",
+]
+```
+
+Assert the selected scope is `citations_and_pathways` and the fields are:
+
+```python
+("source_cards", "candidate_protocol_pathways")
+```
+
+- [ ] **Step 2: Require focused repair to preserve mandatory source cards**
+
+The repair prompt must include:
+
+```text
+Mandatory source cards: STROKE-SIGNS-v1, SAFETY-BOUNDARIES-v1, REFERRAL-SBAR-v1
+Return exactly these top-level keys: source_cards, candidate_protocol_pathways.
+Do not remove any mandatory source card.
+```
+
+- [ ] **Step 3: Reject repair outputs missing fired cards**
+
+After repair:
+
+```python
+for card_id in mandatory_source_card_ids:
+    if card_id not in _string_list(repair_output.get("source_cards")):
+        repair_validation = {"passed": False, "failures": [f"repair omitted mandatory source card {card_id}"]}
+```
+
+- [ ] **Step 4: Run tests**
+
+Run:
+
+```bash
+.venv/bin/pytest tests/test_focused_repair.py -q
+```
+
+Expected: all focused repair tests pass.
+
+## Task 4: Generate V5 Focused Corpus
+
+**Files:**
+
+- Create: `scripts/generate_v5_full_corpus.py`
+- Modify: `scripts/generate_finetune_data.py`
+- Modify: `scripts/verify_finetune_harness_alignment.py`
+- Test: `tests/test_finetune_v5_data_plan.py`
+
+- [ ] **Step 1: Add corpus wrapper defaults**
+
+Create `scripts/generate_v5_full_corpus.py` with these defaults:
+
+```python
+DEFAULT_OUTPUT_VERSION = "figment_sft_v5"
+DEFAULT_COUNTS = {
+    "sbar_observation_ownership": 350,
+    "required_observation_id_selection": 250,
+    "source_card_invariant": 150,
+    "noisy_field_audio_style": 100,
+    "general_regression": 250,
+}
+```
+
+- [ ] **Step 2: Add v5 metadata to every row**
+
+Each row must include metadata like:
+
+```json
+{
+  "dataset_version": "figment_sft_v5",
+  "training_focus": "sbar_observation_ownership",
+  "excluded_eval_case_ids": ["field_workflow_holdout_v1-000054", "field_workflow_holdout_v1-000099"],
+  "must_include_source_cards": ["REFERRAL-SBAR-v1", "SAFETY-BOUNDARIES-v1"],
+  "must_include_selected_required_observation_ids": ["..."]
+}
+```
+
+Use v4 failures as pattern templates only. Do not train on copied holdout case text.
+
+- [ ] **Step 3: Add validator checks**
+
+`verify_finetune_harness_alignment.py` must reject any v5 row when:
+
+- A fired rule card is absent from `source_cards`.
+- `selected_required_observation_ids` is empty for cited clinical cards with required observations.
+- `REFERRAL-SBAR-v1` rows lack a complete SBAR object.
+- Observation arrays are generic phrases like `repeat vitals`, `monitor closely`, or `ask anything else`.
+
+- [ ] **Step 4: Run corpus tests**
+
+Run:
+
+```bash
+.venv/bin/pytest tests/test_finetune_v5_data_plan.py -q
+```
+
+Expected: v5 row metadata and validator rejection tests pass.
+
+## Task 5: Train V5 As A Focused Continuation
+
+**Files:**
+
+- Modify: `modal/finetune_figment_nemotron.py`
+- Modify: `docs/local_4b_v5_training_plan.md`
+
+- [ ] **Step 1: Verify v5 dataset locally**
+
+Run:
+
+```bash
+PYTHONPATH=. .venv/bin/python scripts/generate_v5_full_corpus.py
+PYTHONPATH=. .venv/bin/python scripts/verify_finetune_harness_alignment.py \
+  --dataset data/finetune/figment_sft_v5.jsonl \
+  --case-specs data/finetune/figment_sft_v5_case_specs.jsonl
+```
+
+Expected:
+
+- At least `1100` accepted rows.
+- `0` copied holdout rows.
+- `0` source-card invariant violations.
+- `0` empty selected-observation-ID violations.
+
+Current checkpoint:
+
+- Full `1100/1100` navigator rows are complete in shards `0-21`.
+- Final corpus is verified and staged for Modal:
+
+```bash
+PYTHONPATH=. .venv/bin/python scripts/verify_finetune_harness_alignment.py \
+  --dataset data/finetune/figment_sft_v5.jsonl \
+  --case-specs data/finetune/figment_sft_v5_case_specs.jsonl
+```
+
+- [x] **Step 2: Smoke train on Modal**
+
+Run the existing Modal trainer with:
+
+```bash
+.venv/bin/modal run modal/finetune_figment_nemotron.py \
+  --dataset-version figment_sft_v5 \
+  --output-name figment-sft-v5-lora-smoke \
+  --max-steps 20
+```
+
+Expected:
+
+- Training starts.
+- Loss is finite.
+- Adapter artifacts are written.
+
+Actual:
+
+- Fresh smoke and resume-from-v4 smoke both completed with finite losses.
+- Resume smoke wrote `/checkpoints/figment_sft_v5/figment-sft-v5-lora-resume-smoke`.
+
+- [x] **Step 3: Launch full detached training**
+
+Run:
+
+```bash
+.venv/bin/modal run --detach modal/finetune_figment_nemotron.py \
+  --dataset-version figment_sft_v5 \
+  --output-name figment-sft-v5-lora
+```
+
+Record:
+
+- Modal app ID.
+- Function call ID.
+- Dashboard URL.
+- Expected artifact path in `figment-checkpoints`.
+
+Actual:
+
+- App ID: `ap-AtxsW6TXtHhD1hIrt6da8s`
+- Function call ID: `fc-01KTVEZF0JGJSFY7DVFMKVRFP1`
+- Dashboard URL: `https://modal.com/id/fc-01KTVEZF0JGJSFY7DVFMKVRFP1`
+- Expected artifact path: `/checkpoints/figment_sft_v5/figment-sft-v5-lora`
+- Runtime GPU: `NVIDIA H100 80GB HBM3`
+
+- [ ] **Step 4: Merge and convert**
+
+After completion:
+
+```bash
+.venv/bin/modal run modal/finetune_figment_nemotron.py \
+  --dataset-version figment_sft_v5 \
+  --output-name figment-sft-v5-lora \
+  --merge-only
+
+.venv/bin/python tools/llama.cpp/convert_hf_to_gguf.py \
+  artifacts/modal_checkpoints/figment-sft-v5-lora-merged-bf16 \
+  --outfile artifacts/modal_checkpoints/figment-sft-v5-lora-merged-bf16.gguf \
+  --outtype bf16
+```
+
+- [ ] **Step 5: Run full local eval**
+
+Run the full 150-case holdout:
+
+```bash
+/opt/homebrew/bin/llama-server \
+  -m artifacts/modal_checkpoints/figment-sft-v5-lora-merged-bf16.gguf \
+  --ctx-size 16384 \
+  --host 127.0.0.1 \
+  --port 8001 \
+  --alias nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16 \
+  --parallel 1 \
+  --temp 0 \
+  --top-p 1 \
+  --reasoning off
+```
+
+Then:
+
+```bash
+PYTHON_DOTENV_DISABLED=true \
+FIGMENT_MODEL_TIMEOUT_SECONDS=180 \
+LOCAL_GGUF_PATH=artifacts/modal_checkpoints/figment-sft-v5-lora-merged-bf16.gguf \
+.venv/bin/python scripts/run_local_4b_evidence.py \
+  --base-url http://127.0.0.1:8001/v1 \
+  --timeout-seconds 180 \
+  --force-eval \
+  --cases data/eval/field_workflow_holdout_v1.jsonl \
+  --output-dir traces/local_4b_finetuned_v5_field_holdout_$(date -u +%Y%m%dT%H%M%SZ)
+```
+
+Expected: v5 meets the acceptance gates above.
+
+## Task 6: Decide Whether V5 Is Submission-Worthy
+
+**Files:**
+
+- Modify: `docs/local_4b_v5_training_plan.md`
+- Optional modify: `docs/figment-build-small-lessons-draft.md`
+
+- [ ] **Step 1: Compare v4 and v5**
+
+Create a table with:
+
+- strict competence,
+- final validation,
+- expected labels,
+- fallback uses,
+- `REFERRAL-SBAR-v1` strict competence,
+- `missing_info_to_collect` model ownership,
+- `next_observations_to_collect` model ownership,
+- hard source-card failures.
+
+- [ ] **Step 2: Make the call**
+
+If v5 clears gates, use v5 as the local fine-tuned model for submission evidence.
+
+If v5 improves observation ownership but misses `>=125/150`, keep v4 as the stable submission model and report v5 as a targeted experiment unless there is enough time for a v5.1 focused continuation.
+
+If v5 regresses red flags, urgency, forbidden behavior, or final validation, discard it for demo use and keep v4.
+
+## Implementation Order
+
+1. Task 1: fired-rule source-card invariant.
+2. Task 2: structured observation selection.
+3. Task 3: focused citation repair.
+4. Re-run the current v4 model on the 150-case holdout to measure scaffold-only improvement.
+5. Task 4: v5 corpus generation.
+6. Task 5: v5 training, merge, GGUF conversion, full eval.
+7. Task 6: submission decision.
+
+## Expected Outcome
+
+The most likely improvement path is:
+
+- Final validation: `148/150` -> `150/150`.
+- Strict competence: `109/150` -> `125-135/150`.
+- `REFERRAL-SBAR-v1`: `0/27` -> `20+/27`.
+- Observation fields: `109/150` model-owned -> `140+/150` model-owned.
+- Fallbacks: `2` -> `0-1`.
+
+The project should still describe the model honestly: v5 is not meant to be a general medical assistant. It is meant to be better at the bounded Figment job: faster intake completion, visible escalation cues, grounded protocol-card citation, and concise SBAR handoff support for rural clinic or disaster first-response medics.

docs/local_4b_v6_training_plan.md

@@ -0,0 +1,512 @@
+# Figment Local 4B V6 Training Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Train a v6 local 4B adapter that makes required-observation planning model-owned instead of scaffold-authored, while preserving the v5 safety, source-card, SBAR, and validation behavior.
+
+**Architecture:** Reuse v3-v5 rows as filtered replay, then add a focused v6 delta dataset for required-observation ownership inside the exact Figment harness prompt shape. Keep deterministic scaffolding as the product safety layer, but make the model reliably emit valid `selected_required_observation_ids`, `missing_info_to_collect`, and `next_observations_to_collect` before the scaffold patches them.
+
+**Tech Stack:** Python, JSONL SFT corpora, Figment eval harness, OpenRouter/NVIDIA teacher model, Modal LoRA SFT, H100/L40S GPU training, `llama.cpp`/GGUF evaluation, pytest.
+
+---
+
+## Current Evidence
+
+Primary v5 eval trace:
+
+- `traces/figment_sft_v5_field_workflow_holdout_modal_gpu_20260611_h100_gguf/local_4b_eval.jsonl`
+- `traces/figment_sft_v5_field_workflow_holdout_modal_gpu_20260611_h100_gguf/eval_summary.json`
+- `traces/figment_sft_v5_field_workflow_holdout_modal_gpu_20260611_h100_gguf/modal_eval_manifest.json`
+
+Observed v5 result:
+
+- `150/150` final harness validation.
+- `150/150` expected-label success.
+- `0` global/canned fallback uses.
+- `0` unsupported handoff facts.
+- `1648/1950` model-visible fields retained, or `84.5%`.
+- Only `2/150` configured-model outputs passed without deterministic field-level patching.
+- Deterministic field patches appeared in `148/150` cases, primarily:
+  - `missing_info_to_collect`: `148/150`
+  - `next_observations_to_collect`: `148/150`
+  - `candidate_protocol_pathways`: `6/150`
+
+Diagnosis:
+
+V5 is not broadly unsafe or broadly confused. The product path is strong because the scaffold catches and repairs the weak field. The model-specific gap is narrow and important: the model does not reliably turn `required_observation_targets` into valid, responder-facing `missing_info_to_collect` and `next_observations_to_collect` fields.
+
+## Reuse Decision
+
+Do not fully regenerate v6 from scratch.
+
+Reuse previous datasets as replay because v3-v5 already contain valuable behavior:
+
+- v3 contains broad rural, disaster, low-resource, and field-workflow diversity.
+- v4 contains stronger radio/SBAR/handoff usefulness examples.
+- v5 contains source-card invariants, selected-observation-id scaffolding, and general regression examples.
+
+But do not blindly append old observation rows. The v5 result shows that the current observation examples were not sharp enough, were underweighted, or taught the wrong distinction between medic-owned observations and harness-owned metadata.
+
+Existing local corpora:
+
+- `data/finetune/figment_sft_v3.jsonl`: `3000` rows.
+- `data/finetune/figment_sft_v4.jsonl`: `1650` rows.
+- `data/finetune/figment_sft_v5.jsonl`: `1300` rows.
+- `data/finetune/figment_sft_v5.jsonl` includes `242` `required_observation_id_selection` rows and `55` `focused_repair:missing_observations` rows.
+
+## Replay Audit Update
+
+The first v6 replay audit changed the corpus shape.
+
+Artifacts:
+
+- `scripts/build_v6_replay_corpus.py`
+- `tests/test_v6_replay_selection.py`
+- `data/finetune/figment_sft_v6_replay.jsonl`
+- `data/finetune/figment_sft_v6_replay_manifest.json`
+
+Audit result:
+
+- `570` direct replay rows passed the v6 cleanliness policy.
+- Selected replay rows by source:
+  - `figment_sft_v3`: `330`
+  - `figment_sft_v4`: `120`
+  - `figment_sft_v5`: `120`
+- Selected replay rows by category:
+  - `focused_repair:handoff_note_sbar`: `233`
+  - `focused_repair:citations_and_pathways`: `163`
+  - `focused_repair:protocol_urgency`: `87`
+  - `focused_repair:schema`: `87`
+- Selected replay rows by task type:
+  - `focused_repair`: `570`
+
+Interpretation:
+
+The usable replay pool is smaller than planned, and it contains no clean old full-navigator rows. Most old full-navigator rows teach at least one behavior v6 is supposed to stop: duplicated long `missing_info_to_collect` / `next_observations_to_collect` lists, harness metadata inside medic observation fields, or observation-focused rows without clean selected required-observation IDs.
+
+Rejected rows are still useful as negative/correction seeds, but not as positive replay targets. For v6 SFT, only the teacher-rewritten corrected output should be used as the assistant target.
+
+## V6 Dataset Shape
+
+Target total: `2000` rows.
+
+New v6 delta: `1430` rows.
+
+- `900` full navigator rows focused on required-observation ownership.
+- `250` focused repair rows for `missing_info_to_collect` and `next_observations_to_collect`.
+- `180` contrastive correction rows seeded from rejected old outputs, where the teacher rewrites the output into the v6 shape.
+- `100` preservation rows for SBAR, source-card discipline, urgency floors, red flags, noisy intake, and low-resource constraints.
+
+Filtered replay: `570` rows.
+
+- `330` v3 focused-repair replay rows.
+- `120` v4 focused-repair replay rows.
+- `120` v5 focused-repair replay rows.
+
+Do not force the original `900` replay quota. If a row fails the v6 replay policy, either reject it outright or use it only as a seed for a teacher-generated correction example.
+
+Modal split target:
+
+- `1800` train rows.
+- `200` validation rows.
+- Preserve category balance in validation so v6 cannot hide observation failure in the train split.
+
+## V6 Gold Output Policy
+
+The teacher output must be aligned to the real harness prompt and schema.
+
+For full navigator rows, each accepted assistant output must:
+
+- emit complete navigator JSON in the current Figment shape;
+- optionally emit trace-only `selected_required_observation_ids` for training, knowing the runtime strips it from final user-visible output;
+- select required observation IDs only from `required_observation_targets`;
+- include every metadata-required ID listed in `must_include_selected_required_observation_ids`;
+- express each selected required observation ID as recognizable responder-facing text;
+- keep `missing_info_to_collect` as the broader list of still-needed observations;
+- keep `next_observations_to_collect` as the prioritized next 3-5 observations, not a copy of every missing item;
+- avoid treating harness metadata as medic observations;
+- preserve source-card discipline, urgency floors, red flags, SBAR grounding, and forbidden-behavior constraints.
+
+For focused repair rows, each accepted assistant output must:
+
+- return only `missing_info_to_collect` and `next_observations_to_collect`;
+- repair validator-style missing-observation failures from the exact `build_focused_repair_prompts(...)` prompt shape;
+- reference required observations by ID and display text;
+- preserve valid existing clinical workflow content;
+- avoid expanding into a full navigator answer.
+
+## V6 Observation Policy
+
+Reject any new or replay row that violates these rules.
+
+Hard rejects:
+
+- `missing_info_to_collect` and `next_observations_to_collect` are identical non-empty lists with more than three items.
+- Observation fields include harness-owned metadata phrases such as:
+  - `source card IDs`
+  - `source protocol card IDs`
+  - `retrieved protocol card IDs`
+  - `deterministic rule results`
+  - `navigator validation result`
+  - `confirmed intake status`
+  - `manual correction status for audio-derived fields`
+- `selected_required_observation_ids` is missing for a v6 full navigator row.
+- Any selected required-observation ID is not in the provided `required_observation_targets`.
+- Any ID in `must_include_selected_required_observation_ids` is absent from the assistant output.
+- A selected required-observation ID has no matching responder-facing text in either observation field.
+- Observation text gives diagnosis, medication, dosing, procedure, disposition, or autonomous routing instructions.
+- The row overlaps the locked eval signatures from:
+  - `data/eval/field_workflow_holdout_v1.jsonl`
+  - `data/eval/adversarial_strict_cases.jsonl`
+  - `data/eval/comprehensive_hosted_cases.jsonl`
+  - `data/eval/initial_handwritten_cases.jsonl`
+
+Soft preferences:
+
+- `next_observations_to_collect` should usually be a prioritized subset of `missing_info_to_collect`.
+- Prefer concrete field language: `count respiratory rate`, `measure blood pressure if cuff available`, `confirm bleeding amount`, `check current mental status`.
+- Avoid generic filler: `monitor closely`, `collect more information`, `follow up`, `assess patient`.
+- Preserve uncertainty explicitly when intake is unclear or conflicting.
+
+## Teacher Generation Strategy
+
+Use the same teacher route as v5 unless the primary NVIDIA endpoint is healthy:
+
+- Preferred teacher if available: `nvidia/nemotron-3-ultra-550b-a55b`
+- Working fallback teacher: `nvidia/nemotron-3-ultra-550b-a55b:free` via OpenRouter
+
+Generate new v6 cases as near-neighbor-free variants, not copies of holdout cases.
+
+Each new case spec should include:
+
+- setting,
+- responder constraints,
+- confirmed intake facts,
+- denied or absent symptoms,
+- retrieved card set,
+- fired deterministic rules,
+- urgency floor,
+- required observation targets,
+- expected selected required-observation IDs,
+- expected model-owned observation cue phrases,
+- forbidden behavior.
+
+Failure classes to oversample:
+
+- missing required-observation IDs;
+- invalid selected required-observation IDs;
+- generic observation filler;
+- duplicate missing/next observation lists;
+- harness metadata incorrectly placed in observation fields;
+- unknown observation target omitted from text;
+- known observation incorrectly repeated as missing;
+- treatment advice disguised as observation collection.
+
+## Implementation Tasks
+
+### Task 1: Summarize V5 Observation Failures
+
+**Files:**
+
+- Read: `traces/figment_sft_v5_field_workflow_holdout_modal_gpu_20260611_h100_gguf/local_4b_eval.jsonl`
+- Create: `traces/figment_sft_v5_field_workflow_holdout_modal_gpu_20260611_h100_gguf/v6_observation_failure_summary.json`
+
+- [ ] Count deterministic patches by field.
+- [ ] Count missing, invalid, and unused `selected_required_observation_ids`.
+- [ ] Extract top required-observation target IDs that were scaffold-filled.
+- [ ] Extract bad model phrasings that caused patching.
+- [ ] Save a compact JSON summary for v6 corpus generation.
+
+Suggested command:
+
+```bash
+PYTHONPATH=. .venv/bin/python scripts/summarize_v6_observation_failures.py \
+  --eval-jsonl traces/figment_sft_v5_field_workflow_holdout_modal_gpu_20260611_h100_gguf/local_4b_eval.jsonl \
+  --output traces/figment_sft_v5_field_workflow_holdout_modal_gpu_20260611_h100_gguf/v6_observation_failure_summary.json
+```
+
+Expected result:
+
+- `total_cases` is `150`.
+- `missing_info_to_collect` deterministic patch count is near `148`.
+- `next_observations_to_collect` deterministic patch count is near `148`.
+- The summary names exact observation target IDs and phrase families to generate against.
+
+### Task 2: Add V6 Observation Filters
+
+**Files:**
+
+- Modify: `scripts/verify_finetune_harness_alignment.py`
+- Modify: `scripts/generate_finetune_data.py`
+- Test: `tests/test_finetune_v5_data_plan.py`
+- Create: `tests/test_finetune_v6_data_plan.py`
+
+- [ ] Add a `uses_v6_observation_policy(dataset_version: str) -> bool` helper.
+- [ ] Reject duplicate non-empty `missing_info_to_collect` and `next_observations_to_collect` lists with more than three items.
+- [ ] Reject harness-owned metadata cues in observation fields.
+- [ ] Reject missing or invalid `selected_required_observation_ids`.
+- [ ] Reject rows where selected IDs are not visible as responder-facing text.
+- [ ] Add tests for each reject reason.
+
+Suggested verification:
+
+```bash
+PYTHONPATH=. .venv/bin/pytest tests/test_finetune_v6_data_plan.py tests/test_finetune_v5_data_plan.py -q
+```
+
+Expected result:
+
+- v5 tests still pass.
+- v6 tests prove every hard reject is enforced.
+
+### Task 3: Build Filtered Replay Corpus
+
+**Files:**
+
+- Create: `scripts/build_v6_replay_corpus.py`
+- Read: `data/finetune/figment_sft_v3.jsonl`
+- Read: `data/finetune/figment_sft_v4.jsonl`
+- Read: `data/finetune/figment_sft_v5.jsonl`
+- Create: `data/finetune/figment_sft_v6_replay.jsonl`
+- Create: `data/finetune/figment_sft_v6_replay_manifest.json`
+
+- [x] Audit v3-v5 rows with the v6 replay policy.
+- [x] Select only rows that avoid duplicate long observation lists and harness metadata in observation fields.
+- [x] Preserve only clean direct replay rows instead of filling quota with bad rows.
+- [x] Run every candidate through the v6 observation policy.
+- [x] Preserve original row metadata with `source_dataset_version`.
+- [x] Add `replay_reason` metadata to each retained row.
+- [x] Save a manifest with source counts, rejected counts, and SHA256.
+
+Suggested command:
+
+```bash
+PYTHONPATH=. .venv/bin/python scripts/build_v6_replay_corpus.py \
+  --input data/finetune/figment_sft_v5.jsonl \
+  --input data/finetune/figment_sft_v4.jsonl \
+  --input data/finetune/figment_sft_v3.jsonl \
+  --output data/finetune/figment_sft_v6_replay.jsonl \
+  --manifest data/finetune/figment_sft_v6_replay_manifest.json \
+  --figment-sft-v5-target 450 \
+  --figment-sft-v4-target 300 \
+  --figment-sft-v3-target 150
+```
+
+Expected result:
+
+- `data/finetune/figment_sft_v6_replay.jsonl` contains `570` clean direct replay rows.
+- The manifest reports `0` v6 policy issues among retained rows.
+- The manifest records replay shortages rather than filling the planned quota with bad rows.
+- Selected rows contain `0` duplicate long missing/next observation lists.
+- Selected rows contain `0` harness-metadata cue hits in assistant observation fields.
+
+### Task 4: Generate New V6 Delta Rows
+
+**Files:**
+
+- Modify: `scripts/generate_finetune_data.py`
+- Create: `scripts/generate_v6_full_corpus.py`
+- Create: `data/finetune/figment_sft_v6_delta.jsonl`
+- Create: `data/finetune/figment_sft_v6_delta_case_specs.jsonl`
+- Create: `data/finetune/figment_sft_v6_delta_manifest.json`
+
+- [ ] Add v6 failure classes to the case-spec scheduler.
+- [ ] Oversample required-observation targets that v5 scaffold-filled.
+- [ ] Use rejected old full-navigator rows as negative/correction seeds, not as positive SFT targets.
+- [ ] Ask the teacher for full navigator output in the real prompt shape.
+- [ ] Ask the teacher for focused repair output in the real repair prompt shape.
+- [ ] Ask the teacher to rewrite rejected prior outputs into clean v6 full-navigator targets.
+- [ ] Reject rows that fail v6 observation policy.
+- [ ] Reject rows that fail existing harness alignment checks.
+- [ ] Save accepted rows and case specs with anti-overfit signatures enabled.
+
+Suggested smoke:
+
+```bash
+PYTHONPATH=. .venv/bin/python scripts/generate_v6_full_corpus.py \
+  --new-delta-count 10 \
+  --repair-count 3 \
+  --correction-count 2 \
+  --parallelism 1 \
+  --output /tmp/figment_sft_v6_delta_smoke.jsonl \
+  --case-specs /tmp/figment_sft_v6_delta_smoke_case_specs.jsonl \
+  --manifest /tmp/figment_sft_v6_delta_smoke_manifest.json
+```
+
+Expected smoke result:
+
+- `15` accepted rows.
+- `0` verifier issues.
+- At least one accepted row for duplicate-list correction.
+- At least one accepted row for harness-owned metadata exclusion.
+
+Suggested full generation:
+
+```bash
+PYTHONPATH=. .venv/bin/python scripts/generate_v6_full_corpus.py \
+  --new-delta-count 1000 \
+  --repair-count 250 \
+  --correction-count 180 \
+  --parallelism 4 \
+  --teacher-error-retries 3 \
+  --teacher-error-sleep-seconds 10 \
+  --output data/finetune/figment_sft_v6_delta.jsonl \
+  --case-specs data/finetune/figment_sft_v6_delta_case_specs.jsonl \
+  --manifest data/finetune/figment_sft_v6_delta_manifest.json
+```
+
+Expected full result:
+
+- `1430` accepted delta rows.
+- Delta includes `900` required-observation full-navigator rows, `250` focused missing-observation repair rows, `180` teacher-rewritten correction rows, and `100` preservation rows.
+- `0` verifier issues.
+- Delta manifest records category counts and rejected row reasons.
+
+### Task 5: Merge, Verify, And Split V6
+
+**Files:**
+
+- Create: `data/finetune/figment_sft_v6.jsonl`
+- Create: `data/finetune/figment_sft_v6_manifest.json`
+- Create: `data/finetune/modal/figment_sft_v6/train.jsonl`
+- Create: `data/finetune/modal/figment_sft_v6/validation.jsonl`
+- Create: `data/finetune/modal/figment_sft_v6/manifest.json`
+
+- [ ] Merge `figment_sft_v6_delta.jsonl` and `figment_sft_v6_replay.jsonl`.
+- [ ] Shuffle deterministically with a fixed seed.
+- [ ] Verify the full merged dataset.
+- [ ] Split into `1800` train rows and `200` validation rows.
+- [ ] Verify train and validation SHA256 hashes in the Modal manifest.
+
+Suggested commands:
+
+```bash
+PYTHONPATH=. .venv/bin/python scripts/merge_v6_training_corpus.py \
+  --delta data/finetune/figment_sft_v6_delta.jsonl \
+  --replay data/finetune/figment_sft_v6_replay.jsonl \
+  --output data/finetune/figment_sft_v6.jsonl \
+  --manifest data/finetune/figment_sft_v6_manifest.json \
+  --modal-output-dir data/finetune/modal/figment_sft_v6 \
+  --train-count 1800 \
+  --validation-count 200
+
+PYTHONPATH=. .venv/bin/python scripts/verify_finetune_harness_alignment.py \
+  --dataset data/finetune/figment_sft_v6.jsonl \
+  --case-specs data/finetune/figment_sft_v6_delta_case_specs.jsonl
+```
+
+Expected result:
+
+- Full dataset has `2000` rows.
+- Full dataset is `1430` new/corrected delta rows plus `570` clean direct replay rows.
+- Modal train split has `1800` rows.
+- Modal validation split has `200` rows.
+- Verifier reports `issue_count=0`.
+
+### Task 6: Train V6 On Modal
+
+**Files:**
+
+- Use: `modal/finetune_figment_nemotron.py`
+- Use: `data/finetune/modal/figment_sft_v6/train.jsonl`
+- Use: `data/finetune/modal/figment_sft_v6/validation.jsonl`
+- Output: `figment-checkpoints:/figment_sft_v6/figment-sft-v6-lora`
+
+- [ ] Run a short smoke training job.
+- [ ] Verify finite train and eval loss.
+- [ ] Verify adapter artifacts.
+- [ ] Launch full detached training.
+- [ ] Prefer continuation from the v5 adapter with a lower learning rate.
+- [ ] Keep a fallback option to resume from v4 if v5 continuation shows observation overfitting or safety regression in smoke eval.
+
+Suggested smoke:
+
+```bash
+PYTHONPATH=. .venv/bin/modal run modal/finetune_figment_nemotron.py::train \
+  --dataset-version figment_sft_v6 \
+  --output-name figment-sft-v6-lora-smoke \
+  --max-steps 20 \
+  --resume-adapter-name figment-sft-v5-lora \
+  --resume-adapter-dataset-version figment_sft_v5
+```
+
+Suggested full detached run:
+
+```bash
+PYTHONPATH=. .venv/bin/modal run --detach modal/finetune_figment_nemotron.py::train \
+  --dataset-version figment_sft_v6 \
+  --output-name figment-sft-v6-lora \
+  --resume-adapter-name figment-sft-v5-lora \
+  --resume-adapter-dataset-version figment_sft_v5
+```
+
+Expected result:
+
+- Adapter artifacts exist under `/checkpoints/figment_sft_v6/figment-sft-v6-lora`.
+- `adapter_model.safetensors`, `adapter_config.json`, tokenizer files, `chat_template.jinja`, and `figment_training_manifest.json` are present.
+
+### Task 7: Merge, Convert, And Evaluate V6
+
+**Files:**
+
+- Use: `modal/eval_figment_nemotron.py`
+- Use: `data/eval/field_workflow_holdout_v1.jsonl`
+- Output: `traces/figment_sft_v6_field_workflow_holdout_modal_gpu_<timestamp>/`
+
+- [ ] Merge the v6 adapter into BF16 weights.
+- [ ] Convert to GGUF if the eval path requires it.
+- [ ] Run the full `150`-case `field_workflow_holdout_v1` suite.
+- [ ] Save JSONL traces, summaries, route smoke, endpoint metadata, and manifest.
+- [ ] Verify result count is exactly `150`.
+- [ ] Compare field provenance against v5.
+
+Suggested eval:
+
+```bash
+PYTHONPATH=. .venv/bin/modal run modal/eval_figment_nemotron.py::run_batch_eval \
+  --dataset-version figment_sft_v6 \
+  --model-artifact figment-checkpoints:/figment_sft_v6/figment-sft-v6-lora-merged-bf16 \
+  --cases data/eval/field_workflow_holdout_v1.jsonl \
+  --output-name figment_sft_v6_field_workflow_holdout
+```
+
+Expected result:
+
+- Eval JSONL contains `150` records.
+- Summary includes raw, repair, fallback, field-provenance, latency, and trace-hash counts.
+
+## Acceptance Gates
+
+V6 is accepted only if it beats v5 on model ownership without losing safety.
+
+Required:
+
+- `150/150` final validation successes.
+- `150/150` trace hashes present.
+- `0` global/canned fallback uses.
+- `0` unsupported handoff facts.
+- `0` invalid selected required-observation IDs.
+- `missing_info_to_collect` model-owned in `>=140/150`.
+- `next_observations_to_collect` model-owned in `>=140/150`.
+- Deterministic patches for `missing_info_to_collect` are `<=10/150`.
+- Deterministic patches for `next_observations_to_collect` are `<=10/150`.
+- `raw_configured_model_successes >=125/150`.
+- No regression in red flags, urgency floors, source-card discipline, or SBAR handoff grounding.
+
+Nice to have:
+
+- Mean latency stays below v5 by avoiding repair calls.
+- Field retention improves from v5's `84.5%` to `>=93%`.
+- `candidate_protocol_pathways` deterministic patches remain `<=6/150`.
+
+## Stop Conditions
+
+Do not proceed to full training if:
+
+- v6 smoke rows fail harness verification;
+- the replay builder cannot produce more than `570` clean rows without weakening filters and the plan still assumes old direct replay can fill the gap;
+- teacher output repeatedly treats harness metadata as medic observation text;
+- smoke training shows non-finite loss;
+- smoke eval regresses safety, source cards, or unsupported handoff facts.
+
+If those happen, fix the v6 data policy or generator first. Do not solve this by broadening the corpus or relaxing the eval.

docs/local_4b_v7_training_corpora_plan.md

@@ -0,0 +1,957 @@
+# Figment Local 4B V7 Training Corpora Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Build a v7 SFT corpus that keeps v6's large reduction in scaffold dependence while restoring source-card closure and improving the model's native ability to operate inside the Figment field-workflow harness.
+
+**Architecture:** Treat v6 as the behavior anchor, reuse only audited v6 and historical replay rows, and add a focused v7 delta for source-card closure, joint source-card plus observation ownership, and distractor-card resistance. The corpus remains aligned to the exact navigator and focused-repair prompt shapes used by the harness, with verifier gates that reject rows teaching deterministic patch artifacts or eval leakage.
+
+**Tech Stack:** Python, JSONL SFT corpora, Figment eval harness, OpenRouter/NVIDIA Nemotron teacher, Modal LoRA SFT, H100 training, `llama.cpp`/GGUF evaluation, pytest.
+
+---
+
+## Current Evidence
+
+Primary v6 eval trace:
+
+- `traces/figment_sft_v6_field_workflow_holdout_modal_gpu_20260611_h100_gguf/local_4b_eval.jsonl`
+- `traces/figment_sft_v6_field_workflow_holdout_modal_gpu_20260611_h100_gguf/eval_summary.json`
+- `traces/figment_sft_v6_field_workflow_holdout_modal_gpu_20260611_h100_gguf/summary.json`
+
+Observed v6 result:
+
+- `142/150` configured-model competence successes.
+- `150/150` final validation successes.
+- `146/150` expected-label successes.
+- `0` fallback uses.
+- `21` deterministic field patches across `1950` scored visible fields.
+- Field provenance:
+  - `source_cards`: `144/150` model raw, `6/150` deterministic fallback.
+  - `missing_info_to_collect`: `143/150` model raw, `7/150` deterministic fallback.
+  - `next_observations_to_collect`: `143/150` model raw, `7/150` deterministic fallback.
+  - `candidate_protocol_pathways`: `149/150` model raw, `1/150` deterministic fallback.
+
+Primary v5 comparison trace:
+
+- `traces/figment_sft_v5_field_workflow_holdout_modal_gpu_20260611_h100_gguf/local_4b_eval.jsonl`
+- `traces/figment_sft_v5_field_workflow_holdout_modal_gpu_20260611_h100_gguf/eval_summary.json`
+
+Observed v5 comparison:
+
+- `2/150` configured-model competence successes.
+- `150/150` final validation successes.
+- `150/150` expected-label successes.
+- `0` fallback uses.
+- `302` deterministic field patches.
+
+Interpretation:
+
+V6 is the right base. It made the model far less scaffold-dependent than v5 while preserving final safety. The v7 corpus should not revert to the v5 shape, where final outputs looked good because scaffolding was doing too much. The v7 target is to keep v6's model-owned observation behavior and close the small source-card exactness gap.
+
+## V6 Failure Shape
+
+Configured-model competence failures:
+
+- `field_workflow_holdout_v1-000019`
+- `field_workflow_holdout_v1-000050`
+- `field_workflow_holdout_v1-000055`
+- `field_workflow_holdout_v1-000067`
+- `field_workflow_holdout_v1-000078`
+- `field_workflow_holdout_v1-000091`
+- `field_workflow_holdout_v1-000099`
+- `field_workflow_holdout_v1-000115`
+
+Expected-label failures:
+
+- `field_workflow_holdout_v1-000067`
+- `field_workflow_holdout_v1-000091`
+- `field_workflow_holdout_v1-000099`
+- `field_workflow_holdout_v1-000115`
+
+Source-card misses inside expected-label failures:
+
+- `field_workflow_holdout_v1-000067`: missing `REFERRAL-SBAR-v1` and `SAFETY-BOUNDARIES-v1`; actual source cards were `PREG-DANGER-SIGNS-v1`, `CHEST-PAIN-ESCALATION-v1`, and `FEVER-RED-FLAGS-v1`.
+- `field_workflow_holdout_v1-000091`: missing `REFERRAL-SBAR-v1` and `SAFETY-BOUNDARIES-v1`; actual source cards were `PREG-DANGER-SIGNS-v1`, `CHEST-PAIN-ESCALATION-v1`, and `FEVER-RED-FLAGS-v1`.
+- `field_workflow_holdout_v1-000099`: missing `SAFETY-BOUNDARIES-v1`; actual source cards were `PREG-DANGER-SIGNS-v1`, `CHEST-PAIN-ESCALATION-v1`, `FEVER-RED-FLAGS-v1`, and `REFERRAL-SBAR-v1`.
+- `field_workflow_holdout_v1-000115`: missing `REFERRAL-SBAR-v1` and `SAFETY-BOUNDARIES-v1`; actual source cards were `PREG-DANGER-SIGNS-v1`, `CHEST-PAIN-ESCALATION-v1`, and `FEVER-RED-FLAGS-v1`.
+
+Patch fields across the eight competence failures:
+
+- `source_cards`: `6`
+- `missing_info_to_collect`: `7`
+- `next_observations_to_collect`: `7`
+- `candidate_protocol_pathways`: `1`
+
+Diagnosis:
+
+The main v7 data need is not broad medical reasoning. It is harness-native closure behavior:
+
+- If the answer uses SBAR handoff structure, cite `REFERRAL-SBAR-v1`.
+- If the answer emits safety boundaries, forbidden-action constraints, or protocol-only disclaimers, cite `SAFETY-BOUNDARIES-v1`.
+- If multiple clinical protocol cards are relevant, do not let the support cards disappear.
+- If retrieval includes distractor protocol cards, cite the mandatory support cards without over-citing irrelevant clinical cards.
+- Preserve v6's model-owned observation fields while improving source-card closure in the same output.
+
+## Existing Corpora
+
+Canonical local files:
+
+- `data/finetune/figment_sft_v3.jsonl`: `3000` rows.
+- `data/finetune/figment_sft_v4.jsonl`: `1650` rows.
+- `data/finetune/figment_sft_v5.jsonl`: `1300` rows.
+- `data/finetune/figment_sft_v6_delta.jsonl`: `1430` rows.
+- `data/finetune/figment_sft_v6_replay.jsonl`: `570` rows.
+- `data/finetune/figment_sft_v6.jsonl`: `2000` rows.
+
+V6 merged corpus shape:
+
+- `1430` v6 delta rows.
+- `570` audited historical replay rows.
+- Replay source counts:
+  - `figment_sft_v3`: `330`
+  - `figment_sft_v4`: `120`
+  - `figment_sft_v5`: `120`
+- Task type counts:
+  - `navigator_full`: `1180`
+  - `focused_repair`: `820`
+- V6 verifier result: `2000` rows, `1413` case specs, `0` issues.
+
+V6 category counts:
+
+- `required_observation_ownership`: `879`
+- `observation_correction`: `218`
+- `v6_preservation`: `83`
+- `focused_repair:missing_observations`: `250`
+- `focused_repair:handoff_note_sbar`: `233`
+- `focused_repair:citations_and_pathways`: `163`
+- `focused_repair:protocol_urgency`: `87`
+- `focused_repair:schema`: `87`
+
+## Reuse Decision
+
+Do not fully regenerate v7 from scratch.
+
+Reuse v6 heavily because v6 is the first adapter that made model-owned harness behavior plausible. Blindly adding older corpora would risk reintroducing v5's failure mode, where the output passed only because deterministic scaffolding repaired hundreds of fields.
+
+Safe reuse:
+
+- Reuse all `570` rows from `data/finetune/figment_sft_v6_replay.jsonl`.
+- Reuse all `1430` rows from `data/finetune/figment_sft_v6_delta.jsonl`.
+- Prefer v6 delta rows that already passed source-card, observation, SBAR, urgency, schema, and forbidden-behavior checks.
+- Keep category balance so v7 does not overfit to source cards and forget required-observation ownership.
+
+Replay audit update:
+
+- `scripts/build_v7_replay_corpus.py` selected `2000` reusable rows.
+- Selected by source bucket:
+  - `figment_sft_v6_delta`: `1430`
+  - `figment_sft_v6_replay`: `570`
+- Selected by task type:
+  - `navigator_full`: `1180`
+  - `focused_repair`: `820`
+- Selected by category:
+  - `required_observation_ownership`: `879`
+  - `observation_correction`: `218`
+  - `v6_preservation`: `83`
+  - `focused_repair:missing_observations`: `250`
+  - `focused_repair:handoff_note_sbar`: `233`
+  - `focused_repair:citations_and_pathways`: `163`
+  - `focused_repair:protocol_urgency`: `87`
+  - `focused_repair:schema`: `87`
+
+Interpretation:
+
+The plan no longer needs `1200` newly generated rows. All v6 anchor rows pass the v7 replay gate, so v7 should preserve the full v6 behavior base and add a smaller, sharper source-card closure delta.
+
+Selective historical reuse:
+
+- Do not directly append v3, v4, or v5 rows beyond the already audited v6 replay rows.
+- If more historical diversity is needed, run the v7 replay selector against v3-v6 and select only rows passing the v7 policy.
+- Rejected v3-v5 rows may be used as teacher rewrite seeds, not as positive assistant targets.
+
+Hard non-reuse:
+
+- Do not train on exact `field_workflow_holdout_v1` examples as assistant targets.
+- Do not use rows where `source_cards` were correct only after deterministic patching.
+- Do not use rows with harness-owned metadata in responder observation fields.
+- Do not use rows with duplicated long `missing_info_to_collect` and `next_observations_to_collect`.
+- Do not use rows that teach arbitrary over-citation of every retrieved card.
+
+## V7 Target Corpus Shape
+
+Target total: `2800` rows.
+
+Reused anchor rows: `2000`.
+
+- `570` rows from `data/finetune/figment_sft_v6_replay.jsonl`.
+- `1430` rows from `data/finetune/figment_sft_v6_delta.jsonl`.
+
+New v7 delta rows: `800`.
+
+- `240` `source_card_closure` navigator rows.
+- `160` `focused_repair:source_card_closure` rows.
+- `140` `observation_source_joint` navigator rows.
+- `100` `distractor_card_resistance` navigator rows.
+- `80` `sbar_source_coupling` navigator rows.
+- `50` `source_card_negative_correction` focused-repair rows.
+- `30` `observation_patch_repair` focused-repair rows.
+
+Final v7 task type target:
+
+- `navigator_full`: about `1740` rows.
+- `focused_repair`: about `1060` rows.
+
+Final v7 source target:
+
+- `v6_delta_reuse`: `1430`
+- `v6_replay_reuse`: `570`
+- `v7_delta`: `800`
+
+## New Data Category Definitions
+
+### `source_card_closure`
+
+Purpose: Teach the model to include mandatory support cards in `source_cards` when the output relies on their content.
+
+Each row must include:
+
+- at least one clinical target card;
+- `SAFETY-BOUNDARIES-v1` whenever `safety_boundary` or `do_not_do` uses protocol-only, no-orders, no-diagnosis, no-treatment, or local-protocol language;
+- `REFERRAL-SBAR-v1` whenever `handoff_note_sbar` is present and actionable;
+- at least one multi-card scenario where clinical cards compete for attention;
+- complete source-card list in the assistant output before any deterministic repair.
+
+Oversample source-card combinations matching the v6 misses:
+
+- `PREG-DANGER-SIGNS-v1` plus `CHEST-PAIN-ESCALATION-v1` plus `FEVER-RED-FLAGS-v1` plus `SAFETY-BOUNDARIES-v1` plus `REFERRAL-SBAR-v1`.
+- The same clinical triad plus only one support-card distractor, requiring the teacher to add the missing support card.
+- Clinical card plus `SAFETY-BOUNDARIES-v1` plus `REFERRAL-SBAR-v1`, with irrelevant retrieved distractors excluded.
+
+### `focused_repair:source_card_closure`
+
+Purpose: Teach minimal repair behavior for source-card exactness without requiring a full navigator regeneration.
+
+Prompt shape:
+
+- Use the same focused-repair path as `focused_repair:citations_and_pathways`.
+- Input includes a flawed model output whose `source_cards` omit one or both support cards.
+- Assistant target returns only the repaired fields requested by the focused-repair prompt.
+
+Each accepted row must:
+
+- add missing mandatory source cards;
+- preserve valid existing clinical source cards;
+- avoid adding irrelevant retrieved cards;
+- preserve candidate pathway card IDs when they are already correct;
+- avoid changing urgency, handoff, observation, or safety text unless the focused-repair scope asks for it.
+
+### `observation_source_joint`
+
+Purpose: Preserve v6's observation gains while teaching source-card closure in the same full output.
+
+Each row must require:
+
+- valid `selected_required_observation_ids`;
+- responder-facing text for every selected ID;
+- non-identical `missing_info_to_collect` and `next_observations_to_collect`;
+- complete `source_cards` including relevant clinical and support cards;
+- SBAR handoff that cites or depends on `REFERRAL-SBAR-v1`;
+- safety boundary that cites or depends on `SAFETY-BOUNDARIES-v1`.
+
+### `distractor_card_resistance`
+
+Purpose: Prevent v7 from solving closure by over-citing every retrieved card.
+
+Each row must include:
+
+- retrieved cards containing at least one irrelevant clinical protocol;
+- expected source cards that include the target clinical card and mandatory support cards;
+- explicit exclusion of irrelevant retrieved card IDs from `source_cards`;
+- normal final validation behavior.
+
+The verifier must reject rows where the teacher adds every retrieved card to `source_cards` without justification.
+
+### `sbar_source_coupling`
+
+Purpose: Make the SBAR card load-bearing when the answer includes SBAR structure.
+
+Each row must include:
+
+- an SBAR handoff with situation, background, objective assessment observations, and request;
+- `REFERRAL-SBAR-v1` in `source_cards`;
+- no unsupported facts in the SBAR assessment;
+- no treatment orders, dosing, or autonomous disposition.
+
+### `source_card_negative_correction`
+
+Purpose: Use bad source-card outputs as correction seeds.
+
+Input flaws to seed:
+
+- missing `SAFETY-BOUNDARIES-v1`;
+- missing `REFERRAL-SBAR-v1`;
+- missing both support cards;
+- over-citing unrelated clinical cards;
+- replacing the target clinical card with a support card;
+- placing source-card IDs in `missing_info_to_collect` or `next_observations_to_collect`.
+
+Assistant target:
+
+- a corrected focused-repair JSON answer that fixes source-card fields only.
+
+### `observation_patch_repair`
+
+Purpose: Keep pressure on the remaining v6 observation patch cases without rebuilding the whole corpus around observations.
+
+Input flaws to seed:
+
+- missing selected required-observation IDs;
+- selected IDs not visible in observation text;
+- duplicate long missing and next observation lists;
+- generic filler such as `monitor closely` as a standalone observation;
+- known observations repeated as missing.
+
+Assistant target:
+
+- corrected observation fields in the exact focused-repair prompt shape.
+
+### `v7_preservation`
+
+Purpose: Guard against regressions in the behavior already working in v6.
+
+Rows should cover:
+
+- red-flag urgency floors;
+- protocol-only safety boundaries;
+- no diagnosis or treatment instructions;
+- noisy/radio-style intake;
+- rural clinic and disaster first-response constraints;
+- short SBAR handoffs with objective observations only;
+- target clinical card retained in both candidate pathways and source cards.
+
+## V7 Gold Output Policy
+
+For full navigator rows, accepted assistant output must:
+
+- emit complete navigator JSON in the current Figment shape;
+- include valid `candidate_protocol_pathways`;
+- include complete `source_cards` with all cards used by the output;
+- include `SAFETY-BOUNDARIES-v1` when safety boundary or forbidden-action content is present;
+- include `REFERRAL-SBAR-v1` when SBAR handoff structure is present;
+- include the target protocol card in `source_cards`;
+- include any fired deterministic rule card in `source_cards`;
+- optionally include trace-only `selected_required_observation_ids`;
+- select required observation IDs only from `required_observation_targets`;
+- include every metadata-required ID listed in `must_include_selected_required_observation_ids`;
+- express every selected required-observation ID as recognizable responder-facing text;
+- keep `next_observations_to_collect` as a prioritized subset or near-subset of `missing_info_to_collect`, not a full copy;
+- preserve protocol-only, no-treatment, no-diagnosis, and no-autonomous-disposition boundaries.
+
+For focused repair rows, accepted assistant output must:
+
+- return only the fields requested by the focused-repair prompt;
+- repair the targeted failure;
+- preserve valid existing fields;
+- avoid expanding into a full navigator answer;
+- avoid visible reasoning tags, teacher notes, or commentary.
+
+## V7 Replay Policy
+
+Use the v6 replay policy as the base and add source-card closure checks.
+
+Hard rejects:
+
+- Any row failing the existing v6 policy in `scripts/build_v6_replay_corpus.py`.
+- Any row with `metadata.expected_label_score.all_expected_labels_passed == false`.
+- Any row where `metadata.validation_result.passed == false`.
+- Any row where source-card provenance says deterministic fallback authored `source_cards`.
+- Any full navigator row with `handoff_note_sbar` present and `REFERRAL-SBAR-v1` absent from `source_cards`.
+- Any full navigator row with safety-boundary text present and `SAFETY-BOUNDARIES-v1` absent from `source_cards`.
+- Any full navigator row where the target protocol card is absent from `source_cards`.
+- Any row that adds all retrieved cards when irrelevant distractors are marked in metadata.
+- Any row overlapping locked eval signatures from:
+  - `data/eval/field_workflow_holdout_v1.jsonl`
+  - `data/eval/adversarial_strict_cases.jsonl`
+  - `data/eval/comprehensive_hosted_cases.jsonl`
+  - `data/eval/initial_handwritten_cases.jsonl`
+
+Soft preferences:
+
+- Prefer v6 rows where all scored fields were `model_raw`.
+- Prefer rows with source-card lists of length `3` to `5`.
+- Prefer multi-card cases with one target clinical card plus support cards.
+- Prefer concise observation fields and short SBAR handoffs.
+- Prefer rows with no deterministic patches in their originating eval metadata.
+
+## Teacher Generation Strategy
+
+Teacher model:
+
+- Primary: `nvidia/nemotron-3-ultra-550b-a55b:free` through the existing OpenRouter endpoint.
+- Alternate: `nvidia/nemotron-3-ultra-550b-a55b` through the NVIDIA-compatible endpoint if quota and reliability are healthy.
+
+Teacher prompt requirements:
+
+- Build prompts through the existing Figment case/preparation path, not a generic clinical note format.
+- Include the exact current navigator JSON schema.
+- Include retrieved card IDs and card summaries.
+- Include required-observation targets for full navigator rows.
+- Include closure rules for `SAFETY-BOUNDARIES-v1` and `REFERRAL-SBAR-v1`.
+- Include distractor-card instructions where relevant.
+- Tell the teacher to produce only JSON.
+
+Generation must use near-neighbor variants of failures, not copied holdout cases.
+
+Variant knobs:
+
+- rural clinic versus disaster first-response setting;
+- adult, pediatric, pregnant, fever, chest pain, respiratory distress, stroke, wound infection, and altered mental status presentations;
+- partial vitals, noisy intake, radio-style handoff, missing transport availability, language uncertainty, and power/network constraints;
+- retrieved card order permutations;
+- support cards placed first, middle, or last in retrieval context;
+- irrelevant clinical distractors included but not cited.
+
+## File Map
+
+Create:
+
+- `docs/local_4b_v7_training_corpora_plan.md`: this plan.
+- `scripts/summarize_v7_corpus_needs.py`: extracts v6 failure IDs, source-card misses, deterministic patch fields, and source-card closure combinations.
+- `scripts/build_v7_replay_corpus.py`: selects clean v6 and historical replay rows under the v7 policy.
+- `scripts/generate_v7_full_corpus.py`: generates v7 delta rows, verifies them, and prepares Modal train/validation splits.
+- `scripts/merge_v7_training_corpus.py`: merges v7 delta rows with v7 replay rows and writes manifest plus Modal split artifacts.
+- `tests/test_finetune_v7_data_plan.py`: verifies v7 category counts, replay policy, closure rules, and wrapper defaults.
+- `data/finetune/figment_sft_v7_replay.jsonl`: selected reusable rows.
+- `data/finetune/figment_sft_v7_replay_manifest.json`: replay selection evidence.
+- `data/finetune/figment_sft_v7_delta.jsonl`: newly generated v7 rows.
+- `data/finetune/figment_sft_v7_delta_case_specs.jsonl`: case specs for new v7 navigator rows.
+- `data/finetune/figment_sft_v7_delta_manifest.json`: v7 delta generation evidence.
+- `data/finetune/figment_sft_v7.jsonl`: final merged corpus.
+- `data/finetune/figment_sft_v7_case_specs.jsonl`: merged case specs.
+- `data/finetune/figment_sft_v7_manifest.json`: final corpus manifest.
+- `data/finetune/modal/figment_sft_v7/train.jsonl`: Modal training split.
+- `data/finetune/modal/figment_sft_v7/validation.jsonl`: Modal validation split.
+- `data/finetune/modal/figment_sft_v7/manifest.json`: Modal split manifest.
+
+Modify:
+
+- `scripts/generate_finetune_data.py`: add v7 failure classes, v7 scoring checks, and source-card closure policy.
+- `scripts/augment_finetune_repair_rows.py`: add `source_card_closure` and `observation_patch_repair` repair scopes.
+- `scripts/verify_finetune_harness_alignment.py`: add v7 closure verifier checks.
+- `scripts/prepare_modal_finetune_dataset.py`: no behavioral change expected; use existing split command and verify group balance.
+
+## Implementation Tasks
+
+### Task 1: Summarize V6 Corpus Needs
+
+**Files:**
+
+- Create: `scripts/summarize_v7_corpus_needs.py`
+- Read: `traces/figment_sft_v6_field_workflow_holdout_modal_gpu_20260611_h100_gguf/local_4b_eval.jsonl`
+- Write: `traces/figment_sft_v6_field_workflow_holdout_modal_gpu_20260611_h100_gguf/v7_corpus_needs_summary.json`
+
+- [ ] Add a script that reads the v6 eval JSONL and writes:
+  - `total_cases`
+  - `competence_failure_case_ids`
+  - `expected_label_failure_case_ids`
+  - `expected_label_check_failures`
+  - `missing_source_card_ids_by_case`
+  - `deterministic_patch_fields_by_case`
+  - `deterministic_patch_field_counts`
+  - `actual_source_card_sets_for_failures`
+
+- [ ] Run:
+
+```bash
+PYTHONPATH=. .venv/bin/python scripts/summarize_v7_corpus_needs.py \
+  --eval-jsonl traces/figment_sft_v6_field_workflow_holdout_modal_gpu_20260611_h100_gguf/local_4b_eval.jsonl \
+  --output traces/figment_sft_v6_field_workflow_holdout_modal_gpu_20260611_h100_gguf/v7_corpus_needs_summary.json
+```
+
+Expected:
+
+- `total_cases` is `150`.
+- `competence_failure_case_ids` has `8` items.
+- `expected_label_failure_case_ids` has `4` items.
+- `missing_source_card_ids_by_case` includes `SAFETY-BOUNDARIES-v1` and `REFERRAL-SBAR-v1`.
+- `deterministic_patch_field_counts.source_cards` is `6`.
+
+### Task 2: Add V7 Source-Card Closure Policy
+
+**Files:**
+
+- Modify: `scripts/generate_finetune_data.py`
+- Modify: `scripts/verify_finetune_harness_alignment.py`
+- Test: `tests/test_finetune_v7_data_plan.py`
+
+- [ ] Add a helper in `scripts/generate_finetune_data.py`:
+
+```python
+def v7_source_card_closure_issues(output: dict, *, target_protocol_card_id: str | None = None) -> list[str]:
+    issues: list[str] = []
+    source_cards = {str(card_id) for card_id in output.get("source_cards") or []}
+    if target_protocol_card_id and target_protocol_card_id not in source_cards:
+        issues.append(f"missing_target_source_card:{target_protocol_card_id}")
+    if output.get("handoff_note_sbar") and "REFERRAL-SBAR-v1" not in source_cards:
+        issues.append("missing_referral_sbar_source_card")
+    safety_text = json.dumps(
+        {
+            "safety_boundary": output.get("safety_boundary"),
+            "do_not_do": output.get("do_not_do"),
+            "responder_plain_language_script": output.get("responder_plain_language_script"),
+        },
+        sort_keys=True,
+    ).lower()
+    safety_terms = (
+        "local protocol",
+        "do not diagnose",
+        "do not provide clinical orders",
+        "do not provide treatment instructions",
+        "safety boundary",
+    )
+    if any(term in safety_text for term in safety_terms) and "SAFETY-BOUNDARIES-v1" not in source_cards:
+        issues.append("missing_safety_boundaries_source_card")
+    return issues
+```
+
+- [ ] Wire the helper into the v7 candidate scorer so v7 rows are rejected on any closure issue.
+
+- [ ] Add verifier checks that report issue types:
+  - `v7_missing_referral_sbar_source_card`
+  - `v7_missing_safety_boundaries_source_card`
+  - `v7_missing_target_source_card`
+
+- [ ] Add tests that construct one row per failure type and assert the verifier rejects them.
+
+Run:
+
+```bash
+PYTHONPATH=. .venv/bin/pytest tests/test_finetune_v7_data_plan.py tests/test_finetune_v6_data_plan.py -q
+```
+
+Expected:
+
+- v7 tests pass.
+- v6 tests still pass.
+
+### Task 3: Build V7 Replay Selector
+
+**Files:**
+
+- Create: `scripts/build_v7_replay_corpus.py`
+- Test: `tests/test_finetune_v7_data_plan.py`
+- Write: `data/finetune/figment_sft_v7_replay.jsonl`
+- Write: `data/finetune/figment_sft_v7_replay_manifest.json`
+
+- [ ] Start from `scripts/build_v6_replay_corpus.py`.
+
+- [ ] Set default inputs:
+
+```python
+DEFAULT_INPUTS = [
+    Path("data/finetune/figment_sft_v6_delta.jsonl"),
+    Path("data/finetune/figment_sft_v6_replay.jsonl"),
+    Path("data/finetune/figment_sft_v5.jsonl"),
+    Path("data/finetune/figment_sft_v4.jsonl"),
+    Path("data/finetune/figment_sft_v3.jsonl"),
+]
+```
+
+- [ ] Set default targets:
+
+```python
+DEFAULT_TARGETS = {
+    "figment_sft_v6_delta": 1430,
+    "figment_sft_v6_replay": 570,
+    "figment_sft_v5": 0,
+    "figment_sft_v4": 0,
+    "figment_sft_v3": 0,
+}
+```
+
+- [ ] Apply v6 replay policy first, then apply v7 closure policy.
+
+- [ ] Annotate selected rows with:
+
+```json
+{
+  "v7_replay_audit": {
+    "source_dataset_version": "figment_sft_v6_delta",
+    "policy_version": 1,
+    "accepted": true
+  }
+}
+```
+
+- [ ] Run:
+
+```bash
+PYTHONPATH=. .venv/bin/python scripts/build_v7_replay_corpus.py
+```
+
+Expected:
+
+- `data/finetune/figment_sft_v7_replay.jsonl` has `2000` rows.
+- Manifest `selected_by_source_bucket` includes `figment_sft_v6_delta: 1430` and `figment_sft_v6_replay: 570`.
+- Manifest includes nonzero rejected counts for v7 closure issues if any candidate rows fail closure.
+
+### Task 4: Add V7 Generation Categories
+
+**Files:**
+
+- Modify: `scripts/generate_finetune_data.py`
+- Modify: `scripts/augment_finetune_repair_rows.py`
+- Test: `tests/test_finetune_v7_data_plan.py`
+
+- [ ] Add v7 navigator counts:
+
+```python
+V7_NAVIGATOR_COUNTS = {
+    "source_card_closure": 240,
+    "observation_source_joint": 140,
+    "distractor_card_resistance": 100,
+    "sbar_source_coupling": 80,
+}
+```
+
+- [ ] Add v7 repair counts:
+
+```python
+V7_REPAIR_COUNTS = {
+    "source_card_closure": 160,
+    "source_card_negative_correction": 50,
+    "observation_patch_repair": 30,
+}
+```
+
+- [ ] Add `_failure_class_for_index(..., dataset_version="figment_sft_v7_delta")` coverage so the first `20` rows interleave all v7 navigator categories.
+
+- [ ] Add repair scope scheduling so `figment_sft_v7_delta` produces:
+
+```python
+{
+    "source_card_closure": 160,
+    "source_card_negative_correction": 50,
+    "observation_patch_repair": 30,
+}
+```
+
+- [ ] Add tests asserting the category counts exactly match the constants.
+
+Run:
+
+```bash
+PYTHONPATH=. .venv/bin/pytest tests/test_finetune_v7_data_plan.py tests/test_finetune_v6_data_plan.py -q
+```
+
+Expected:
+
+- v7 category-count tests pass.
+- v6 category-count tests still pass.
+
+### Task 5: Add V7 Full-Corpus Wrapper
+
+**Files:**
+
+- Create: `scripts/generate_v7_full_corpus.py`
+- Test: `tests/test_finetune_v7_data_plan.py`
+
+- [ ] Create a wrapper patterned after `scripts/generate_v6_full_corpus.py`.
+
+- [ ] Pin defaults:
+
+```python
+DEFAULT_OUTPUT_VERSION = "figment_sft_v7_delta"
+DEFAULT_TEACHER_MODEL_ID = "nvidia/nemotron-3-ultra-550b-a55b:free"
+DEFAULT_NAVIGATOR_COUNT = 560
+DEFAULT_REPAIR_COUNT = 240
+DEFAULT_BASE_START_INDEX = "80000"
+DEFAULT_SHARD_PREFIX = "data/finetune/shards/figment_sft_v7_delta_full_shard"
+```
+
+- [ ] Ensure the default output paths are:
+
+```text
+data/finetune/figment_sft_v7_delta.jsonl
+data/finetune/figment_sft_v7_delta_case_specs.jsonl
+data/finetune/figment_sft_v7_delta_manifest.json
+data/finetune/modal/figment_sft_v7_delta
+```
+
+- [ ] Add a wrapper test that calls:
+
+```python
+args = build_corpus_args(["--navigator-count", "5", "--repair-count", "3", "--dry-run"])
+```
+
+and asserts:
+
+- dataset version is `figment_sft_v7_delta`;
+- teacher model is `nvidia/nemotron-3-ultra-550b-a55b:free`;
+- output path is `data/finetune/figment_sft_v7_delta.jsonl`;
+- dry-run flag is preserved.
+
+Run:
+
+```bash
+PYTHONPATH=. .venv/bin/pytest tests/test_finetune_v7_data_plan.py -q
+```
+
+Expected:
+
+- wrapper defaults are pinned by tests.
+
+### Task 6: Smoke Generate V7 Delta
+
+**Files:**
+
+- Write: `/tmp/figment_v7_smoke/figment_sft_v7_delta.jsonl`
+- Write: `/tmp/figment_v7_smoke/figment_sft_v7_delta_case_specs.jsonl`
+- Write: `/tmp/figment_v7_smoke/figment_sft_v7_delta_manifest.json`
+
+- [ ] Run a deterministic dry-run smoke:
+
+```bash
+PYTHONPATH=. .venv/bin/python scripts/generate_v7_full_corpus.py \
+  --navigator-count 8 \
+  --repair-count 4 \
+  --rows-per-shard 2 \
+  --parallelism 1 \
+  --base-start-index 88000 \
+  --shard-prefix /tmp/figment_v7_smoke/shard \
+  --output /tmp/figment_v7_smoke/figment_sft_v7_delta.jsonl \
+  --case-specs /tmp/figment_v7_smoke/figment_sft_v7_delta_case_specs.jsonl \
+  --manifest /tmp/figment_v7_smoke/figment_sft_v7_delta_manifest.json \
+  --modal-output-dir /tmp/figment_v7_smoke/modal \
+  --dry-run
+```
+
+Expected:
+
+- `12` total rows.
+- At least one row from each v7 navigator category appears in the manifest or smoke distribution.
+- Harness verifier reports `issue_count=0`.
+
+- [ ] Run a real-teacher smoke:
+
+```bash
+PYTHONPATH=. .venv/bin/python scripts/generate_v7_full_corpus.py \
+  --navigator-count 8 \
+  --repair-count 4 \
+  --rows-per-shard 2 \
+  --parallelism 1 \
+  --teacher-error-retries 3 \
+  --teacher-error-sleep-seconds 10 \
+  --base-start-index 88100 \
+  --shard-prefix /tmp/figment_v7_teacher_smoke/shard \
+  --output /tmp/figment_v7_teacher_smoke/figment_sft_v7_delta.jsonl \
+  --case-specs /tmp/figment_v7_teacher_smoke/figment_sft_v7_delta_case_specs.jsonl \
+  --manifest /tmp/figment_v7_teacher_smoke/figment_sft_v7_delta_manifest.json \
+  --modal-output-dir /tmp/figment_v7_teacher_smoke/modal \
+  --log-rejections
+```
+
+Expected:
+
+- `12` accepted rows.
+- Verifier reports `issue_count=0`.
+- Teacher rejection reasons, if any, are source-card closure, JSON parsing, or policy rejections recorded in the manifest.
+
+### Task 7: Generate Full V7 Delta
+
+**Files:**
+
+- Write: `data/finetune/figment_sft_v7_delta.jsonl`
+- Write: `data/finetune/figment_sft_v7_delta_case_specs.jsonl`
+- Write: `data/finetune/figment_sft_v7_delta_manifest.json`
+- Write: `data/finetune/modal/figment_sft_v7_delta/train.jsonl`
+- Write: `data/finetune/modal/figment_sft_v7_delta/validation.jsonl`
+- Write: `data/finetune/modal/figment_sft_v7_delta/manifest.json`
+
+- [ ] Run:
+
+```bash
+PYTHONPATH=. .venv/bin/python scripts/generate_v7_full_corpus.py \
+  --parallelism 4 \
+  --teacher-error-retries 3 \
+  --teacher-error-sleep-seconds 10 \
+  --log-rejections
+```
+
+Expected:
+
+- `data/finetune/figment_sft_v7_delta.jsonl` has `800` rows.
+- `data/finetune/figment_sft_v7_delta_case_specs.jsonl` has case specs for all navigator rows.
+- Manifest category counts match the v7 target.
+- Harness verifier reports `issue_count=0`.
+
+### Task 8: Merge V7 Corpus
+
+**Files:**
+
+- Create: `scripts/merge_v7_training_corpus.py`
+- Read: `data/finetune/figment_sft_v7_delta.jsonl`
+- Read: `data/finetune/figment_sft_v7_replay.jsonl`
+- Write: `data/finetune/figment_sft_v7.jsonl`
+- Write: `data/finetune/figment_sft_v7_case_specs.jsonl`
+- Write: `data/finetune/figment_sft_v7_manifest.json`
+- Write: `data/finetune/modal/figment_sft_v7/train.jsonl`
+- Write: `data/finetune/modal/figment_sft_v7/validation.jsonl`
+- Write: `data/finetune/modal/figment_sft_v7/manifest.json`
+
+- [ ] Create merge logic patterned after `scripts/merge_v6_training_corpus.py`.
+
+- [ ] Set defaults:
+
+```python
+DEFAULT_DELTA = Path("data/finetune/figment_sft_v7_delta.jsonl")
+DEFAULT_DELTA_CASE_SPECS = Path("data/finetune/figment_sft_v7_delta_case_specs.jsonl")
+DEFAULT_REPLAY = Path("data/finetune/figment_sft_v7_replay.jsonl")
+DEFAULT_OUTPUT = Path("data/finetune/figment_sft_v7.jsonl")
+DEFAULT_CASE_SPECS = Path("data/finetune/figment_sft_v7_case_specs.jsonl")
+DEFAULT_MANIFEST = Path("data/finetune/figment_sft_v7_manifest.json")
+DEFAULT_MODAL_DIR = Path("data/finetune/modal/figment_sft_v7")
+```
+
+- [ ] Run:
+
+```bash
+PYTHONPATH=. .venv/bin/python scripts/merge_v7_training_corpus.py
+```
+
+Expected:
+
+- `data/finetune/figment_sft_v7.jsonl` has `2800` rows.
+- Manifest records `delta_rows=800`.
+- Manifest records `replay_rows=2000`.
+- Harness verifier reports `issue_count=0`.
+- Modal split has `2520` train rows and `280` validation rows.
+
+### Task 9: Final Verification
+
+**Files:**
+
+- Read: `data/finetune/figment_sft_v7.jsonl`
+- Read: `data/finetune/figment_sft_v7_case_specs.jsonl`
+- Read: `data/finetune/figment_sft_v7_manifest.json`
+- Read: `data/finetune/modal/figment_sft_v7/manifest.json`
+
+- [ ] Count rows:
+
+```bash
+wc -l data/finetune/figment_sft_v7.jsonl \
+  data/finetune/figment_sft_v7_delta.jsonl \
+  data/finetune/figment_sft_v7_replay.jsonl \
+  data/finetune/modal/figment_sft_v7/train.jsonl \
+  data/finetune/modal/figment_sft_v7/validation.jsonl
+```
+
+Expected:
+
+- `2800 data/finetune/figment_sft_v7.jsonl`
+- `800 data/finetune/figment_sft_v7_delta.jsonl`
+- `2000 data/finetune/figment_sft_v7_replay.jsonl`
+- `2520 data/finetune/modal/figment_sft_v7/train.jsonl`
+- `280 data/finetune/modal/figment_sft_v7/validation.jsonl`
+
+- [ ] Run verifier:
+
+```bash
+PYTHONPATH=. .venv/bin/python scripts/verify_finetune_harness_alignment.py \
+  --dataset data/finetune/figment_sft_v7.jsonl \
+  --case-specs data/finetune/figment_sft_v7_case_specs.jsonl
+```
+
+Expected:
+
+- `passed` is `true`.
+- `issue_count` is `0`.
+- `rows` is `2800`.
+
+- [ ] Run tests:
+
+```bash
+PYTHONPATH=. .venv/bin/pytest \
+  tests/test_finetune_v7_data_plan.py \
+  tests/test_finetune_v6_data_plan.py \
+  tests/test_v6_replay_selection.py \
+  tests/test_modal_finetune_prep.py \
+  -q
+```
+
+Expected:
+
+- all tests pass.
+
+## Training Readiness Gates
+
+Do not launch v7 training until all gates pass:
+
+- `data/finetune/figment_sft_v7.jsonl` exists and has `2800` rows.
+- `data/finetune/figment_sft_v7_manifest.json` exists.
+- `data/finetune/modal/figment_sft_v7/train.jsonl` has `2520` rows.
+- `data/finetune/modal/figment_sft_v7/validation.jsonl` has `280` rows.
+- Harness alignment verifier reports `issue_count=0`.
+- V7 tests pass.
+- The manifest shows nonzero counts for:
+  - `source_card_closure`
+  - `focused_repair:source_card_closure`
+  - `observation_source_joint`
+  - `distractor_card_resistance`
+  - `sbar_source_coupling`
+- The manifest shows the corpus includes all `570` v6 replay rows and all `1430` v6 delta rows.
+- No holdout eval case IDs appear as training `case_id` values.
+
+## Post-Training Acceptance Gates
+
+Run v7 against `data/eval/field_workflow_holdout_v1.jsonl` on Modal GPU using the same harness/scoring path as v5 and v6.
+
+V7 should beat v6 if:
+
+- `final_validation_successes == 150`
+- `fallback_uses == 0`
+- `expected_label_successes == 150`
+- `expected_label_check_failures.expected_source_cards_present == 0`
+- `competence_successes >= 146`
+- `deterministic_patch_count <= 15`
+- `field_provenance_by_field.source_cards.deterministic_fallback <= 1`
+- `field_provenance_by_field.missing_info_to_collect.deterministic_fallback <= 5`
+- `field_provenance_by_field.next_observations_to_collect.deterministic_fallback <= 5`
+- `handoff_unsupported_fact_total == 0`
+
+V7 should be rejected or rolled into a v8 data plan if:
+
+- expected-label success stays below `150/150`;
+- source-card deterministic fallback remains above `1/150`;
+- observation deterministic fallback rises above v6's `7/150` for either observation field;
+- final validation fails on any case;
+- fallback uses become nonzero;
+- source-card closure improves only by over-citing irrelevant retrieved cards.
+
+## Notes For Training
+
+Recommended training posture:
+
+- Start from the v6 adapter, not from base, because v6 already internalized most of the harness behavior.
+- Use H100 for the full run because the user has already chosen faster Modal iteration.
+- Keep the learning rate conservative relative to v6 if the trainer supports it, because v7 is a targeted continuation rather than a broad skill rebuild.
+- After training, upload the merged BF16 artifact to the existing Hugging Face archive path for v7.
+- Evaluate on Modal as a detached batch job that exits after writing artifacts.
+
+Suggested output names:
+
+- Dataset version: `figment_sft_v7`
+- Adapter output name: `figment-sft-v7-lora`
+- Merged output name: `figment-sft-v7-lora-merged-bf16`
+- Modal checkpoint path: `figment-checkpoints:/figment_sft_v7/figment-sft-v7-lora`
+- Modal merged path: `figment-checkpoints:/figment_sft_v7/figment-sft-v7-lora-merged-bf16`
+- Eval result name: `figment_sft_v7_field_workflow_holdout_modal_gpu_20260611_h100_gguf`
+
+## Self-Review Checklist
+
+- [ ] The plan reuses v6 heavily instead of regenerating the entire corpus.
+- [ ] The plan does not directly train on holdout eval examples.
+- [ ] The new data categories target the actual v6 failures.
+- [ ] The replay policy blocks v5-style scaffold dependence.
+- [ ] The corpus shape preserves observation ownership instead of over-rotating on source cards.
+- [ ] The final gates can prove whether v7 improved over v6.

docs/local_parakeet_asr_evidence.md

@@ -0,0 +1,56 @@
+# Local Parakeet ASR evidence
+
+Date: 2026-06-07
+
+This note separates local Parakeet ASR artifact availability from a real local ASR proof. Artifact availability is not proof that the app has used Parakeet audio or that the local/off-grid audio path is ready.
+
+## Artifact status
+
+- Model repo: `nvidia/parakeet-rnnt-1.1b`
+- Revision: `a07b19e98a26c1873a3f2622c446a4a1ca6316cb`
+- Local snapshot path: `/Users/drake.thomsen/.cache/huggingface/hub/models--nvidia--parakeet-rnnt-1.1b/snapshots/a07b19e98a26c1873a3f2622c446a4a1ca6316cb`
+- Local artifact: `parakeet-rnnt-1.1b.nemo`
+- Artifact size: `4283105280` bytes
+- Artifact SHA-256: `535896f014953d945b287ac533560e20da8103c6781b152de4645528e2b60738`
+
+## Evidence helper
+
+Use the helper to capture local ASR evidence:
+
+```bash
+PYTHON_DOTENV_DISABLED=true \
+python3 scripts/run_local_asr_evidence.py \
+  --provider-payload <local-parakeet-provider-output.json> \
+  --audio <optional-source-audio.wav> \
+  --provider-note "<device/runtime command>"
+```
+
+The helper writes a timestamped evidence directory under `traces/local_asr_parakeet_evidence_*`:
+
+- `artifact_metadata.json`: Parakeet `.nemo` presence, size, and hash
+- `audio_metadata.json`: optional source-audio metadata and hash only; raw audio is not copied into the evidence bundle
+- `provider_payload_metadata.json`: provider output file hash
+- `audio_draft.json`: Figment draft generated from the provider payload under `AUDIO_BACKEND=parakeet_nemo` and `ALLOW_LOCAL_ASR=true`
+- `draft_checks.json`: evidence-gated checks for Parakeet provenance, confirmation status, and raw-audio handling
+- `asr_evidence_manifest.json`: compact manifest with artifact status, provider-payload hash, configured route, draft summary, raw-audio handling, and proof flags
+- `summary.json`: top-level proof flags
+
+Exit codes are evidence-gated:
+
+- `0`: provider payload passed all local ASR draft checks
+- `2`: artifact exists but provider payload is missing or not proof
+- `1`: artifact missing
+
+## Current result
+
+The artifact-only helper run succeeded in finding the Parakeet `.nemo` file but exited `2` because no real local ASR provider payload was supplied:
+
+```text
+status=artifact_present_provider_payload_required
+counts_as_local_asr_artifact=true
+counts_as_local_asr_proof=false
+raw_audio_stored=false
+asr_evidence_manifest_path=/tmp/figment-local-asr-artifact-only-manifest-check/asr_evidence_manifest.json
+```
+
+This means Parakeet remains not demo-visible as local ASR proof. It can become proof only after a real local ASR adapter or device runtime produces a provider payload, and the helper records `counts_as_local_asr_proof=true`.

docs/superpowers/plans/2026-06-10-figment-build-small-blog.md

@@ -0,0 +1,90 @@
+# Figment Build Small Blog Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Create a rough judge-facing Build Small blog post about what Figment taught me.
+
+**Architecture:** The deliverable is a single markdown draft under `docs/`, based on the approved design spec. It should use evidence from project docs, prior thread summaries, git history, Modal metadata, and eval traces while keeping the tone public and readable.
+
+**Tech Stack:** Markdown documentation in the existing Figment repo.
+
+---
+
+### Task 1: Draft The Blog Post
+
+**Files:**
+- Create: `docs/figment-build-small-lessons-draft.md`
+- Reference: `docs/superpowers/specs/2026-06-10-figment-build-small-blog-design.md`
+
+- [x] **Step 1: Create the draft markdown file**
+
+Write a rough post with these sections:
+
+```markdown
+# Building Figment For Build Small: What I Learned About Making Small Models Useful
+
+Opening: Figment is a protocol navigator for field responders, not an AI doctor or autonomous clinical decision tool.
+
+## Audio Should Draft, Not Decide
+
+## Deterministic Safety Rules Are The Floor
+
+## App Safety And Model Competence Are Different Numbers
+
+## Field-Level Provenance Changed My Relationship With Fallback
+
+## Fine-Tuning Only Helped After The Eval Got More Honest
+
+## What Still Is Not Good Enough
+
+## What I Would Build Next
+
+## The Lesson I Am Taking From Build Small
+```
+
+- [x] **Step 2: Include evidence anchors**
+
+Use these exact evidence points where they naturally fit:
+
+- Hosted Omni baseline: `28/50` model competence, `50/50` final validation.
+- Hosted load-bearing follow-up: `31/50` whole-output competence, `480/650` model-retained fields.
+- Local v1 pilot: proved the train/merge/convert/serve/eval loop but regressed to `11/50`.
+- Local v2: `33/50` on the locked 50-case eval.
+- Local v3 holdout: `107/150` competence, `148/150` final validation.
+- V3 weakness: `REFERRAL-SBAR-v1` `0/27`, `radio_handoff` `0/16`, `sbar_handoff_usefulness` `0/10`.
+- Modal v3: `700/700` steps, final `eval_loss=0.04357146`, final `train_loss=0.60960097`.
+
+- [x] **Step 3: Avoid unsafe or unsupported claims**
+
+Check the draft does not claim:
+
+- clinical validation,
+- diagnosis or treatment,
+- autonomous triage,
+- complete local/off-grid proof,
+- final validation as pure model competence,
+- v3 as solved handoff usefulness.
+
+- [x] **Step 4: Self-edit for public readability**
+
+Read the full draft once and tighten:
+
+- make section openings concrete,
+- remove repeated metrics,
+- keep caveats readable,
+- preserve the "systems of restraint" closing.
+
+- [x] **Step 5: Verify and commit**
+
+Run:
+
+```bash
+rg -n "diagnose|prescribe|clinical validation|autonomous triage|off-grid proof|TODO|TBD" docs/figment-build-small-lessons-draft.md
+git diff --check
+```
+
+Expected:
+
+- Any medical-risk phrases appear only as disclaimers or non-goals.
+- No placeholder text.
+- `git diff --check` prints no output.

docs/superpowers/specs/2026-06-10-figment-build-small-blog-design.md

@@ -0,0 +1,83 @@
+# Figment Build Small Blog Post Design
+
+Date: 2026-06-10
+
+## Purpose
+
+Write a rough blog post about what I learned while building Figment for the Hugging Face Build Small Hackathon.
+
+The post should primarily serve judges and public project readers, while still being useful to other builders. It should be candid about failed evals, fallback, training iteration, and remaining weaknesses, but should not read like an internal debugging diary.
+
+## Audience
+
+Primary:
+
+- Hugging Face Build Small judges.
+- Public readers who want to understand what Figment proves and what it does not yet prove.
+
+Secondary:
+
+- Hackathon builders and AI engineers interested in small-model product design.
+
+## Tone
+
+Use earned candor:
+
+- Be honest about the messy parts.
+- Separate app safety from model competence.
+- Keep the center of gravity on what was learned and why the project improved.
+- Avoid overclaiming medical, local/off-grid, or fine-tuning success.
+
+The voice should sound like a reflective builder: technical, grounded, and specific, but still readable as a public blog post.
+
+## Recommended Angle
+
+Use a hybrid of:
+
+1. The field tool story.
+   - Figment is a protocol navigator for rural clinics, mobile units, and disaster response contexts.
+   - It is not an AI doctor.
+
+2. The small models need systems story.
+   - Small models become useful when surrounded by scope, contracts, validators, provenance, traces, and honest evals.
+
+Use the evaluation/training evidence as proof points throughout rather than opening with a wall of metrics.
+
+## Proposed Structure
+
+1. Set the scene: why Figment exists and why the Build Small constraint mattered.
+2. Lesson 1: audio should draft, not decide.
+3. Lesson 2: deterministic safety rules are not a fallback; they are the floor.
+4. Lesson 3: model competence and app safety are different numbers.
+5. Lesson 4: field-level provenance changed how fallback felt.
+6. Lesson 5: fine-tuning worked only after the eval measured the real workflow.
+7. What still is not good enough: SBAR and radio handoff usefulness.
+8. What comes next: focused v4 work on handoff, cue ownership, and workflow metrics.
+9. Closing: Build Small taught me that useful small-model apps are systems of restraint.
+
+## Evidence To Include
+
+Use these numbers sparingly, as anchors:
+
+- Hosted Omni baseline eval: `28/50` hosted model competence and `50/50` final validation.
+- Hosted Omni load-bearing follow-up: `31/50` whole-output competence, `8/50` full fallback, `480/650` model-retained fields, `170/650` deterministic patches.
+- Local 4B baseline after scaffolding: `26/50` competence, mostly via repair, with `50/50` final validation.
+- Local 4B v1 pilot: proved the train, merge, convert, serve, and eval loop, but regressed competence to `11/50`.
+- Local 4B v2: improved to `33/50` on the locked 50-case eval.
+- Local 4B v3 field-workflow holdout: `107/150` competence, `93/150` raw successes, `14/150` repairs, `2/150` full fallbacks, `148/150` final validation.
+- V3 weakness: `REFERRAL-SBAR-v1` at `0/27`, `radio_handoff` at `0/16`, `sbar_handoff_usefulness` at `0/10`.
+- Modal v3 training: `700/700` steps, final `eval_loss=0.04357146`, final `train_loss=0.60960097`, artifacts present in `figment-checkpoints:/figment_sft_v3/figment-sft-v3-lora`.
+
+## Claims To Avoid
+
+- Do not claim Figment is clinically validated.
+- Do not claim it diagnoses, prescribes, or makes autonomous triage decisions.
+- Do not claim final validation is pure model competence.
+- Do not claim local/off-grid proof is complete unless referring narrowly to recorded local text-model evidence.
+- Do not imply the v3 fine-tune solved handoff usefulness.
+
+## Drafting Notes
+
+The rough draft should read as a blog post, not a project README. It can use section headings, short paragraphs, and a few concrete metrics. It should emphasize decisions and lessons, not implementation minutiae.
+
+The strongest closing thought is that "small" was not just about model size. It was about narrowing the job until the model could be useful, measured, and corrected.

figment/harness_evidence.py

@@ -0,0 +1,94 @@
+"""Deterministic evidence surfaces for Figment navigator outputs and traces."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+from typing import Any
+
+
+def build_harness_evidence(
+    *,
+    confirmed_intake: Mapping[str, Any] | None,
+    retrieved_card_ids: Iterable[Any],
+    rule_results: Iterable[Mapping[str, Any]],
+    urgency_floor: str,
+    validator_result: Mapping[str, Any],
+    final_output: Mapping[str, Any] | None = None,
+    model_route: Mapping[str, Any] | None = None,
+    audio: Mapping[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Build app-owned, non-secret evidence badges for a navigator result."""
+
+    route = model_route or {}
+    output = final_output or {}
+    validation_status = _validation_status(validator_result)
+    return {
+        "confirmed_intake": bool(confirmed_intake and confirmed_intake.get("confirmed") is True),
+        "retrieved_card_ids": _unique_strings(retrieved_card_ids),
+        "deterministic_rule_ids": _unique_strings(rule.get("rule_id") for rule in rule_results),
+        "deterministic_rule_card_ids": _unique_strings(rule.get("card_id") for rule in rule_results),
+        "urgency_floor": str(urgency_floor),
+        "validator_status": validation_status,
+        "audio_correction_status": _audio_correction_status(audio),
+        "source_card_ids": _unique_strings(output.get("source_cards", [])),
+        "fallback_tier": _optional_string(route.get("fallback_tier")),
+        "fallback_reason": _optional_string(route.get("fallback_reason")),
+        "final_route": _optional_string(route.get("final_route") or route.get("runtime_contribution"))
+        or "unknown",
+        "field_level_fallback_used": bool(route.get("field_level_fallback_used")),
+        "repair_attempt_count": _int_or_zero(route.get("repair_attempt_count")),
+        "repair_scopes": _unique_strings(route.get("repair_scopes", [])),
+    }
+
+
+def _validation_status(validator_result: Mapping[str, Any]) -> str:
+    if validator_result.get("passed") is True:
+        return "passed"
+    if validator_result.get("passed") is False:
+        return "failed"
+    return "unknown"
+
+
+def _audio_correction_status(audio: Mapping[str, Any] | None) -> str:
+    if not isinstance(audio, Mapping) or not audio:
+        return "not_applicable"
+    correction_keys = (
+        "manual_corrections",
+        "manual_correction",
+        "corrected_fields",
+        "corrections_applied",
+        "human_corrected_fields",
+    )
+    if any(audio.get(key) for key in correction_keys):
+        return "corrected"
+    if audio.get("transcript") or audio.get("fields") or audio.get("structured_intake_patch"):
+        return "no_manual_correction_recorded"
+    return "not_applicable"
+
+
+def _unique_strings(values: Iterable[Any]) -> list[str]:
+    out: list[str] = []
+    for value in values:
+        if value is None:
+            continue
+        text = str(value).strip()
+        if text and text not in out:
+            out.append(text)
+    return out
+
+
+def _optional_string(value: Any) -> str | None:
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text or None
+
+
+def _int_or_zero(value: Any) -> int:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return 0
+
+
+__all__ = ["build_harness_evidence"]

figment/observation_targets.py

@@ -0,0 +1,521 @@
+"""Deterministic scaffolding helpers for required observation targets."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+from copy import deepcopy
+from dataclasses import dataclass, field
+import re
+from typing import Any
+
+
+CARD_IDS_EXEMPT_FROM_OBSERVATION_TARGETS = {"SAFETY-BOUNDARIES-v1", "REFERRAL-SBAR-v1"}
+TRACE_ONLY_REQUIRED_OBSERVATION_IDS_KEY = "selected_required_observation_ids"
+URGENCY_ORDER = {"routine": 0, "monitor": 1, "urgent": 2, "emergency": 3}
+TARGET_TOKEN_STOPWORDS = {
+    "a",
+    "an",
+    "and",
+    "or",
+    "the",
+    "to",
+    "of",
+    "for",
+    "if",
+    "by",
+    "with",
+}
+
+
+@dataclass(frozen=True)
+class NavigationScaffoldResult:
+    """Navigator output after deterministic scaffolding plus changed fields."""
+
+    output: dict[str, Any]
+    patched_fields: set[str]
+    filled_required_observation_ids: list[str]
+    model_selected_required_observation_ids: list[str] = field(default_factory=list)
+    invalid_selected_required_observation_ids: list[str] = field(default_factory=list)
+    stripped_trace_only_fields: list[str] = field(default_factory=list)
+
+
+def required_observation_targets(retrieved_cards: Iterable[Mapping[str, Any]]) -> list[dict[str, Any]]:
+    """Return stable target ids for required observations on retrieved cards."""
+
+    targets: list[dict[str, Any]] = []
+    for item in retrieved_cards:
+        card = _card_payload(item)
+        card_id = str(card.get("card_id", "")).strip()
+        if not card_id:
+            continue
+        required_observations = card.get("required_observations")
+        if not isinstance(required_observations, list):
+            continue
+        title = str(card.get("title", "")).strip()
+        for index, observation in enumerate(required_observations, start=1):
+            display_text = str(observation).strip()
+            if not display_text:
+                continue
+            targets.append(
+                {
+                    "id": f"{card_id}::required_observation::{index}",
+                    "card_id": card_id,
+                    "title": title,
+                    "display_text": display_text,
+                    "cue_tokens": _target_tokens(display_text),
+                }
+            )
+    return targets
+
+
+def build_case_fact_ledger(intake: Mapping[str, Any]) -> dict[str, list[dict[str, Any]]]:
+    """Split confirmed intake into present, absent/denied, and unclear facts."""
+
+    ledger = {"present": [], "absent_or_denied": [], "unclear": []}
+    if intake.get("confirmed") is not True:
+        ledger["unclear"].append(
+            {
+                "field": "confirmed",
+                "value": "intake is not confirmed",
+            }
+        )
+        return ledger
+
+    for field, value in sorted(intake.items()):
+        if field == "confirmed" or value is None:
+            continue
+        text = str(value).strip()
+        if not text:
+            continue
+        lowered = text.lower()
+        if lowered in {"unknown", "pending", "not recorded", "not available", "unclear"}:
+            ledger["unclear"].append({"field": field, "value": text})
+            continue
+        ledger["present"].append({"field": field, "value": text})
+        for phrase in _negated_phrases(text):
+            ledger["absent_or_denied"].append({"field": field, "value": phrase})
+    return ledger
+
+
+def apply_navigation_scaffolding(
+    output: Mapping[str, Any],
+    *,
+    retrieved_cards: Iterable[Mapping[str, Any]],
+    rule_results: list[dict[str, Any]],
+    urgency_floor: str,
+    confirmed_intake: Mapping[str, Any] | None = None,
+) -> NavigationScaffoldResult:
+    """Patch deterministic control fields and required-observation omissions."""
+
+    patched = deepcopy(dict(output))
+    patched_fields: set[str] = set()
+    retrieved_card_list = list(retrieved_cards)
+    retrieved_ids = _retrieved_card_ids(retrieved_card_list)
+    fired_card_ids = _fired_rule_card_ids(rule_results)
+    targets = required_observation_targets(retrieved_card_list)
+    (
+        model_selected_required_observation_ids,
+        invalid_selected_required_observation_ids,
+        stripped_trace_only_fields,
+    ) = _pop_trace_only_required_observation_ids(patched, targets)
+
+    current_urgency = str(patched.get("protocol_urgency", "")).strip()
+    if not _urgency_at_least(current_urgency, urgency_floor):
+        patched["protocol_urgency"] = urgency_floor
+        patched_fields.add("protocol_urgency")
+
+    expected_red_flags = list(rule_results)
+    if patched.get("red_flags") != expected_red_flags:
+        patched["red_flags"] = expected_red_flags
+        patched_fields.add("red_flags")
+
+    source_cards = _scaffold_source_cards(patched.get("source_cards"), retrieved_ids, fired_card_ids)
+    if patched.get("source_cards") != source_cards:
+        patched["source_cards"] = source_cards
+        patched_fields.add("source_cards")
+
+    pathways = _scaffold_candidate_pathways(
+        patched.get("candidate_protocol_pathways"),
+        source_cards,
+        fired_card_ids,
+    )
+    if patched.get("candidate_protocol_pathways") != pathways:
+        patched["candidate_protocol_pathways"] = pathways
+        patched_fields.add("candidate_protocol_pathways")
+
+    filled_ids = _fill_required_observation_targets(
+        patched,
+        targets,
+        selected_required_observation_ids=model_selected_required_observation_ids,
+    )
+    if filled_ids:
+        patched_fields.update({"missing_info_to_collect", "next_observations_to_collect"})
+
+    if confirmed_intake is not None and _scaffold_handoff_note_sbar(
+        patched,
+        confirmed_intake=confirmed_intake,
+        rule_results=rule_results,
+        urgency_floor=urgency_floor,
+        source_card_ids=source_cards,
+    ):
+        patched_fields.add("handoff_note_sbar")
+
+    return NavigationScaffoldResult(
+        output=patched,
+        patched_fields=patched_fields,
+        filled_required_observation_ids=filled_ids,
+        model_selected_required_observation_ids=model_selected_required_observation_ids,
+        invalid_selected_required_observation_ids=invalid_selected_required_observation_ids,
+        stripped_trace_only_fields=stripped_trace_only_fields,
+    )
+
+
+def build_handoff_note_sbar_template(
+    intake: Mapping[str, Any],
+    rule_results: list[dict[str, Any]],
+    urgency_floor: str,
+    *,
+    source_card_ids: Iterable[Any] = (),
+) -> dict[str, str]:
+    """Build a deterministic, grounded SBAR draft from confirmed harness facts."""
+
+    situation = _first_text(
+        intake.get("chief_concern"),
+        intake.get("responder_note"),
+        "Confirmed field concern",
+    )
+    background_parts = []
+    if _has_value(intake.get("setting")):
+        background_parts.append(f"Setting: {intake['setting']}.")
+    if _has_value(intake.get("patient_age")):
+        background_parts.append(f"Age: {intake['patient_age']}.")
+    if _has_value(intake.get("pregnancy_status")):
+        background_parts.append(f"Pregnancy status: {intake['pregnancy_status']}.")
+
+    assessment_parts = []
+    if _has_value(intake.get("symptoms")):
+        assessment_parts.append(f"Symptoms: {intake['symptoms']}.")
+    if _has_value(intake.get("vitals")):
+        assessment_parts.append(f"Vitals: {intake['vitals']}.")
+    red_flag_labels = [
+        str(rule.get("label") or rule.get("rule_id"))
+        for rule in rule_results
+        if rule.get("label") or rule.get("rule_id")
+    ]
+    if red_flag_labels:
+        assessment_parts.append(f"Red flags: {'; '.join(red_flag_labels)}.")
+
+    source_suffix = _source_card_suffix(source_card_ids)
+    return {
+        "situation": str(situation),
+        "background": " ".join(background_parts) or "Background details pending from confirmed intake.",
+        "assessment_observations_only": " ".join(assessment_parts)
+        or "Assessment observations pending from confirmed intake.",
+        "handoff_request": f"Request {urgency_floor} review/escalation per cited local protocol cards{source_suffix}.",
+    }
+
+
+def targets_for_failure_cards(
+    targets: Iterable[Mapping[str, Any]],
+    failures: Iterable[str],
+) -> list[dict[str, Any]]:
+    """Filter required-observation targets to card ids named in validation failures."""
+
+    failure_text = "\n".join(str(failure) for failure in failures)
+    card_ids = set(re.findall(r"\b[A-Z][A-Z0-9-]+-v\d+\b", failure_text))
+    selected = []
+    for target in targets:
+        card_id = str(target.get("card_id", "")).strip()
+        if card_id and (not card_ids or card_id in card_ids):
+            selected.append(dict(target))
+    return selected
+
+
+def _scaffold_handoff_note_sbar(
+    output: dict[str, Any],
+    *,
+    confirmed_intake: Mapping[str, Any],
+    rule_results: list[dict[str, Any]],
+    urgency_floor: str,
+    source_card_ids: Iterable[Any],
+) -> bool:
+    template = build_handoff_note_sbar_template(
+        confirmed_intake,
+        rule_results,
+        urgency_floor,
+        source_card_ids=source_card_ids,
+    )
+    handoff = output.get("handoff_note_sbar")
+    if not isinstance(handoff, Mapping):
+        output["handoff_note_sbar"] = template
+        return True
+
+    patched_handoff = dict(handoff)
+    changed = False
+    for field, template_value in template.items():
+        value = str(patched_handoff.get(field) or "").strip()
+        if not value or _handoff_slot_needs_scaffold(field, value, rule_results):
+            patched_handoff[field] = template_value
+            changed = True
+    if changed:
+        output["handoff_note_sbar"] = patched_handoff
+    return changed
+
+
+def _handoff_slot_needs_scaffold(field: str, value: str, rule_results: list[dict[str, Any]]) -> bool:
+    normalized = _normalize_text(value)
+    if field == "assessment_observations_only" and _unsafe_assessment_language(normalized):
+        return True
+    if field == "assessment_observations_only" and rule_results:
+        rule_markers = [
+            _normalize_text(str(rule.get(key, "")))
+            for rule in rule_results
+            for key in ("rule_id", "label")
+            if rule.get(key)
+        ]
+        return "red flag" not in normalized and "rule" not in normalized and not any(
+            marker and marker in normalized for marker in rule_markers
+        )
+    return False
+
+
+def _unsafe_assessment_language(normalized_text: str) -> bool:
+    return bool(
+        re.search(
+            r"\b(?:diagnos(?:e|is|ed)|prescrib(?:e|ed|ing)|administer|discharge|treat(?:ment)?|dose|dosing)\b",
+            normalized_text,
+        )
+    )
+
+
+def _fill_required_observation_targets(
+    output: dict[str, Any],
+    targets: list[dict[str, Any]],
+    *,
+    selected_required_observation_ids: Iterable[str] = (),
+) -> list[str]:
+    source_cards = {str(card_id) for card_id in output.get("source_cards", []) if str(card_id)}
+    candidate_cards = {
+        str(pathway.get("card_id", "")).strip()
+        for pathway in output.get("candidate_protocol_pathways", [])
+        if isinstance(pathway, Mapping) and str(pathway.get("card_id", "")).strip() in source_cards
+    }
+    observation_cards = candidate_cards or source_cards
+    actionable_targets = [
+        target
+        for target in targets
+        if target["card_id"] in observation_cards
+        and target["card_id"] not in CARD_IDS_EXEMPT_FROM_OBSERVATION_TARGETS
+    ]
+    if not actionable_targets:
+        return []
+
+    missing_info = _string_list(output.get("missing_info_to_collect"))
+    next_observations = _string_list(output.get("next_observations_to_collect"))
+    combined_text = "\n".join(missing_info + next_observations)
+    combined_tokens = set(_target_tokens(combined_text))
+
+    missing_targets = [
+        target
+        for target in actionable_targets
+        if not _target_present(target, combined_text, combined_tokens)
+    ]
+    if not missing_targets:
+        return []
+
+    for target in missing_targets:
+        display_text = str(target["display_text"])
+        missing_info.append(display_text)
+        next_observations.append(display_text)
+    output["missing_info_to_collect"] = missing_info
+    output["next_observations_to_collect"] = next_observations
+    return [str(target["id"]) for target in missing_targets]
+
+
+def _target_present(target: Mapping[str, Any], text: str, tokens: set[str]) -> bool:
+    target_tokens = set(str(token) for token in target.get("cue_tokens", []))
+    if target_tokens and target_tokens <= tokens:
+        return True
+    normalized_target = _normalize_text(str(target.get("display_text", "")))
+    return bool(normalized_target and normalized_target in _normalize_text(text))
+
+
+def _pop_trace_only_required_observation_ids(
+    output: dict[str, Any],
+    targets: Iterable[Mapping[str, Any]],
+) -> tuple[list[str], list[str], list[str]]:
+    if TRACE_ONLY_REQUIRED_OBSERVATION_IDS_KEY not in output:
+        return [], [], []
+
+    raw_value = output.pop(TRACE_ONLY_REQUIRED_OBSERVATION_IDS_KEY)
+    target_ids = {str(target.get("id", "")).strip() for target in targets}
+    target_ids.discard("")
+    selected: list[str] = []
+    invalid: list[str] = []
+    for target_id in _string_list(raw_value):
+        normalized = str(target_id).strip()
+        if not normalized:
+            continue
+        if normalized in target_ids:
+            _append_unique(selected, normalized)
+        else:
+            _append_unique(invalid, normalized)
+    return selected, invalid, [TRACE_ONLY_REQUIRED_OBSERVATION_IDS_KEY]
+
+
+def _target_tokens(value: Any) -> list[str]:
+    tokens = re.findall(r"[a-z0-9]+", str(value).lower())
+    return [token for token in tokens if token and token not in TARGET_TOKEN_STOPWORDS]
+
+
+def _normalize_text(value: str) -> str:
+    return " ".join(re.findall(r"[a-z0-9]+", value.lower()))
+
+
+def _card_payload(item: Mapping[str, Any]) -> Mapping[str, Any]:
+    card = item.get("card", item)
+    return card if isinstance(card, Mapping) else {}
+
+
+def _retrieved_card_ids(retrieved_cards: Iterable[Mapping[str, Any]]) -> list[str]:
+    ids: list[str] = []
+    for item in retrieved_cards:
+        card = _card_payload(item)
+        card_id = str(item.get("card_id") or card.get("card_id") or "").strip()
+        if card_id and card_id not in ids:
+            ids.append(card_id)
+    return ids
+
+
+def _fired_rule_card_ids(rule_results: Iterable[Mapping[str, Any]]) -> list[str]:
+    ids: list[str] = []
+    for rule in rule_results:
+        card_id = str(rule.get("card_id", "")).strip()
+        if card_id and card_id not in ids:
+            ids.append(card_id)
+    return ids
+
+
+def _scaffold_source_cards(value: Any, retrieved_ids: list[str], fired_card_ids: list[str]) -> list[str]:
+    raw_source_cards = [str(card_id) for card_id in value if str(card_id)] if isinstance(value, list) else []
+    allowed = set(retrieved_ids) | set(fired_card_ids)
+    source_cards: list[str] = []
+    for card_id in fired_card_ids:
+        if card_id not in source_cards:
+            source_cards.append(card_id)
+    for card_id in raw_source_cards:
+        if card_id in allowed and card_id not in source_cards:
+            source_cards.append(card_id)
+    if not source_cards:
+        source_cards.extend(retrieved_ids[:3])
+    return source_cards[:6]
+
+
+def _scaffold_candidate_pathways(
+    value: Any,
+    source_cards: list[str],
+    fired_card_ids: list[str],
+) -> list[dict[str, str]]:
+    source_set = set(source_cards)
+    pathways: list[dict[str, str]] = []
+    if isinstance(value, list):
+        for item in value:
+            if not isinstance(item, Mapping):
+                continue
+            card_id = str(item.get("card_id", "")).strip()
+            if card_id not in source_set:
+                continue
+            pathways.append(
+                {
+                    "card_id": card_id,
+                    "reason_relevant": str(item.get("reason_relevant") or "Retrieved from confirmed intake."),
+                }
+            )
+    pathway_ids = {pathway["card_id"] for pathway in pathways}
+    for card_id in fired_card_ids:
+        if card_id not in source_set:
+            continue
+        if card_id in pathway_ids:
+            continue
+        pathways.append(
+            {
+                "card_id": card_id,
+                "reason_relevant": "Required by deterministic rule or cited protocol context.",
+            }
+        )
+        pathway_ids.add(card_id)
+    if not pathways:
+        fallback_source_cards = [
+            card_id for card_id in source_cards if card_id not in CARD_IDS_EXEMPT_FROM_OBSERVATION_TARGETS
+        ] or source_cards
+        pathways = [
+            {
+                "card_id": card_id,
+                "reason_relevant": "Retrieved from confirmed intake and deterministic protocol context.",
+            }
+            for card_id in fallback_source_cards[:3]
+        ]
+    return pathways
+
+
+def _urgency_at_least(actual: str, expected_minimum: str) -> bool:
+    if actual not in URGENCY_ORDER or expected_minimum not in URGENCY_ORDER:
+        return False
+    return URGENCY_ORDER[actual] >= URGENCY_ORDER[expected_minimum]
+
+
+def _string_list(value: Any) -> list[str]:
+    if isinstance(value, list):
+        return [str(item) for item in value if str(item)]
+    if isinstance(value, str) and value.strip():
+        return [value.strip()]
+    return []
+
+
+def _append_unique(items: list[str], value: str) -> None:
+    if value not in items:
+        items.append(value)
+
+
+def _negated_phrases(text: str) -> list[str]:
+    phrases: list[str] = []
+    for match in re.finditer(r"\b(no|denies|denied|without)\s+([^,.;]+)", text, re.IGNORECASE):
+        marker = match.group(1).lower()
+        phrase = " ".join(match.group(2).split()).strip()
+        if not phrase:
+            continue
+        phrases.append(f"{marker} {phrase}")
+    return phrases
+
+
+def _source_card_suffix(source_card_ids: Iterable[Any]) -> str:
+    ids = [str(card_id).strip() for card_id in source_card_ids if str(card_id).strip()]
+    return f" ({', '.join(ids[:4])})" if ids else ""
+
+
+def _first_text(*values: Any) -> str:
+    for value in values:
+        if _has_value(value):
+            return str(value).strip()
+    return ""
+
+
+def _has_value(value: Any) -> bool:
+    if value is None:
+        return False
+    if isinstance(value, str):
+        return bool(value.strip())
+    if isinstance(value, (list, tuple, set, dict)):
+        return bool(value)
+    return True
+
+
+__all__ = [
+    "NavigationScaffoldResult",
+    "TRACE_ONLY_REQUIRED_OBSERVATION_IDS_KEY",
+    "apply_navigation_scaffolding",
+    "build_case_fact_ledger",
+    "build_handoff_note_sbar_template",
+    "required_observation_targets",
+    "targets_for_failure_cards",
+]

modal/eval_figment_nemotron.py

@@ -0,0 +1,635 @@
+"""Modal batch eval for Figment's merged Nemotron 4B checkpoint.
+
+Run the full 150-case v5 eval as a terminating Modal job:
+
+    .venv/bin/modal run modal/eval_figment_nemotron.py
+"""
+
+from __future__ import annotations
+
+from datetime import UTC
+from datetime import datetime
+import json
+from pathlib import Path
+from typing import Any
+
+import modal
+
+
+APP_NAME = "figment-nemotron-4b-batch-eval"
+DEFAULT_DATASET_VERSION = "figment_sft_v5"
+DEFAULT_MERGED_MODEL_NAME = "figment-sft-v5-lora-merged-bf16"
+DEFAULT_MODEL_ID = "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16"
+DEFAULT_CASE_PATH = "data/eval/field_workflow_holdout_v1.jsonl"
+DEFAULT_CASE_COUNT = 150
+DEFAULT_EVAL_GPU = "H100"
+DEFAULT_CUDA_ARCHITECTURES = "90"
+CHECKPOINT_VOLUME_NAME = "figment-checkpoints"
+RESULT_VOLUME_NAME = "figment-eval-results"
+CHECKPOINT_DIR = "/checkpoints"
+RESULT_DIR = "/eval_results"
+LLAMA_CPP_DIR = "/opt/llama.cpp"
+LLAMA_SERVER_BIN = f"{LLAMA_CPP_DIR}/build/bin/llama-server"
+LLAMA_CONVERT_SCRIPT = f"{LLAMA_CPP_DIR}/convert_hf_to_gguf.py"
+MODEL_SERVER_BASE_URL = "http://127.0.0.1:8001/v1"
+MODEL_SERVER_HOST = "127.0.0.1"
+MODEL_SERVER_PORT = 8001
+
+
+app = modal.App(APP_NAME)
+
+checkpoint_volume = modal.Volume.from_name(CHECKPOINT_VOLUME_NAME, create_if_missing=True)
+result_volume = modal.Volume.from_name(RESULT_VOLUME_NAME, create_if_missing=True)
+
+eval_image = (
+    modal.Image.from_registry("pytorch/pytorch:2.7.1-cuda12.6-cudnn9-devel")
+    .apt_install("git", "build-essential", "cmake", "curl", "ninja-build")
+    .uv_pip_install(
+        "accelerate>=1.8,<2",
+        "einops>=0.8,<1",
+        "ninja>=1.11,<2",
+        "protobuf>=5,<7",
+        "safetensors>=0.5,<1",
+        "sentencepiece>=0.2,<1",
+        "setuptools>=70",
+        "torch==2.7.1",
+        "transformers>=4.52,<5",
+        "wheel>=0.45",
+    )
+    .add_local_dir("tools/llama.cpp", LLAMA_CPP_DIR, copy=True)
+    .run_commands(
+        f"python -m pip install -r {LLAMA_CPP_DIR}/requirements.txt",
+        f"cmake -S {LLAMA_CPP_DIR} -B {LLAMA_CPP_DIR}/build "
+        f"-DGGML_CUDA=ON -DLLAMA_CURL=OFF -DCMAKE_BUILD_TYPE=Release "
+        f"-DCMAKE_CUDA_ARCHITECTURES={DEFAULT_CUDA_ARCHITECTURES}",
+        f"cmake --build {LLAMA_CPP_DIR}/build --config Release --target llama-server -j $(nproc)",
+    )
+    .env(
+        {
+            "TOKENIZERS_PARALLELISM": "false",
+            "PYTHON_DOTENV_DISABLED": "true",
+        }
+    )
+    .add_local_python_source("figment", "scripts")
+)
+
+
+def build_eval_config(
+    *,
+    dataset_version: str = DEFAULT_DATASET_VERSION,
+    merged_model_name: str = DEFAULT_MERGED_MODEL_NAME,
+    output_name: str = "",
+    model_id: str = DEFAULT_MODEL_ID,
+    timeout_seconds: float = 300.0,
+    expected_case_count: int = DEFAULT_CASE_COUNT,
+    max_context_tokens: int = 16384,
+    max_generation_tokens: int = 1536,
+) -> dict[str, Any]:
+    if not output_name:
+        stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
+        output_name = f"{dataset_version}_field_workflow_holdout_modal_gpu_{stamp}"
+    checkpoint_model_dir = f"{CHECKPOINT_DIR}/{dataset_version}/{merged_model_name}"
+    return {
+        "dataset_version": dataset_version,
+        "merged_model_name": merged_model_name,
+        "checkpoint_model_dir": checkpoint_model_dir,
+        "checkpoint_artifact": (
+            f"{CHECKPOINT_VOLUME_NAME}:/{dataset_version}/{merged_model_name}"
+        ),
+        "result_volume_name": RESULT_VOLUME_NAME,
+        "output_name": output_name,
+        "output_dir": f"{RESULT_DIR}/{output_name}",
+        "gguf_model_path": f"{RESULT_DIR}/model_cache/{dataset_version}/{merged_model_name}.bf16.gguf",
+        "runtime": "llama_cpp_cuda",
+        "cuda_architectures": DEFAULT_CUDA_ARCHITECTURES,
+        "model_id": model_id,
+        "base_url": MODEL_SERVER_BASE_URL,
+        "case_paths": [f"/tmp/figment_eval_cases/{Path(DEFAULT_CASE_PATH).name}"],
+        "timeout_seconds": timeout_seconds,
+        "expected_case_count": expected_case_count,
+        "max_context_tokens": max_context_tokens,
+        "max_generation_tokens": max_generation_tokens,
+    }
+
+
+@app.function(
+    image=eval_image,
+    gpu=DEFAULT_EVAL_GPU,
+    cpu=8,
+    memory=131072,
+    ephemeral_disk=524288,
+    volumes={
+        CHECKPOINT_DIR: checkpoint_volume,
+        RESULT_DIR: result_volume,
+    },
+    timeout=6 * 60 * 60,
+)
+def run_batch_eval(
+    config: dict[str, Any],
+    case_files: list[dict[str, str]],
+    protocol_cards: list[dict[str, str]],
+) -> dict[str, Any]:
+    import os
+    import time
+
+    import torch
+
+    from scripts.run_local_4b_evidence import run_evidence
+
+    started = datetime.now(UTC)
+    model_dir = Path(str(config["checkpoint_model_dir"]))
+    output_dir = Path(str(config["output_dir"]))
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    if not (model_dir / "config.json").exists():
+        raise FileNotFoundError(f"missing merged model config at {model_dir}")
+    if not (model_dir / "model.safetensors.index.json").exists():
+        raise FileNotFoundError(f"missing merged model index at {model_dir}")
+
+    staged_case_paths = _stage_case_files(case_files)
+    _stage_protocol_cards(protocol_cards)
+
+    runtime_gpu = {
+        "cuda_available": torch.cuda.is_available(),
+        "cuda_device_count": torch.cuda.device_count(),
+        "cuda_device_name": torch.cuda.get_device_name(0) if torch.cuda.is_available() else "",
+    }
+    print(json.dumps({"runtime_gpu": runtime_gpu}, sort_keys=True), flush=True)
+
+    gguf_model_path = _ensure_gguf_model(model_dir=model_dir, config=config)
+    result_volume.commit()
+
+    server = _LlamaCppServer(
+        gguf_model_path=gguf_model_path,
+        model_id=str(config["model_id"]),
+        max_context_tokens=int(config["max_context_tokens"]),
+    )
+    server.start()
+    try:
+        summary = run_evidence(
+            base_url=str(config["base_url"]),
+            model_id=str(config["model_id"]),
+            output_dir=output_dir,
+            case_paths=staged_case_paths,
+            limit=None,
+            timeout_seconds=float(config["timeout_seconds"]),
+            force_eval=True,
+        )
+    finally:
+        server.stop()
+
+    records_path = output_dir / "local_4b_eval.jsonl"
+    record_count = _count_jsonl(records_path)
+    finished = datetime.now(UTC)
+    manifest = {
+        "status": "completed" if record_count == int(config["expected_case_count"]) else "count_mismatch",
+        "config": config,
+        "runtime_gpu": runtime_gpu,
+        "checkpoint_artifact": config["checkpoint_artifact"],
+        "output_volume": RESULT_VOLUME_NAME,
+        "output_dir": str(output_dir),
+        "records_path": str(records_path),
+        "record_count": record_count,
+        "expected_case_count": int(config["expected_case_count"]),
+        "started_at": started.isoformat(),
+        "finished_at": finished.isoformat(),
+        "runtime_seconds": round(time.monotonic() - server.started_monotonic, 3),
+        "summary": summary,
+        "files": sorted(path.name for path in output_dir.iterdir() if path.is_file()),
+    }
+    (output_dir / "modal_eval_manifest.json").write_text(
+        json.dumps(manifest, indent=2, sort_keys=True) + "\n",
+        encoding="utf-8",
+    )
+    result_volume.commit()
+
+    if record_count != int(config["expected_case_count"]):
+        raise ValueError(f"expected {config['expected_case_count']} records, found {record_count}")
+    return manifest
+
+
+def _ensure_gguf_model(*, model_dir: Path, config: dict[str, Any]) -> Path:
+    import subprocess
+    import sys
+
+    gguf_path = Path(str(config["gguf_model_path"]))
+    if gguf_path.exists() and gguf_path.stat().st_size > 0:
+        print(
+            json.dumps(
+                {
+                    "modal_eval_gguf": "reuse_existing",
+                    "path": str(gguf_path),
+                    "bytes": gguf_path.stat().st_size,
+                },
+                sort_keys=True,
+            ),
+            flush=True,
+        )
+        return gguf_path
+
+    gguf_path.parent.mkdir(parents=True, exist_ok=True)
+    command = [
+        sys.executable,
+        LLAMA_CONVERT_SCRIPT,
+        str(model_dir),
+        "--outfile",
+        str(gguf_path),
+        "--outtype",
+        "bf16",
+    ]
+    print(json.dumps({"modal_eval_gguf": "convert_start", "command": command}, sort_keys=True), flush=True)
+    completed = subprocess.run(
+        command,
+        check=False,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    )
+    print(completed.stdout, flush=True)
+    if completed.returncode != 0:
+        raise RuntimeError(f"GGUF conversion failed with exit code {completed.returncode}")
+    print(
+        json.dumps(
+            {
+                "modal_eval_gguf": "convert_finished",
+                "path": str(gguf_path),
+                "bytes": gguf_path.stat().st_size,
+            },
+            sort_keys=True,
+        ),
+        flush=True,
+    )
+    return gguf_path
+
+
+class _LlamaCppServer:
+    def __init__(self, *, gguf_model_path: Path, model_id: str, max_context_tokens: int) -> None:
+        self.gguf_model_path = gguf_model_path
+        self.model_id = model_id
+        self.max_context_tokens = max_context_tokens
+        self.process: Any = None
+        self.started_monotonic = 0.0
+
+    def start(self) -> None:
+        import subprocess
+        import threading
+        import time
+        import urllib.request
+
+        command = [
+            LLAMA_SERVER_BIN,
+            "--model",
+            str(self.gguf_model_path),
+            "--ctx-size",
+            str(self.max_context_tokens),
+            "--host",
+            MODEL_SERVER_HOST,
+            "--port",
+            str(MODEL_SERVER_PORT),
+            "--alias",
+            self.model_id,
+            "--parallel",
+            "1",
+            "--temp",
+            "0",
+            "--top-p",
+            "1",
+            "--reasoning",
+            "off",
+            "--n-gpu-layers",
+            "999",
+        ]
+        print(json.dumps({"modal_eval_llama_server": "start", "command": command}, sort_keys=True), flush=True)
+        self.started_monotonic = time.monotonic()
+        self.process = subprocess.Popen(
+            command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            bufsize=1,
+        )
+        threading.Thread(target=self._stream_logs, daemon=True).start()
+
+        deadline = time.monotonic() + 180
+        last_error = ""
+        while time.monotonic() < deadline:
+            if self.process.poll() is not None:
+                raise RuntimeError(f"llama-server exited early with code {self.process.returncode}")
+            try:
+                with urllib.request.urlopen(f"{MODEL_SERVER_BASE_URL}/models", timeout=2) as response:
+                    if response.status == 200:
+                        print(json.dumps({"modal_eval_llama_server": "ready"}, sort_keys=True), flush=True)
+                        return
+            except OSError as exc:
+                last_error = str(exc)
+                time.sleep(1)
+        raise RuntimeError(f"llama-server did not become ready: {last_error}")
+
+    def stop(self) -> None:
+        if self.process is None or self.process.poll() is not None:
+            return
+        self.process.terminate()
+        try:
+            self.process.wait(timeout=30)
+        except Exception:
+            self.process.kill()
+            self.process.wait(timeout=10)
+
+    def _stream_logs(self) -> None:
+        if self.process is None or self.process.stdout is None:
+            return
+        for line in self.process.stdout:
+            print(f"[llama-server] {line.rstrip()}", flush=True)
+
+
+class _OpenAICompatibleTransformersServer:
+    def __init__(
+        self,
+        *,
+        model_dir: Path,
+        model_id: str,
+        max_context_tokens: int,
+        max_generation_tokens: int,
+    ) -> None:
+        self.model_dir = model_dir
+        self.model_id = model_id
+        self.max_context_tokens = max_context_tokens
+        self.max_generation_tokens = max_generation_tokens
+        self.httpd: Any = None
+        self.thread: Any = None
+        self.started_monotonic = 0.0
+
+    def start(self) -> None:
+        import threading
+        import time
+        import urllib.request
+        from http.server import ThreadingHTTPServer
+
+        model_runtime = _TransformersRuntime(
+            model_dir=self.model_dir,
+            model_id=self.model_id,
+            max_context_tokens=self.max_context_tokens,
+            max_generation_tokens=self.max_generation_tokens,
+        )
+        handler = _handler_factory(model_runtime)
+        self.httpd = ThreadingHTTPServer((MODEL_SERVER_HOST, MODEL_SERVER_PORT), handler)
+        self.thread = threading.Thread(target=self.httpd.serve_forever, daemon=True)
+        self.started_monotonic = time.monotonic()
+        self.thread.start()
+        deadline = time.monotonic() + 30
+        last_error = ""
+        while time.monotonic() < deadline:
+            try:
+                with urllib.request.urlopen(f"{MODEL_SERVER_BASE_URL}/models", timeout=2) as response:
+                    if response.status == 200:
+                        return
+            except OSError as exc:
+                last_error = str(exc)
+                time.sleep(0.25)
+        raise RuntimeError(f"model server did not become ready: {last_error}")
+
+    def stop(self) -> None:
+        if self.httpd is not None:
+            self.httpd.shutdown()
+            self.httpd.server_close()
+        if self.thread is not None:
+            self.thread.join(timeout=10)
+
+
+class _TransformersRuntime:
+    def __init__(
+        self,
+        *,
+        model_dir: Path,
+        model_id: str,
+        max_context_tokens: int,
+        max_generation_tokens: int,
+    ) -> None:
+        import threading
+
+        import torch
+        from transformers import AutoModelForCausalLM
+        from transformers import AutoTokenizer
+
+        self.model_id = model_id
+        self.max_context_tokens = max_context_tokens
+        self.max_generation_tokens = max_generation_tokens
+        self.lock = threading.Lock()
+        self.request_count = 0
+        self.torch = torch
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            model_dir,
+            trust_remote_code=True,
+            local_files_only=True,
+        )
+        if self.tokenizer.pad_token is None:
+            self.tokenizer.pad_token = self.tokenizer.eos_token
+        self.model = AutoModelForCausalLM.from_pretrained(
+            model_dir,
+            trust_remote_code=True,
+            local_files_only=True,
+            dtype=torch.bfloat16,
+        )
+        if torch.cuda.is_available():
+            self.model.to("cuda")
+        self.model.eval()
+        self.model.config.use_cache = True
+
+    def models_payload(self) -> dict[str, Any]:
+        return {
+            "object": "list",
+            "data": [
+                {
+                    "id": self.model_id,
+                    "object": "model",
+                    "owned_by": "figment-modal-batch",
+                }
+            ],
+        }
+
+    def chat_completion(self, body: dict[str, Any]) -> dict[str, Any]:
+        import time
+        import uuid
+
+        messages = body.get("messages")
+        if not isinstance(messages, list):
+            raise ValueError("messages must be a list")
+        requested_max_tokens = int(body.get("max_tokens") or 1024)
+        with self.lock:
+            self.request_count += 1
+            request_index = self.request_count
+            started = time.monotonic()
+            input_ids = self.tokenizer.apply_chat_template(
+                messages,
+                add_generation_prompt=True,
+                return_tensors="pt",
+            )
+            device = next(self.model.parameters()).device
+            input_ids = input_ids.to(device)
+            input_len = int(input_ids.shape[-1])
+            available_tokens = max(128, self.max_context_tokens - input_len - 8)
+            max_new_tokens = max(1, min(requested_max_tokens, available_tokens, self.max_generation_tokens))
+            print(
+                json.dumps(
+                    {
+                        "modal_eval_request": request_index,
+                        "status": "started",
+                        "prompt_tokens": input_len,
+                        "max_new_tokens": max_new_tokens,
+                    },
+                    sort_keys=True,
+                ),
+                flush=True,
+            )
+            attention_mask = self.torch.ones_like(input_ids)
+            with self.torch.inference_mode():
+                output_ids = self.model.generate(
+                    input_ids=input_ids,
+                    attention_mask=attention_mask,
+                    max_new_tokens=max_new_tokens,
+                    do_sample=False,
+                    pad_token_id=self.tokenizer.pad_token_id,
+                    eos_token_id=self.tokenizer.eos_token_id,
+                )
+            generated_ids = output_ids[0, input_len:]
+            content = self.tokenizer.decode(generated_ids, skip_special_tokens=True).strip()
+            print(
+                json.dumps(
+                    {
+                        "modal_eval_request": request_index,
+                        "status": "finished",
+                        "completion_tokens": int(generated_ids.numel()),
+                        "elapsed_seconds": round(time.monotonic() - started, 3),
+                    },
+                    sort_keys=True,
+                ),
+                flush=True,
+            )
+        return {
+            "id": f"chatcmpl-{uuid.uuid4().hex}",
+            "object": "chat.completion",
+            "created": int(time.time()),
+            "model": self.model_id,
+            "choices": [
+                {
+                    "index": 0,
+                    "message": {"role": "assistant", "content": content},
+                    "finish_reason": "stop",
+                }
+            ],
+            "usage": {
+                "prompt_tokens": input_len,
+                "completion_tokens": int(generated_ids.numel()),
+                "total_tokens": input_len + int(generated_ids.numel()),
+            },
+        }
+
+
+def _handler_factory(runtime: _TransformersRuntime) -> Any:
+    import json
+    from http.server import BaseHTTPRequestHandler
+
+    class Handler(BaseHTTPRequestHandler):
+        def do_GET(self) -> None:  # noqa: N802
+            if self.path.rstrip("/") == "/v1/models":
+                self._send_json(200, runtime.models_payload())
+            else:
+                self._send_json(404, {"error": f"unknown path: {self.path}"})
+
+        def do_POST(self) -> None:  # noqa: N802
+            if self.path.rstrip("/") != "/v1/chat/completions":
+                self._send_json(404, {"error": f"unknown path: {self.path}"})
+                return
+            try:
+                length = int(self.headers.get("Content-Length", "0"))
+                body = json.loads(self.rfile.read(length).decode("utf-8"))
+                payload = runtime.chat_completion(body)
+            except Exception as exc:  # pragma: no cover - exercised inside Modal runtime.
+                self._send_json(500, {"error": str(exc)})
+                return
+            self._send_json(200, payload)
+
+        def log_message(self, format: str, *args: Any) -> None:
+            return
+
+        def _send_json(self, status: int, payload: dict[str, Any]) -> None:
+            encoded = json.dumps(payload).encode("utf-8")
+            self.send_response(status)
+            self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(encoded)))
+            self.end_headers()
+            self.wfile.write(encoded)
+
+    return Handler
+
+
+def _stage_case_files(case_files: list[dict[str, str]]) -> list[Path]:
+    case_dir = Path("/tmp/figment_eval_cases")
+    case_dir.mkdir(parents=True, exist_ok=True)
+    paths: list[Path] = []
+    for item in case_files:
+        path = case_dir / Path(item["name"]).name
+        path.write_text(item["text"], encoding="utf-8")
+        paths.append(path)
+    return paths
+
+
+def _stage_protocol_cards(protocol_cards: list[dict[str, str]]) -> None:
+    from figment import retrieval
+
+    card_dir = Path(retrieval.DEFAULT_CARD_DIR)
+    card_dir.mkdir(parents=True, exist_ok=True)
+    for path in card_dir.glob("*.json"):
+        path.unlink()
+    for item in protocol_cards:
+        (card_dir / Path(item["name"]).name).write_text(item["text"], encoding="utf-8")
+
+
+def _count_jsonl(path: Path) -> int:
+    if not path.exists():
+        return 0
+    return sum(1 for line in path.read_text(encoding="utf-8").splitlines() if line.strip())
+
+
+def _read_case_files(paths: list[str]) -> list[dict[str, str]]:
+    return [
+        {"name": Path(path).name, "text": Path(path).read_text(encoding="utf-8")}
+        for path in paths
+    ]
+
+
+def _read_protocol_cards() -> list[dict[str, str]]:
+    return [
+        {"name": path.name, "text": path.read_text(encoding="utf-8")}
+        for path in sorted(Path("data/protocol_cards").glob("*.json"))
+    ]
+
+
+@app.local_entrypoint()
+def main(
+    output_name: str = "",
+    dataset_version: str = DEFAULT_DATASET_VERSION,
+    merged_model_name: str = DEFAULT_MERGED_MODEL_NAME,
+    cases: str = DEFAULT_CASE_PATH,
+    timeout_seconds: float = 300.0,
+    gpu: str = DEFAULT_EVAL_GPU,
+) -> None:
+    case_paths = [item.strip() for item in cases.split(",") if item.strip()]
+    config = build_eval_config(
+        dataset_version=dataset_version,
+        merged_model_name=merged_model_name,
+        output_name=output_name,
+        timeout_seconds=timeout_seconds,
+    )
+    if len(case_paths) != 1 or Path(case_paths[0]).name != Path(DEFAULT_CASE_PATH).name:
+        config["case_paths"] = [f"/tmp/figment_eval_cases/{Path(path).name}" for path in case_paths]
+        config["expected_case_count"] = sum(
+            1 for path in case_paths for line in Path(path).read_text(encoding="utf-8").splitlines() if line.strip()
+        )
+
+    result = run_batch_eval.with_options(gpu=gpu).remote(
+        config,
+        _read_case_files(case_paths),
+        _read_protocol_cards(),
+    )
+    print(json.dumps({"modal_batch_eval": result}, indent=2, sort_keys=True))

modal/finetune_figment_nemotron.py

@@ -0,0 +1,581 @@
+"""Modal LoRA fine-tuning job for Figment's local Nemotron 4B navigator.
+
+Run a smoke job first:
+
+    .venv/bin/modal run modal/finetune_figment_nemotron.py --smoke true
+
+Then run the full pilot:
+
+    .venv/bin/modal run modal/finetune_figment_nemotron.py
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import modal
+
+
+APP_NAME = "figment-nemotron-4b-lora"
+DEFAULT_DATASET_VERSION = "figment_sft_v1"
+DEFAULT_BASE_MODEL_ID = "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16"
+DEFAULT_DATASET_PATH = "data/finetune/figment_sft_v1.jsonl"
+DEFAULT_TRAIN_GPU = "H100"
+DATA_VOLUME_NAME = "figment-sft-data"
+MODEL_CACHE_VOLUME_NAME = "figment-model-cache"
+CHECKPOINT_VOLUME_NAME = "figment-checkpoints"
+DATA_DIR = "/data"
+MODEL_CACHE_DIR = "/model_cache"
+CHECKPOINT_DIR = "/checkpoints"
+
+
+app = modal.App(APP_NAME)
+
+model_cache_volume = modal.Volume.from_name(MODEL_CACHE_VOLUME_NAME, create_if_missing=True)
+data_volume = modal.Volume.from_name(DATA_VOLUME_NAME, create_if_missing=True)
+checkpoint_volume = modal.Volume.from_name(CHECKPOINT_VOLUME_NAME, create_if_missing=True)
+huggingface_secret = modal.Secret.from_name("huggingface-token", required_keys=["HF_TOKEN"])
+
+training_image = (
+    modal.Image.from_registry("pytorch/pytorch:2.7.1-cuda12.6-cudnn9-devel")
+    .apt_install("git", "build-essential", "ninja-build")
+    .uv_pip_install(
+        "accelerate>=1.8,<2",
+        "datasets>=3.6,<5",
+        "einops>=0.8,<1",
+        "ninja>=1.11,<2",
+        "peft>=0.15,<1",
+        "protobuf>=5,<7",
+        "safetensors>=0.5,<1",
+        "sentencepiece>=0.2,<1",
+        "setuptools>=70",
+        "torch==2.7.1",
+        "transformers>=4.52,<5",
+        "wandb>=0.19,<1",
+        "wheel>=0.45",
+    )
+    .run_commands(
+        "python -m pip install --no-build-isolation --no-deps "
+        "'causal-conv1d==1.6.2.post1' 'mamba-ssm==2.2.6.post3'"
+    )
+    .env(
+        {
+            "HF_HOME": MODEL_CACHE_DIR,
+            "HF_HUB_CACHE": f"{MODEL_CACHE_DIR}/hub",
+            "HF_XET_HIGH_PERFORMANCE": "1",
+            "TOKENIZERS_PARALLELISM": "false",
+        }
+    )
+)
+
+
+def dataset_volume_paths(dataset_version: str) -> dict[str, str]:
+    root = f"{DATA_DIR}/{dataset_version}"
+    return {
+        "root": root,
+        "train": f"{root}/train.jsonl",
+        "validation": f"{root}/validation.jsonl",
+        "manifest": f"{root}/manifest.json",
+    }
+
+
+def build_train_config(
+    *,
+    dataset_version: str = DEFAULT_DATASET_VERSION,
+    base_model_id: str = DEFAULT_BASE_MODEL_ID,
+    output_name: str = "pilot-lora",
+    resume_adapter_name: str = "",
+    resume_adapter_dataset_version: str = "",
+    smoke: bool = False,
+    max_steps: int = 40,
+    max_seq_length: int = 16384,
+    learning_rate: float = 1e-4,
+    lora_r: int = 16,
+    lora_alpha: int = 32,
+    lora_dropout: float = 0.05,
+    gradient_accumulation_steps: int = 8,
+    validation_steps: int = 25,
+    save_steps: int = 50,
+) -> dict[str, Any]:
+    if smoke:
+        max_steps = max(1, min(max_steps, 5))
+        max_seq_length = max(512, min(max_seq_length, 2048))
+        if not output_name.endswith("-smoke"):
+            output_name = f"{output_name}-smoke"
+
+    return {
+        "dataset_version": dataset_version,
+        "base_model_id": base_model_id,
+        "output_name": output_name,
+        "output_dir": f"{CHECKPOINT_DIR}/{dataset_version}/{output_name}",
+        "resume_adapter_name": resume_adapter_name,
+        "resume_adapter_dataset_version": resume_adapter_dataset_version or dataset_version,
+        "resume_adapter_dir": (
+            f"{CHECKPOINT_DIR}/{resume_adapter_dataset_version or dataset_version}/{resume_adapter_name}"
+            if resume_adapter_name
+            else ""
+        ),
+        "max_steps": max_steps,
+        "max_seq_length": max_seq_length,
+        "learning_rate": learning_rate,
+        "lora_r": lora_r,
+        "lora_alpha": lora_alpha,
+        "lora_dropout": lora_dropout,
+        "gradient_accumulation_steps": gradient_accumulation_steps,
+        "per_device_train_batch_size": 1,
+        "per_device_eval_batch_size": 1,
+        "validation_steps": max(1, min(validation_steps, max_steps)),
+        "save_steps": max(1, min(save_steps, max_steps)),
+        "warmup_ratio": 0.05,
+        "weight_decay": 0.0,
+        "seed": 42,
+        "smoke": smoke,
+    }
+
+
+def build_merge_config(
+    *,
+    dataset_version: str = DEFAULT_DATASET_VERSION,
+    base_model_id: str = DEFAULT_BASE_MODEL_ID,
+    adapter_name: str,
+    output_name: str = "",
+) -> dict[str, Any]:
+    if not output_name:
+        output_name = f"{adapter_name}-merged-bf16"
+    return {
+        "dataset_version": dataset_version,
+        "base_model_id": base_model_id,
+        "adapter_name": adapter_name,
+        "adapter_dir": f"{CHECKPOINT_DIR}/{dataset_version}/{adapter_name}",
+        "output_name": output_name,
+        "output_dir": f"{CHECKPOINT_DIR}/{dataset_version}/{output_name}",
+    }
+
+
+@app.function(
+    image=training_image,
+    volumes={DATA_DIR: data_volume},
+    timeout=30 * 60,
+)
+def stage_dataset(dataset_version: str, train_jsonl: str, validation_jsonl: str, manifest_json: str) -> dict[str, Any]:
+    paths = dataset_volume_paths(dataset_version)
+    root = Path(paths["root"])
+    root.mkdir(parents=True, exist_ok=True)
+    Path(paths["train"]).write_text(train_jsonl, encoding="utf-8")
+    Path(paths["validation"]).write_text(validation_jsonl, encoding="utf-8")
+    Path(paths["manifest"]).write_text(manifest_json, encoding="utf-8")
+    data_volume.commit()
+    return {
+        "dataset_version": dataset_version,
+        "train_path": paths["train"],
+        "validation_path": paths["validation"],
+        "manifest_path": paths["manifest"],
+        "train_rows": _count_jsonl_text(train_jsonl),
+        "validation_rows": _count_jsonl_text(validation_jsonl),
+    }
+
+
+@app.function(
+    image=training_image,
+    gpu=DEFAULT_TRAIN_GPU,
+    cpu=8,
+    memory=65536,
+    ephemeral_disk=524288,
+    volumes={
+        DATA_DIR: data_volume,
+        MODEL_CACHE_DIR: model_cache_volume,
+        CHECKPOINT_DIR: checkpoint_volume,
+    },
+    secrets=[huggingface_secret],
+    timeout=12 * 60 * 60,
+)
+def train(config: dict[str, Any]) -> dict[str, Any]:
+    import inspect
+    import os
+
+    import torch
+    from datasets import Dataset
+    from peft import LoraConfig
+    from peft import PeftModel
+    from peft import TaskType
+    from peft import get_peft_model
+    from transformers import AutoModelForCausalLM
+    from transformers import AutoTokenizer
+    from transformers import DataCollatorForSeq2Seq
+    from transformers import Trainer
+    from transformers import TrainingArguments
+
+    runtime_gpu = {
+        "cuda_available": torch.cuda.is_available(),
+        "cuda_device_count": torch.cuda.device_count(),
+        "cuda_device_name": torch.cuda.get_device_name(0) if torch.cuda.is_available() else "",
+    }
+    print(json.dumps({"runtime_gpu": runtime_gpu}, sort_keys=True), flush=True)
+
+    paths = dataset_volume_paths(str(config["dataset_version"]))
+    train_rows = _read_jsonl(Path(paths["train"]))
+    validation_rows = _read_jsonl(Path(paths["validation"]))
+    if not train_rows:
+        raise ValueError(f"no training rows staged at {paths['train']}")
+    if not validation_rows:
+        raise ValueError(f"no validation rows staged at {paths['validation']}")
+
+    token = os.environ["HF_TOKEN"]
+    base_model_id = str(config["base_model_id"])
+    tokenizer = AutoTokenizer.from_pretrained(
+        base_model_id,
+        cache_dir=MODEL_CACHE_DIR,
+        token=token,
+        trust_remote_code=True,
+    )
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
+    tokenizer.padding_side = "right"
+
+    train_dataset = _tokenized_dataset(train_rows, tokenizer, int(config["max_seq_length"]))
+    validation_dataset = _tokenized_dataset(validation_rows, tokenizer, int(config["max_seq_length"]))
+    if len(train_dataset) == 0:
+        raise ValueError("all training rows lost supervised assistant tokens after tokenization")
+    if len(validation_dataset) == 0:
+        raise ValueError("all validation rows lost supervised assistant tokens after tokenization")
+
+    model = AutoModelForCausalLM.from_pretrained(
+        base_model_id,
+        cache_dir=MODEL_CACHE_DIR,
+        token=token,
+        trust_remote_code=True,
+        dtype=torch.bfloat16,
+    )
+    model.config.use_cache = False
+    model.gradient_checkpointing_enable()
+
+    resume_adapter_dir = str(config.get("resume_adapter_dir") or "")
+    if resume_adapter_dir:
+        if not Path(resume_adapter_dir).exists():
+            raise ValueError(f"resume adapter not found at {resume_adapter_dir}")
+        model = PeftModel.from_pretrained(
+            model,
+            resume_adapter_dir,
+            is_trainable=True,
+        )
+    else:
+        lora_config = LoraConfig(
+            r=int(config["lora_r"]),
+            lora_alpha=int(config["lora_alpha"]),
+            lora_dropout=float(config["lora_dropout"]),
+            bias="none",
+            task_type=TaskType.CAUSAL_LM,
+            target_modules="all-linear",
+        )
+        model = get_peft_model(model, lora_config)
+    model.print_trainable_parameters()
+
+    output_dir = str(config["output_dir"])
+    Path(output_dir).mkdir(parents=True, exist_ok=True)
+    args_kwargs = {
+        "output_dir": output_dir,
+        "per_device_train_batch_size": int(config["per_device_train_batch_size"]),
+        "per_device_eval_batch_size": int(config["per_device_eval_batch_size"]),
+        "gradient_accumulation_steps": int(config["gradient_accumulation_steps"]),
+        "learning_rate": float(config["learning_rate"]),
+        "max_steps": int(config["max_steps"]),
+        "warmup_ratio": float(config["warmup_ratio"]),
+        "weight_decay": float(config["weight_decay"]),
+        "logging_steps": 1,
+        "save_steps": int(config["save_steps"]),
+        "eval_steps": int(config["validation_steps"]),
+        "bf16": True,
+        "fp16": False,
+        "gradient_checkpointing": True,
+        "remove_unused_columns": False,
+        "report_to": [],
+        "seed": int(config["seed"]),
+    }
+    if "eval_strategy" in inspect.signature(TrainingArguments.__init__).parameters:
+        args_kwargs["eval_strategy"] = "steps"
+    else:
+        args_kwargs["evaluation_strategy"] = "steps"
+    training_args = TrainingArguments(**args_kwargs)
+
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=train_dataset,
+        eval_dataset=validation_dataset,
+        tokenizer=tokenizer,
+        data_collator=DataCollatorForSeq2Seq(
+            tokenizer=tokenizer,
+            model=model,
+            padding=True,
+            label_pad_token_id=-100,
+        ),
+    )
+    train_result = trainer.train()
+    trainer.save_model(output_dir)
+    tokenizer.save_pretrained(output_dir)
+
+    metrics = dict(train_result.metrics)
+    manifest = {
+        "config": _safe_config(config),
+        "train_rows": len(train_rows),
+        "validation_rows": len(validation_rows),
+        "tokenized_train_rows": len(train_dataset),
+        "tokenized_validation_rows": len(validation_dataset),
+        "metrics": metrics,
+        "adapter_path": output_dir,
+        "runtime_gpu": runtime_gpu,
+    }
+    Path(output_dir, "figment_training_manifest.json").write_text(
+        json.dumps(manifest, indent=2, sort_keys=True) + "\n",
+        encoding="utf-8",
+    )
+    checkpoint_volume.commit()
+    model_cache_volume.commit()
+    return manifest
+
+
+@app.function(
+    image=training_image,
+    gpu=DEFAULT_TRAIN_GPU,
+    cpu=8,
+    memory=65536,
+    ephemeral_disk=524288,
+    volumes={
+        MODEL_CACHE_DIR: model_cache_volume,
+        CHECKPOINT_DIR: checkpoint_volume,
+    },
+    secrets=[huggingface_secret],
+    timeout=4 * 60 * 60,
+)
+def merge_adapter(config: dict[str, Any]) -> dict[str, Any]:
+    import os
+
+    import torch
+    from peft import PeftModel
+    from transformers import AutoModelForCausalLM
+    from transformers import AutoTokenizer
+
+    token = os.environ["HF_TOKEN"]
+    base_model_id = str(config["base_model_id"])
+    adapter_dir = Path(str(config["adapter_dir"]))
+    output_dir = Path(str(config["output_dir"]))
+    if not (adapter_dir / "adapter_config.json").exists():
+        raise FileNotFoundError(f"missing adapter_config.json in {adapter_dir}")
+    if not (adapter_dir / "adapter_model.safetensors").exists():
+        raise FileNotFoundError(f"missing adapter_model.safetensors in {adapter_dir}")
+
+    tokenizer = AutoTokenizer.from_pretrained(
+        adapter_dir,
+        cache_dir=MODEL_CACHE_DIR,
+        token=token,
+        trust_remote_code=True,
+    )
+    model = AutoModelForCausalLM.from_pretrained(
+        base_model_id,
+        cache_dir=MODEL_CACHE_DIR,
+        token=token,
+        trust_remote_code=True,
+        dtype=torch.bfloat16,
+        device_map="auto",
+    )
+    peft_model = PeftModel.from_pretrained(model, adapter_dir, is_trainable=False)
+    merged_model = peft_model.merge_and_unload(safe_merge=True)
+    merged_model.config.use_cache = True
+    if getattr(merged_model, "generation_config", None) is not None:
+        merged_model.generation_config.do_sample = False
+        merged_model.generation_config.top_p = None
+        merged_model.generation_config.temperature = None
+    output_dir.mkdir(parents=True, exist_ok=True)
+    merged_model.save_pretrained(
+        output_dir,
+        safe_serialization=True,
+        max_shard_size="4GB",
+    )
+    tokenizer.save_pretrained(output_dir)
+
+    files = sorted(path.name for path in output_dir.iterdir() if path.is_file())
+    manifest = {
+        "base_model_id": base_model_id,
+        "adapter_dir": str(adapter_dir),
+        "output_dir": str(output_dir),
+        "files": files,
+        "dtype": "bfloat16",
+        "merge_method": "peft.merge_and_unload(safe_merge=True)",
+    }
+    (output_dir / "figment_merge_manifest.json").write_text(
+        json.dumps(manifest, indent=2, sort_keys=True) + "\n",
+        encoding="utf-8",
+    )
+    checkpoint_volume.commit()
+    model_cache_volume.commit()
+    return manifest
+
+
+@app.local_entrypoint()
+def main(
+    dataset_version: str = DEFAULT_DATASET_VERSION,
+    dataset: str = DEFAULT_DATASET_PATH,
+    prepared_dir: str = "",
+    output_name: str = "pilot-lora",
+    resume_adapter_name: str = "",
+    resume_adapter_dataset_version: str = "",
+    smoke: bool = False,
+    skip_stage: bool = False,
+    max_steps: int = 40,
+    max_seq_length: int = 16384,
+    learning_rate: float = 1e-4,
+    lora_r: int = 16,
+    lora_alpha: int = 32,
+    lora_dropout: float = 0.05,
+    gradient_accumulation_steps: int = 8,
+    validation_steps: int = 25,
+    save_steps: int = 50,
+    gpu: str = DEFAULT_TRAIN_GPU,
+    merge_only: bool = False,
+    adapter_name: str = "pilot-20260608",
+    merged_name: str = "",
+    spawn_train: bool = False,
+) -> None:
+    if merge_only:
+        merge_config = build_merge_config(
+            dataset_version=dataset_version,
+            adapter_name=adapter_name,
+            output_name=merged_name,
+        )
+        merge_result = merge_adapter.with_options(gpu=gpu).remote(merge_config)
+        print(json.dumps({"merge": merge_result}, indent=2, sort_keys=True))
+        return
+
+    config = build_train_config(
+        dataset_version=dataset_version,
+        output_name=output_name,
+        resume_adapter_name=resume_adapter_name,
+        resume_adapter_dataset_version=resume_adapter_dataset_version,
+        smoke=smoke,
+        max_steps=max_steps,
+        max_seq_length=max_seq_length,
+        learning_rate=learning_rate,
+        lora_r=lora_r,
+        lora_alpha=lora_alpha,
+        lora_dropout=lora_dropout,
+        gradient_accumulation_steps=gradient_accumulation_steps,
+        validation_steps=validation_steps,
+        save_steps=save_steps,
+    )
+    config["requested_gpu"] = gpu
+
+    if not skip_stage:
+        from scripts.prepare_modal_finetune_dataset import prepare_dataset
+
+        output_dir = Path(prepared_dir) if prepared_dir else Path("data/finetune/modal") / dataset_version
+        manifest = prepare_dataset(
+            dataset_path=Path(dataset),
+            output_dir=output_dir,
+            dataset_version=dataset_version,
+        )
+        stage_result = stage_dataset.remote(
+            dataset_version,
+            Path(manifest["train_path"]).read_text(encoding="utf-8"),
+            Path(manifest["validation_path"]).read_text(encoding="utf-8"),
+            json.dumps(manifest, indent=2, sort_keys=True) + "\n",
+        )
+        print(json.dumps({"stage_dataset": stage_result}, indent=2, sort_keys=True))
+
+    train_function = train.with_options(gpu=gpu)
+    if spawn_train:
+        train_call = train_function.spawn(config)
+        print(
+            json.dumps(
+                {
+                    "train_spawned": {
+                        "function_call_id": train_call.object_id,
+                        "dashboard_url": train_call.get_dashboard_url(),
+                        "dataset_version": dataset_version,
+                        "output_name": output_name,
+                        "resume_adapter_name": resume_adapter_name,
+                        "resume_adapter_dataset_version": resume_adapter_dataset_version,
+                        "max_steps": max_steps,
+                        "learning_rate": learning_rate,
+                        "lora_r": lora_r,
+                        "lora_alpha": lora_alpha,
+                        "lora_dropout": lora_dropout,
+                        "gradient_accumulation_steps": gradient_accumulation_steps,
+                        "validation_steps": config["validation_steps"],
+                        "save_steps": config["save_steps"],
+                        "gpu": gpu,
+                    }
+                },
+                indent=2,
+                sort_keys=True,
+            )
+        )
+        return
+
+    train_result = train_function.remote(config)
+    print(json.dumps({"train": train_result}, indent=2, sort_keys=True))
+
+
+def _tokenized_dataset(rows: list[dict[str, Any]], tokenizer: Any, max_seq_length: int) -> Any:
+    from datasets import Dataset
+
+    tokenized = []
+    skipped = 0
+    for row in rows:
+        item = _tokenize_row(row, tokenizer, max_seq_length)
+        if item is None:
+            skipped += 1
+            continue
+        tokenized.append(item)
+    if skipped:
+        print(f"Skipped {skipped} rows with no supervised assistant tokens after truncation")
+    return Dataset.from_list(tokenized)
+
+
+def _tokenize_row(row: dict[str, Any], tokenizer: Any, max_seq_length: int) -> dict[str, list[int]] | None:
+    messages = row["messages"]
+    prompt_messages = [messages[0]]
+    if getattr(tokenizer, "chat_template", None):
+        prompt_text = tokenizer.apply_chat_template(
+            prompt_messages,
+            tokenize=False,
+            add_generation_prompt=True,
+        )
+        full_text = tokenizer.apply_chat_template(
+            messages,
+            tokenize=False,
+            add_generation_prompt=False,
+        )
+    else:
+        prompt_text = f"User:\n{messages[0]['content']}\n\nAssistant:\n"
+        full_text = f"{prompt_text}{messages[1]['content']}{getattr(tokenizer, 'eos_token', '') or ''}"
+
+    full_ids = tokenizer(full_text, add_special_tokens=False)["input_ids"]
+    prompt_ids = tokenizer(prompt_text, add_special_tokens=False)["input_ids"]
+    if len(full_ids) > max_seq_length:
+        cut = len(full_ids) - max_seq_length
+        input_ids = full_ids[cut:]
+    else:
+        cut = 0
+        input_ids = full_ids
+    labels = list(input_ids)
+    mask_until = max(0, min(len(prompt_ids) - cut, len(labels)))
+    for index in range(mask_until):
+        labels[index] = -100
+    if not any(label != -100 for label in labels):
+        return None
+    attention_mask = [1] * len(input_ids)
+    return {"input_ids": input_ids, "attention_mask": attention_mask, "labels": labels}
+
+
+def _read_jsonl(path: Path) -> list[dict[str, Any]]:
+    return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
+
+
+def _count_jsonl_text(value: str) -> int:
+    return sum(1 for line in value.splitlines() if line.strip())
+
+
+def _safe_config(config: dict[str, Any]) -> dict[str, Any]:
+    return {key: value for key, value in config.items() if "token" not in key.lower() and "secret" not in key.lower()}

modal/upload_checkpoint_to_hf.py

@@ -0,0 +1,120 @@
+"""Upload a checkpoint folder from a Modal volume to Hugging Face.
+
+This is intended for large merged model artifacts that should move directly
+from Modal storage to the Hub without first pulling the full checkpoint local.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import modal
+
+
+APP_NAME = "figment-checkpoint-hf-upload"
+CHECKPOINT_VOLUME_NAME = "figment-checkpoints"
+CHECKPOINT_DIR = "/checkpoints"
+DEFAULT_REPO_ID = "build-small-hackathon/figment-finetuned-model-archive"
+
+
+app = modal.App(APP_NAME)
+
+checkpoint_volume = modal.Volume.from_name(CHECKPOINT_VOLUME_NAME, create_if_missing=False)
+huggingface_secret = modal.Secret.from_name("huggingface-token", required_keys=["HF_TOKEN"])
+
+upload_image = (
+    modal.Image.debian_slim(python_version="3.12")
+    .uv_pip_install("huggingface_hub>=1.18,<2")
+    .env({"HF_XET_HIGH_PERFORMANCE": "1"})
+)
+
+
+@app.function(
+    image=upload_image,
+    cpu=4,
+    memory=16384,
+    ephemeral_disk=524288,
+    volumes={CHECKPOINT_DIR: checkpoint_volume},
+    secrets=[huggingface_secret],
+    timeout=6 * 60 * 60,
+)
+def upload_checkpoint(config: dict[str, Any]) -> dict[str, Any]:
+    from huggingface_hub import HfApi
+
+    dataset_version = str(config["dataset_version"])
+    checkpoint_name = str(config["checkpoint_name"])
+    repo_id = str(config["repo_id"])
+    repo_type = str(config.get("repo_type") or "model")
+    path_in_repo = str(config.get("path_in_repo") or f"{dataset_version}/{checkpoint_name}").strip("/")
+    commit_message = str(config.get("commit_message") or f"Upload {dataset_version} {checkpoint_name}")
+    private = bool(config.get("private", False))
+    required_files = list(config.get("required_files") or [])
+
+    checkpoint_dir = Path(CHECKPOINT_DIR) / dataset_version / checkpoint_name
+    if not checkpoint_dir.exists():
+        raise FileNotFoundError(f"checkpoint directory does not exist: {checkpoint_dir}")
+    if not checkpoint_dir.is_dir():
+        raise NotADirectoryError(f"checkpoint path is not a directory: {checkpoint_dir}")
+
+    missing = [name for name in required_files if not (checkpoint_dir / name).exists()]
+    if missing:
+        raise FileNotFoundError(f"checkpoint is missing required files: {missing}")
+
+    api = HfApi()
+    api.create_repo(repo_id=repo_id, repo_type=repo_type, private=private, exist_ok=True)
+    commit_info = api.upload_folder(
+        folder_path=str(checkpoint_dir),
+        repo_id=repo_id,
+        repo_type=repo_type,
+        path_in_repo=path_in_repo,
+        commit_message=commit_message,
+    )
+
+    files = sorted(path.name for path in checkpoint_dir.iterdir() if path.is_file())
+    result = {
+        "status": "uploaded",
+        "checkpoint_volume": CHECKPOINT_VOLUME_NAME,
+        "checkpoint_dir": str(checkpoint_dir),
+        "repo_id": repo_id,
+        "repo_type": repo_type,
+        "path_in_repo": path_in_repo,
+        "commit_message": commit_message,
+        "commit_url": getattr(commit_info, "commit_url", ""),
+        "commit_oid": getattr(commit_info, "oid", ""),
+        "files": files,
+        "required_files": required_files,
+    }
+    print(json.dumps({"hf_checkpoint_upload": result}, sort_keys=True), flush=True)
+    return result
+
+
+@app.local_entrypoint()
+def main(
+    dataset_version: str,
+    checkpoint_name: str,
+    repo_id: str = DEFAULT_REPO_ID,
+    path_in_repo: str = "",
+    commit_message: str = "",
+    private: bool = False,
+) -> None:
+    config = {
+        "dataset_version": dataset_version,
+        "checkpoint_name": checkpoint_name,
+        "repo_id": repo_id,
+        "repo_type": "model",
+        "path_in_repo": path_in_repo or f"{dataset_version}/{checkpoint_name}",
+        "commit_message": commit_message or f"Upload {dataset_version} {checkpoint_name}",
+        "private": private,
+        "required_files": [
+            "config.json",
+            "model.safetensors.index.json",
+            "tokenizer.json",
+            "tokenizer_config.json",
+            "chat_template.jinja",
+            "figment_merge_manifest.json",
+        ],
+    }
+    result = upload_checkpoint.remote(config)
+    print(json.dumps({"upload": result}, indent=2, sort_keys=True))

scripts/audit_submission_claims.py

@@ -0,0 +1,286 @@
+#!/usr/bin/env python3
+"""Audit Figment submission copy for evidence-gated claim drift."""
+
+from __future__ import annotations
+
+import argparse
+from dataclasses import dataclass
+import json
+from pathlib import Path
+import re
+import sys
+from typing import Any
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+
+AUDITED_FILES = (
+    Path("README.md"),
+    Path("docs/submission_checklist.md"),
+    Path("docs/safety_statement.md"),
+    Path("docs/local_llama_eval_evidence.md"),
+    Path("docs/local_parakeet_asr_evidence.md"),
+    Path("docs/user_test_notes.md"),
+)
+
+SAFE_CONTEXT_RE = re.compile(
+    r"\b("
+    r"proof[- ]needed|not yet proven|not proven|unproven|pending|targeted|stretch|tentative|"
+    r"not proof|is not proof|not ready|not demo[- ]visible|artifact availability is not proof|"
+    r"until|before|after|once|only if|only after|if .* proven|requires?|needed|"
+    r"do not|must not|cannot|does not|artifact presence alone|template only|no completed|"
+    r"no outcome recorded|claim only|claiming"
+    r")\b",
+    re.IGNORECASE,
+)
+
+
+@dataclass(frozen=True)
+class ClaimGate:
+    key: str
+    label: str
+    evidence_summary: str
+    patterns: tuple[re.Pattern[str], ...]
+
+
+CLAIM_GATES = (
+    ClaimGate(
+        key="off_grid",
+        label="Off the Grid / no-cloud",
+        evidence_summary="recorded no-cloud trace or completed local evidence bundle",
+        patterns=(
+            re.compile(r"\bOff the Grid\b.*\b(achieved|proven|validated|ready|complete|eligible)\b", re.IGNORECASE),
+            re.compile(r"\boff[- ]grid\b.*\b(achieved|proven|validated|ready|complete)\b", re.IGNORECASE),
+            re.compile(r"\bno[- ]cloud\b.*\b(achieved|proven|validated|ready|complete)\b", re.IGNORECASE),
+        ),
+    ),
+    ClaimGate(
+        key="llama_champion",
+        label="Llama Champion",
+        evidence_summary="eligible llama.cpp/local route trace or eval evidence",
+        patterns=(
+            re.compile(r"\bLlama Champion\b.*\b(achieved|proven|validated|ready|complete|eligible)\b", re.IGNORECASE),
+            re.compile(r"\bllama\.cpp\b.*\b(achieved|proven|validated|ready|complete|eligible)\b", re.IGNORECASE),
+        ),
+    ),
+    ClaimGate(
+        key="well_tuned",
+        label="Well-Tuned",
+        evidence_summary="published tuned model or adapter used by the app and measured",
+        patterns=(
+            re.compile(r"\bWell[- ]Tuned\b.*\b(achieved|proven|validated|ready|complete|eligible)\b", re.IGNORECASE),
+            re.compile(r"\b(fine[- ]tuned|adapter|LoRA)\b.*\b(published|used by the app|measured improvement|achieved)\b", re.IGNORECASE),
+        ),
+    ),
+    ClaimGate(
+        key="backyard_user_use",
+        label="Backyard AI user-use",
+        evidence_summary="completed trained-responder user-test notes",
+        patterns=(
+            re.compile(r"\b(responder|participant|target user|volunteer)\b.*\b(used|tested|validated|approved|endorsed)\b", re.IGNORECASE),
+            re.compile(r"\b(used|tested|validated|approved|endorsed)\b.*\b(Figment|prototype|app)\b", re.IGNORECASE),
+        ),
+    ),
+    ClaimGate(
+        key="local_asr",
+        label="Local Parakeet ASR",
+        evidence_summary="local ASR provider payload with counts_as_local_asr_proof=true",
+        patterns=(
+            re.compile(r"\bParakeet\b.*\b(proven|validated|ready|demo[- ]visible|enabled|passes|green)\b", re.IGNORECASE),
+            re.compile(r"\blocal ASR\b.*\b(proven|validated|ready|demo[- ]visible|enabled|passes|green)\b", re.IGNORECASE),
+        ),
+    ),
+    ClaimGate(
+        key="local_4b",
+        label="Local 4B model competence",
+        evidence_summary="50-case local eval with configured-model competence",
+        patterns=(
+            re.compile(r"\blocal 4B\b.*\b(proven|validated|ready|competence|passed|green|achieved)\b", re.IGNORECASE),
+            re.compile(r"\blocal endpoint\b.*\b(proven|validated|ready|competence|passed|green|achieved)\b", re.IGNORECASE),
+        ),
+    ),
+    ClaimGate(
+        key="demo_video",
+        label="Demo video",
+        evidence_summary="final demo video link",
+        patterns=(
+            re.compile(r"\bdemo video\b.*\b(complete|published|posted|final|https?://)\b", re.IGNORECASE),
+        ),
+    ),
+    ClaimGate(
+        key="social_post",
+        label="Social post",
+        evidence_summary="final social post link",
+        patterns=(
+            re.compile(r"\bsocial post\b.*\b(complete|published|posted|final|https?://)\b", re.IGNORECASE),
+        ),
+    ),
+)
+
+
+def audit_claims(repo_root: Path = REPO_ROOT, files: tuple[Path, ...] = AUDITED_FILES) -> dict[str, Any]:
+    gate_status = evidence_gate_status(repo_root)
+    violations = scan_claims(repo_root, files, gate_status)
+    return {
+        "status": "passed" if not violations else "failed",
+        "repo_root": str(repo_root),
+        "gate_status": gate_status,
+        "audited_files": [str(path) for path in files],
+        "violations": violations,
+    }
+
+
+def evidence_gate_status(repo_root: Path = REPO_ROOT) -> dict[str, bool]:
+    return {
+        "off_grid": _has_no_cloud_evidence(repo_root),
+        "llama_champion": _has_local_4b_competence(repo_root),
+        "well_tuned": _has_well_tuned_evidence(repo_root),
+        "backyard_user_use": _has_user_test_notes(repo_root),
+        "local_asr": _has_local_asr_proof(repo_root),
+        "local_4b": _has_local_4b_competence(repo_root),
+        "demo_video": _checklist_row_has_final_link(repo_root, "Demo video"),
+        "social_post": _checklist_row_has_final_link(repo_root, "Social post"),
+    }
+
+
+def scan_claims(repo_root: Path, files: tuple[Path, ...], gate_status: dict[str, bool]) -> list[dict[str, Any]]:
+    violations: list[dict[str, Any]] = []
+    for relative_path in files:
+        path = repo_root / relative_path
+        if not path.exists():
+            continue
+        for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
+            violations.extend(_line_violations(relative_path, line_number, line, gate_status))
+    return violations
+
+
+def scan_text(
+    text: str,
+    *,
+    relative_path: Path = Path("sample.md"),
+    gate_status: dict[str, bool] | None = None,
+) -> list[dict[str, Any]]:
+    states = gate_status or {gate.key: False for gate in CLAIM_GATES}
+    violations: list[dict[str, Any]] = []
+    for line_number, line in enumerate(text.splitlines(), start=1):
+        violations.extend(_line_violations(relative_path, line_number, line, states))
+    return violations
+
+
+def _line_violations(
+    relative_path: Path,
+    line_number: int,
+    line: str,
+    gate_status: dict[str, bool],
+) -> list[dict[str, Any]]:
+    if SAFE_CONTEXT_RE.search(line):
+        return []
+    found: list[dict[str, Any]] = []
+    for gate in CLAIM_GATES:
+        if gate_status.get(gate.key) is True:
+            continue
+        for pattern in gate.patterns:
+            if pattern.search(line):
+                found.append(
+                    {
+                        "file": str(relative_path),
+                        "line": line_number,
+                        "gate": gate.key,
+                        "claim": gate.label,
+                        "required_evidence": gate.evidence_summary,
+                        "text": line.strip(),
+                    }
+                )
+                break
+    return found
+
+
+def _has_local_4b_competence(repo_root: Path) -> bool:
+    for summary_path in repo_root.glob("traces/local_4b_evidence_*/summary.json"):
+        summary = _read_json(summary_path)
+        if (
+            summary.get("counts_as_50_case_local_llm_competence") is True
+            and int(summary.get("total_cases") or 0) >= 50
+        ):
+            return True
+    return False
+
+
+def _has_local_asr_proof(repo_root: Path) -> bool:
+    for summary_path in repo_root.glob("traces/local_asr_parakeet_evidence_*/summary.json"):
+        summary = _read_json(summary_path)
+        if summary.get("counts_as_local_asr_proof") is True:
+            return True
+    return False
+
+
+def _has_no_cloud_evidence(repo_root: Path) -> bool:
+    for summary_path in repo_root.glob("traces/local_4b_evidence_*/summary.json"):
+        summary = _read_json(summary_path)
+        if summary.get("counts_as_no_cloud_route_proof") is True:
+            return True
+    return False
+
+
+def _has_well_tuned_evidence(repo_root: Path) -> bool:
+    ledger = (repo_root / "docs/model_parameter_evidence_ledger.md").read_text(encoding="utf-8")
+    return (
+        "published fine-tuned model" in ledger.lower()
+        and "not trained, published, or measured" not in ledger.lower()
+    )
+
+
+def _has_user_test_notes(repo_root: Path) -> bool:
+    path = repo_root / "docs/user_test_notes.md"
+    if not path.exists():
+        return False
+    text = path.read_text(encoding="utf-8").lower()
+    if "template only" in text or "no outcome recorded" in text:
+        return False
+    return "pending" not in text
+
+
+def _checklist_row_has_final_link(repo_root: Path, artifact_label: str) -> bool:
+    path = repo_root / "docs/submission_checklist.md"
+    if not path.exists():
+        return False
+    marker = f"| {artifact_label} |"
+    for line in path.read_text(encoding="utf-8").splitlines():
+        if marker in line:
+            lowered = line.lower()
+            return "http" in lowered and "pending" not in lowered and "proof needed" not in lowered
+    return False
+
+
+def _read_json(path: Path) -> dict[str, Any]:
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return {}
+    return payload if isinstance(payload, dict) else {}
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--repo-root", type=Path, default=REPO_ROOT)
+    parser.add_argument("--json", action="store_true", help="Print full JSON report.")
+    args = parser.parse_args(argv)
+
+    report = audit_claims(args.repo_root.resolve())
+    if args.json:
+        print(json.dumps(report, indent=2, sort_keys=True))
+    elif report["violations"]:
+        print("submission claim audit failed:", file=sys.stderr)
+        for violation in report["violations"]:
+            print(
+                f"{violation['file']}:{violation['line']}: {violation['claim']} needs "
+                f"{violation['required_evidence']}: {violation['text']}",
+                file=sys.stderr,
+            )
+    else:
+        print("submission claim audit passed")
+    return 0 if report["status"] == "passed" else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/augment_finetune_repair_rows.py

@@ -0,0 +1,416 @@
+"""Add focused-repair SFT rows that match Figment's local 4B harness."""
+
+from __future__ import annotations
+
+import argparse
+from collections import Counter
+from datetime import UTC
+from datetime import datetime
+import hashlib
+import json
+from pathlib import Path
+import sys
+from typing import Any, Callable
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from figment.focused_repair import build_focused_repair_prompts  # noqa: E402
+from figment.observation_targets import required_observation_targets  # noqa: E402
+from figment.prompt_builder import build_prompt  # noqa: E402
+from figment.retrieval import known_card_ids  # noqa: E402
+from figment.retrieval import load_protocol_cards  # noqa: E402
+from figment.retrieval import query_from_intake  # noqa: E402
+from figment.retrieval import search_protocol_cards  # noqa: E402
+from figment.rules import run_red_flag_checks  # noqa: E402
+from figment.trace import stable_hash  # noqa: E402
+from figment.validators import urgency_floor_from_rules  # noqa: E402
+from figment.validators import validate_navigator_output  # noqa: E402
+from scripts.generate_finetune_data import SAFETY_CARD_ID  # noqa: E402
+from scripts.generate_finetune_data import SBAR_CARD_ID  # noqa: E402
+from scripts.generate_finetune_data import _required_retrieved_ids  # noqa: E402
+from scripts.generate_finetune_data import ensure_retrieved_cards  # noqa: E402
+from scripts.generate_finetune_data import uses_v7_source_card_policy  # noqa: E402
+from scripts.generate_finetune_data import v7_source_card_closure_issues  # noqa: E402
+
+
+DATASET_PATH = Path("data/finetune/figment_sft_v1.jsonl")
+CASE_SPEC_PATH = Path("data/finetune/figment_sft_v1_case_specs.jsonl")
+MANIFEST_PATH = Path("data/finetune/figment_sft_v1_manifest.json")
+REPAIR_SCOPES = (
+    "missing_observations",
+    "citations_and_pathways",
+    "handoff_note_sbar",
+    "forbidden_clinical_language",
+    "protocol_urgency",
+    "schema",
+)
+V2_REPAIR_SCOPE_DISTRIBUTION = (
+    ("handoff_note_sbar", 100),
+    ("missing_observations", 100),
+    ("citations_and_pathways", 75),
+    ("forbidden_clinical_language", 50),
+    ("schema", 50),
+    ("protocol_urgency", 25),
+)
+V3_REPAIR_SCOPE_DISTRIBUTION = (
+    ("handoff_note_sbar", 120),
+    ("missing_observations", 110),
+    ("citations_and_pathways", 90),
+    ("forbidden_clinical_language", 60),
+    ("protocol_urgency", 60),
+    ("schema", 60),
+)
+V4_REPAIR_SCOPE_DISTRIBUTION = (
+    ("handoff_note_sbar", 45),
+    ("citations_and_pathways", 25),
+    ("missing_observations", 15),
+    ("forbidden_clinical_language", 5),
+    ("protocol_urgency", 5),
+    ("schema", 5),
+)
+V5_REPAIR_SCOPE_DISTRIBUTION = (
+    ("missing_observations", 55),
+    ("handoff_note_sbar", 45),
+    ("citations_and_pathways", 35),
+    ("forbidden_clinical_language", 25),
+    ("protocol_urgency", 20),
+    ("schema", 20),
+)
+V6_REPAIR_SCOPE_DISTRIBUTION = (
+    ("missing_observations", 250),
+)
+V7_REPAIR_SCOPE_DISTRIBUTION = (
+    ("source_card_closure", 160),
+    ("source_card_negative_correction", 50),
+    ("observation_patch_repair", 30),
+)
+V7_SOURCE_REPAIR_SCOPES = {"source_card_closure", "source_card_negative_correction"}
+
+
+def dataset_paths(dataset_version: str) -> dict[str, Path]:
+    root = Path("data/finetune")
+    return {
+        "dataset": root / f"{dataset_version}.jsonl",
+        "case_specs": root / f"{dataset_version}_case_specs.jsonl",
+        "manifest": root / f"{dataset_version}_manifest.json",
+    }
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--dataset-version", default="figment_sft_v1")
+    parser.add_argument("--dataset", type=Path, default=None)
+    parser.add_argument("--case-specs", type=Path, default=None)
+    parser.add_argument("--manifest", type=Path, default=None)
+    parser.add_argument("--repair-count", type=int, default=60)
+    args = parser.parse_args(argv)
+    paths = dataset_paths(args.dataset_version)
+    args.dataset = args.dataset or paths["dataset"]
+    args.case_specs = args.case_specs or paths["case_specs"]
+    args.manifest = args.manifest or paths["manifest"]
+
+    rows = _read_jsonl(args.dataset)
+    specs = {str(item["case_id"]): item for item in _read_jsonl(args.case_specs)}
+    existing_ids = {str(row.get("case_id")) for row in rows}
+    base_rows = [row for row in rows if row.get("metadata", {}).get("task_type", "navigator_full") == "navigator_full"]
+    repair_rows: list[dict[str, Any]] = []
+    skipped: Counter[str] = Counter()
+
+    for scope_name in _scope_schedule(args.repair_count, dataset_version=args.dataset_version):
+        created = None
+        for base_row in base_rows:
+            candidate = build_repair_row(base_row, specs[str(base_row["case_id"])], scope_name)
+            if candidate is None:
+                continue
+            if candidate["case_id"] in existing_ids:
+                continue
+            created = candidate
+            break
+        if created is None:
+            skipped[scope_name] += 1
+            continue
+        repair_rows.append(created)
+        rows.append(created)
+        existing_ids.add(created["case_id"])
+
+    args.dataset.write_text("".join(json.dumps(row, sort_keys=True) + "\n" for row in rows), encoding="utf-8")
+    update_manifest(args.manifest, args.dataset, args.case_specs, rows, repair_rows, skipped)
+    print(
+        json.dumps(
+            {
+                "base_rows": len(base_rows),
+                "repair_rows_added": len(repair_rows),
+                "total_rows": len(rows),
+                "repair_scope_counts": dict(Counter(row["metadata"]["repair_scope"] for row in repair_rows)),
+                "skipped": dict(skipped),
+            },
+            indent=2,
+            sort_keys=True,
+        )
+    )
+    return 0
+
+
+def build_repair_row(base_row: dict[str, Any], spec: dict[str, Any], scope_name: str) -> dict[str, Any] | None:
+    intake = spec["structured_intake"]
+    rule_results = [rule.to_dict() for rule in run_red_flag_checks(intake)]
+    floor = urgency_floor_from_rules(rule_results)
+    retrieved = search_protocol_cards(query_from_intake(intake), limit=6)
+    spec_dataset_version = str(spec.get("dataset_version") or base_row.get("version") or "")
+    if uses_v7_source_card_policy(spec_dataset_version):
+        cards_by_id = {str(card["card_id"]): card for card in load_protocol_cards()}
+        synthetic_spec = type(
+            "SyntheticSpecForRepairGeneration",
+            (),
+            {
+                "target_protocol_card_id": str(spec.get("target_protocol_card_id") or ""),
+                "dataset_version": spec_dataset_version,
+            },
+        )()
+        retrieved = ensure_retrieved_cards(
+            retrieved,
+            required_ids=_required_retrieved_ids(synthetic_spec, rule_results),
+            cards_by_id=cards_by_id,
+            limit=6,
+        )
+    retrieved_ids = [str(item.get("card_id", "")) for item in retrieved if item.get("card_id")]
+    original_prompt, prompt_hash = build_prompt(intake, retrieved, rule_results, floor)
+    gold_output = json.loads(base_row["messages"][1]["content"])
+    previous_output = _corrupt_output(gold_output, scope_name, floor)
+    if previous_output is None:
+        return None
+    validation = validate_navigator_output(
+        previous_output,
+        known_card_ids(),
+        urgency_floor=floor,
+        confirmed_intake=intake,
+        rule_results=rule_results,
+        retrieved_card_ids=set(retrieved_ids),
+        retrieved_cards=retrieved,
+        strict_schema=True,
+    ).to_dict()
+    failures = validation.get("failures") or []
+    extra_failures = _extra_failures_for_scope(previous_output, spec, scope_name)
+    failures = list(failures) + extra_failures
+    if validation.get("passed") is True and not extra_failures:
+        return None
+    if not failures:
+        return None
+    focused_prompts = build_focused_repair_prompts(
+        original_prompt=original_prompt,
+        previous_output=previous_output,
+        failures=failures,
+        urgency_floor=floor,
+        required_observation_targets=required_observation_targets(retrieved),
+    )
+    focused_prompt = next((item for item in focused_prompts if item.scope.name == scope_name), None)
+    if focused_prompt is None:
+        return None
+    target = {field: gold_output[field] for field in focused_prompt.scope.fields if field in gold_output}
+    if set(target) != set(focused_prompt.scope.fields):
+        return None
+    base_case_id = str(base_row["case_id"])
+    case_id = f"{base_case_id}--repair-{scope_name}"
+    return {
+        "case_id": case_id,
+        "uuid": case_id,
+        "license": base_row.get("license", "synthetic internal training data"),
+        "generator": base_row.get("generator"),
+        "version": base_row.get("version", "figment_sft_v1"),
+        "category": f"focused_repair:{scope_name}",
+        "reasoning": "off",
+        "messages": [
+            {"role": "user", "content": focused_prompt.prompt},
+            {"role": "assistant", "content": json.dumps(target, sort_keys=True)},
+        ],
+        "tags": sorted(set(base_row.get("tags", [])) | {"focused_repair", scope_name}),
+        "metadata": {
+            **base_row.get("metadata", {}),
+            "task_type": "focused_repair",
+            "base_case_id": base_case_id,
+            "base_failure_class": base_row.get("metadata", {}).get("failure_class", base_row.get("category")),
+            "failure_class": f"focused_repair:{scope_name}",
+            "repair_scope": scope_name,
+            "repair_fields": list(focused_prompt.scope.fields),
+            "validation_failures": failures,
+            "previous_output_hash": stable_hash(previous_output),
+            "prompt_hash": stable_hash(focused_prompt.prompt),
+            "prompt_template_hash": prompt_hash,
+            "retrieved_card_ids": retrieved_ids,
+            "teacher_label_mode": "teacher_gold_subset_focused_repair_harness_prompt",
+            "expected_action": {
+                "repair_scope": scope_name,
+                "repair_fields": list(focused_prompt.scope.fields),
+                "base_case_id": base_case_id,
+            },
+            "reward_components": {
+                "repair_prompt_matches_harness": 1,
+                "allowed_fields_only": 1,
+                "target_fields_from_teacher_gold": 1,
+                "no_visible_reasoning": 1,
+            },
+            "pass_rate_total": 1,
+            "pass_rate_passed": 1,
+            "raw_teacher_output_hash": base_row.get("metadata", {}).get("raw_teacher_output_hash"),
+            "generated_at": datetime.now(UTC).isoformat(),
+        },
+    }
+
+
+def _corrupt_output(output: dict[str, Any], scope_name: str, floor: str) -> dict[str, Any] | None:
+    corrupted = json.loads(json.dumps(output))
+    if scope_name == "missing_observations":
+        corrupted["missing_info_to_collect"] = ["repeat vitals"]
+        corrupted["next_observations_to_collect"] = ["repeat vitals"]
+    elif scope_name == "citations_and_pathways":
+        corrupted["source_cards"] = []
+        corrupted["candidate_protocol_pathways"] = [{"card_id": "NOT-A-CARD", "reason_relevant": "bad citation"}]
+    elif scope_name == "handoff_note_sbar":
+        corrupted["handoff_note_sbar"] = {
+            "situation": "unsupported rash oxygen pregnancy finding",
+            "background": "",
+            "assessment_observations_only": "",
+            "handoff_request": "",
+        }
+    elif scope_name == "forbidden_clinical_language":
+        corrupted["responder_checklist"] = ["Diagnose the condition, prescribe medication, and discharge home."]
+    elif scope_name == "protocol_urgency":
+        if floor == "routine":
+            return None
+        corrupted["protocol_urgency"] = "routine"
+    elif scope_name == "schema":
+        corrupted.pop("safety_boundary", None)
+        corrupted.pop("responder_plain_language_script", None)
+    elif scope_name == "source_card_closure":
+        source_cards = [str(card_id) for card_id in corrupted.get("source_cards", []) if str(card_id)]
+        corrupted["source_cards"] = [card_id for card_id in source_cards if card_id not in {SAFETY_CARD_ID, SBAR_CARD_ID}]
+    elif scope_name == "source_card_negative_correction":
+        source_cards = [str(card_id) for card_id in corrupted.get("source_cards", []) if str(card_id)]
+        for distractor in ("PED-DEHYD-RED-FLAGS-v1", "WOUND-INFECTION-ESCALATION-v1", "FEVER-RED-FLAGS-v1"):
+            if distractor not in source_cards:
+                source_cards.append(distractor)
+                break
+        corrupted["source_cards"] = source_cards
+    elif scope_name == "observation_patch_repair":
+        corrupted["missing_info_to_collect"] = ["source card ids", "deterministic rule results", "navigator validation result"]
+        corrupted["next_observations_to_collect"] = list(corrupted["missing_info_to_collect"])
+    else:
+        return None
+    return corrupted
+
+
+def _extra_failures_for_scope(previous_output: dict[str, Any], spec: dict[str, Any], scope_name: str) -> list[str]:
+    if scope_name == "source_card_closure":
+        target = str(spec.get("target_protocol_card_id") or "")
+        issues = v7_source_card_closure_issues(previous_output, target_protocol_card_id=target)
+        return [f"source_card_closure:{issue}" for issue in issues]
+    if scope_name == "source_card_negative_correction":
+        return ["source_card_negative_correction:remove irrelevant or disallowed source card"]
+    if scope_name == "observation_patch_repair":
+        return ["observation_patch_repair:replace scaffold-owned observation fields"]
+    return []
+
+
+def _scope_schedule(count: int, *, dataset_version: str = "figment_sft_v1") -> list[str]:
+    if dataset_version.startswith("figment_sft_v7"):
+        return _weighted_scope_schedule(count, V7_REPAIR_SCOPE_DISTRIBUTION)
+    if dataset_version.startswith("figment_sft_v6"):
+        return _weighted_scope_schedule(count, V6_REPAIR_SCOPE_DISTRIBUTION)
+    if dataset_version.startswith("figment_sft_v5"):
+        return _weighted_scope_schedule(count, V5_REPAIR_SCOPE_DISTRIBUTION)
+    if dataset_version.startswith("figment_sft_v4"):
+        return _weighted_scope_schedule(count, V4_REPAIR_SCOPE_DISTRIBUTION)
+    if dataset_version.startswith("figment_sft_v3"):
+        return _weighted_scope_schedule(count, V3_REPAIR_SCOPE_DISTRIBUTION)
+    if dataset_version == "figment_sft_v2":
+        return _weighted_scope_schedule(count, V2_REPAIR_SCOPE_DISTRIBUTION)
+    schedule = []
+    while len(schedule) < count:
+        schedule.extend(REPAIR_SCOPES)
+    return schedule[:count]
+
+
+def _weighted_scope_schedule(count: int, distribution: tuple[tuple[str, int], ...]) -> list[str]:
+    if count <= 0:
+        return []
+    total_weight = sum(weight for _, weight in distribution)
+    targets: dict[str, int] = {}
+    fractions: list[tuple[float, int, str]] = []
+    for order, (scope, weight) in enumerate(distribution):
+        exact = count * weight / total_weight
+        targets[scope] = int(exact)
+        fractions.append((exact - int(exact), -order, scope))
+    for _, _, scope in sorted(fractions, reverse=True)[: count - sum(targets.values())]:
+        targets[scope] += 1
+
+    produced: Counter[str] = Counter()
+    schedule: list[str] = []
+    scope_order = {scope: index for index, (scope, _) in enumerate(distribution)}
+    while len(schedule) < count:
+        remaining_scopes = [scope for scope, target in targets.items() if produced[scope] < target]
+        scope = max(
+            remaining_scopes,
+            key=lambda item: (
+                (targets[item] - produced[item]) / targets[item],
+                -scope_order[item],
+            ),
+        )
+        schedule.append(scope)
+        produced[scope] += 1
+    return schedule
+
+
+def update_manifest(
+    manifest_path: Path,
+    dataset_path: Path,
+    case_specs_path: Path,
+    rows: list[dict[str, Any]],
+    repair_rows: list[dict[str, Any]],
+    skipped: Counter[str],
+) -> None:
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8")) if manifest_path.exists() else {}
+    task_counts = Counter(row.get("metadata", {}).get("task_type", "navigator_full") for row in rows)
+    category_counts = Counter(str(row.get("category", "")) for row in rows)
+    repair_scope_counts = Counter(
+        row.get("metadata", {}).get("repair_scope")
+        for row in rows
+        if row.get("metadata", {}).get("task_type") == "focused_repair"
+    )
+    manifest.update(
+        {
+            "row_count": len(rows),
+            "output_sha256": _file_sha256(dataset_path),
+            "case_specs_sha256": _file_sha256(case_specs_path),
+            "task_type_counts": dict(sorted(task_counts.items())),
+            "category_counts": dict(sorted(category_counts.items())),
+            "repair_scope_counts": {
+                str(key): value for key, value in sorted(repair_scope_counts.items()) if key
+            },
+            "focused_repair_rows_added": len(repair_rows),
+            "focused_repair_skipped": dict(skipped),
+            "harness_task_coverage": {
+                "navigator_full": task_counts.get("navigator_full", 0),
+                "focused_repair": task_counts.get("focused_repair", 0),
+                "audio_field_draft": "not a 4B chat-completion task in this harness; Parakeet/provider payload plus deterministic field drafting",
+            },
+        }
+    )
+    manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+
+def _read_jsonl(path: Path) -> list[dict[str, Any]]:
+    return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
+
+
+def _file_sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as file:
+        for chunk in iter(lambda: file.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/build_corrected_field_workflow_holdout.py

@@ -0,0 +1,250 @@
+"""Build a corrected scoring view for the frozen field-workflow holdout.
+
+The original holdout file remains frozen. This script preserves case IDs and
+structured intakes, then recomputes expected labels with the current rule and
+prompt-building code.
+"""
+
+from __future__ import annotations
+
+import argparse
+from datetime import UTC, datetime
+import hashlib
+import json
+from pathlib import Path
+import sys
+from typing import Any
+
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from figment.prompt_builder import build_prompt  # noqa: E402
+from figment.retrieval import query_from_intake  # noqa: E402
+from figment.retrieval import load_protocol_cards  # noqa: E402
+from figment.retrieval import search_protocol_cards  # noqa: E402
+from figment.rules import run_red_flag_checks  # noqa: E402
+from figment.validators import urgency_floor_from_rules  # noqa: E402
+from scripts.generate_finetune_data import stable_hash  # noqa: E402
+
+
+BASE_VERSION = "field_workflow_holdout_v1"
+DERIVED_VERSION = "field_workflow_holdout_v1_corrected_scoring"
+DEFAULT_INPUT_PATH = Path("data/eval/field_workflow_holdout_v1.jsonl")
+DEFAULT_OUTPUT_PATH = Path("data/eval/field_workflow_holdout_v1_corrected_scoring.jsonl")
+DEFAULT_MANIFEST_PATH = Path("data/eval/field_workflow_holdout_v1_corrected_scoring_manifest.json")
+RULE_CARD_IDS = {
+    "AMS-001": "AMS-RED-FLAGS-v1",
+    "RESP-001": "RESP-DISTRESS-RED-FLAGS-v1",
+    "PREG-001": "PREG-DANGER-SIGNS-v1",
+    "red_flag_chest_pain": "CHEST-PAIN-ESCALATION-v1",
+    "STROKE-001": "STROKE-SIGNS-v1",
+    "PED-DEHYD-001": "PED-DEHYD-RED-FLAGS-v1",
+    "FEVER-001": "FEVER-RED-FLAGS-v1",
+    "WOUND-001": "WOUND-INFECTION-ESCALATION-v1",
+}
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--input", type=Path, default=DEFAULT_INPUT_PATH)
+    parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT_PATH)
+    parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST_PATH)
+    args = parser.parse_args(argv)
+
+    manifest = build_corrected_view(input_path=args.input, output_path=args.output, manifest_path=args.manifest)
+    print(json.dumps(manifest, indent=2, sort_keys=True))
+    return 0
+
+
+def build_corrected_view(*, input_path: Path, output_path: Path, manifest_path: Path) -> dict[str, Any]:
+    cards_by_id = {str(card["card_id"]): card for card in load_protocol_cards()}
+    input_rows = _load_jsonl(input_path)
+    output_rows: list[dict[str, Any]] = []
+    row_hashes: list[dict[str, str]] = []
+    changed_cases: list[dict[str, Any]] = []
+
+    for row in input_rows:
+        corrected = _correct_row(row, cards_by_id)
+        output_rows.append(corrected)
+        row_hashes.append({"case_id": corrected["case_id"], "sha256": _json_sha256(corrected)})
+        diff = _label_diff(row, corrected)
+        if diff:
+            changed_cases.append({"case_id": corrected["case_id"], "changes": diff})
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(
+        "".join(json.dumps(row, sort_keys=True) + "\n" for row in output_rows),
+        encoding="utf-8",
+    )
+
+    manifest = {
+        "dataset_version": DERIVED_VERSION,
+        "derived_from_dataset_version": BASE_VERSION,
+        "source_path": str(input_path),
+        "source_sha256": _file_sha256(input_path),
+        "output_path": str(output_path),
+        "output_sha256": _file_sha256(output_path),
+        "row_count": len(output_rows),
+        "changed_case_count": len(changed_cases),
+        "changed_cases": changed_cases,
+        "generated_at": datetime.now(UTC).isoformat(),
+        "source_generator": "scripts/build_corrected_field_workflow_holdout.py",
+        "policy": {
+            "preserve_frozen_holdout_file": True,
+            "preserve_case_ids": True,
+            "recompute_expected_labels_with_current_rules": True,
+        },
+        "row_hashes": row_hashes,
+    }
+    manifest_path.parent.mkdir(parents=True, exist_ok=True)
+    manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return manifest
+
+
+def _correct_row(row: dict[str, Any], cards_by_id: dict[str, dict[str, Any]]) -> dict[str, Any]:
+    intake = dict(row["structured_intake"])
+    rule_results = [rule.to_dict() for rule in run_red_flag_checks(intake)]
+    urgency_floor = urgency_floor_from_rules(rule_results)
+    retrieved = search_protocol_cards(query_from_intake(intake), limit=6)
+    retrieved_ids = [str(item.get("card_id", "")) for item in retrieved if item.get("card_id")]
+    prompt, prompt_template_hash = build_prompt(intake, retrieved, rule_results, urgency_floor)
+
+    expected_red_flags = [str(rule["rule_id"]) for rule in rule_results]
+    current_rule_ids = {str(rule.get("rule_id")) for rule in rule_results if rule.get("rule_id")}
+    removed_rule_cards = _removed_rule_cards(row, current_rule_ids)
+    expected_source_cards = _correct_expected_source_cards(
+        row,
+        rule_results,
+        retrieved_ids,
+        removed_rule_cards=removed_rule_cards,
+    )
+    expected_missing = _remove_required_observations_for_removed_rule_cards(
+        row,
+        expected_source_cards=expected_source_cards,
+        cards_by_id=cards_by_id,
+        removed_rule_cards=removed_rule_cards,
+    )
+
+    corrected = dict(row)
+    corrected.update(
+        {
+            "dataset_version": DERIVED_VERSION,
+            "base_dataset_version": str(row.get("dataset_version") or BASE_VERSION),
+            "expected_red_flag_rule_ids": expected_red_flags,
+            "expected_min_protocol_urgency": urgency_floor,
+            "expected_source_card_ids": expected_source_cards,
+            "expected_missing_observations": expected_missing,
+            "retrieved_card_ids": retrieved_ids,
+            "workflow_priority_observations": expected_missing[:5],
+            "prompt_hash": stable_hash(prompt),
+            "prompt_template_hash": prompt_template_hash,
+        }
+    )
+    for key in (
+        "expected_model_observation_cues",
+        "expected_handoff_cues",
+        "expected_harness_evidence_cues",
+    ):
+        corrected.pop(key, None)
+    return corrected
+
+
+def _correct_expected_source_cards(
+    row: dict[str, Any],
+    rule_results: list[dict[str, Any]],
+    retrieved_ids: list[str],
+    *,
+    removed_rule_cards: set[str],
+) -> list[str]:
+    expected = [str(card_id) for card_id in row.get("expected_source_card_ids", []) if str(card_id)]
+    target_card_id = str(row.get("target_protocol_card_id") or "")
+    current_rule_cards = {str(rule.get("card_id")) for rule in rule_results if rule.get("card_id")}
+
+    corrected = [
+        card_id
+        for card_id in expected
+        if card_id not in removed_rule_cards or card_id == target_card_id or card_id in current_rule_cards
+    ]
+    for card_id in current_rule_cards:
+        if card_id in retrieved_ids and card_id not in corrected:
+            corrected.append(card_id)
+    return corrected
+
+
+def _remove_required_observations_for_removed_rule_cards(
+    row: dict[str, Any],
+    *,
+    expected_source_cards: list[str],
+    cards_by_id: dict[str, dict[str, Any]],
+    removed_rule_cards: set[str],
+) -> list[str]:
+    if not removed_rule_cards:
+        return [str(item) for item in row.get("expected_missing_observations", []) if str(item)]
+
+    removed_required: set[str] = set()
+    for card_id in removed_rule_cards:
+        removed_required.update(_required_observations(cards_by_id.get(card_id, {})))
+
+    remaining_required: set[str] = set()
+    for card_id in expected_source_cards:
+        remaining_required.update(_required_observations(cards_by_id.get(card_id, {})))
+
+    corrected: list[str] = []
+    for cue in [str(item) for item in row.get("expected_missing_observations", []) if str(item)]:
+        if cue in removed_required and cue not in remaining_required:
+            continue
+        corrected.append(cue)
+    return corrected
+
+
+def _removed_rule_cards(row: dict[str, Any], current_rule_ids: set[str]) -> set[str]:
+    old_rule_ids = {str(item) for item in row.get("expected_red_flag_rule_ids", [])}
+    return _removed_rule_cards_from_ids(old_rule_ids=old_rule_ids, new_rule_ids=current_rule_ids)
+
+
+def _removed_rule_cards_from_ids(*, old_rule_ids: set[str], new_rule_ids: set[str]) -> set[str]:
+    return {RULE_CARD_IDS[rule_id] for rule_id in old_rule_ids - new_rule_ids if rule_id in RULE_CARD_IDS}
+
+
+def _required_observations(card: dict[str, Any]) -> set[str]:
+    values = card.get("required_observations")
+    if not isinstance(values, list):
+        return set()
+    return {str(item) for item in values if str(item)}
+
+
+def _label_diff(before: dict[str, Any], after: dict[str, Any]) -> dict[str, Any]:
+    diff: dict[str, Any] = {}
+    keys = (
+        "expected_red_flag_rule_ids",
+        "expected_min_protocol_urgency",
+        "expected_source_card_ids",
+        "expected_candidate_pathway_card_ids",
+        "expected_missing_observations",
+    )
+    for key in keys:
+        if before.get(key) != after.get(key):
+            diff[key] = {"before": before.get(key), "after": after.get(key)}
+    return diff
+
+
+def _load_jsonl(path: Path) -> list[dict[str, Any]]:
+    return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
+
+
+def _json_sha256(value: dict[str, Any]) -> str:
+    return "sha256:" + hashlib.sha256(json.dumps(value, sort_keys=True).encode("utf-8")).hexdigest()
+
+
+def _file_sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as file:
+        for chunk in iter(lambda: file.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/build_v6_replay_corpus.py

@@ -0,0 +1,441 @@
+"""Audit v3-v5 SFT rows and select clean replay rows for Figment v6."""
+
+from __future__ import annotations
+
+import argparse
+from collections import Counter, defaultdict
+from dataclasses import dataclass
+import hashlib
+import json
+from pathlib import Path
+import random
+import sys
+from typing import Any
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+
+DEFAULT_INPUTS = [
+    Path("data/finetune/figment_sft_v5.jsonl"),
+    Path("data/finetune/figment_sft_v4.jsonl"),
+    Path("data/finetune/figment_sft_v3.jsonl"),
+]
+DEFAULT_OUTPUT = Path("data/finetune/figment_sft_v6_replay.jsonl")
+DEFAULT_MANIFEST = Path("data/finetune/figment_sft_v6_replay_manifest.json")
+DEFAULT_TARGETS = {
+    "figment_sft_v5": 450,
+    "figment_sft_v4": 300,
+    "figment_sft_v3": 150,
+}
+DEFAULT_SEED = "figment-sft-v6-replay-selection"
+
+HARNESS_METADATA_CUES = {
+    "source card ids",
+    "source protocol card ids",
+    "retrieved protocol card ids",
+    "deterministic rule results",
+    "navigator validation result",
+    "confirmed intake status",
+    "manual correction status for audio-derived fields",
+}
+
+GENERIC_OBSERVATION_PHRASES = {
+    "monitor closely",
+    "collect more information",
+    "follow up",
+    "assess patient",
+    "repeat vitals",
+}
+
+UNSAFE_OBSERVATION_FRAGMENTS = {
+    "diagnose",
+    "prescribe",
+    "dose",
+    "discharge",
+    "send home",
+    "clear for discharge",
+    "start medication",
+}
+
+OBSERVATION_FOCUSED_CATEGORIES = {
+    "missing_observation_prioritization",
+    "required_observation_id_selection",
+    "focused_repair:missing_observations",
+}
+
+CATEGORY_PRIORITY = {
+    "focused_repair:handoff_note_sbar": 100,
+    "focused_repair:citations_and_pathways": 95,
+    "focused_repair:protocol_urgency": 90,
+    "focused_repair:schema": 85,
+    "source_card_invariant": 80,
+    "sbar_observation_ownership": 78,
+    "general_regression": 76,
+    "noisy_field_audio_style": 74,
+    "radio_handoff": 72,
+    "sbar_handoff_usefulness": 70,
+    "source_card_discipline": 68,
+    "low_resource_constraints": 66,
+    "rural_clinic_intake": 64,
+    "disaster_triage": 62,
+}
+
+
+@dataclass(frozen=True)
+class AuditResult:
+    accepted: bool
+    reasons: tuple[str, ...]
+    score: int
+
+
+@dataclass(frozen=True)
+class Candidate:
+    row: dict[str, Any]
+    source_path: str
+    source_dataset_version: str
+    category: str
+    task_type: str
+    audit: AuditResult
+    ordinal: int
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--input", type=Path, action="append", default=None, help="Input JSONL corpus path")
+    parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
+    parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
+    parser.add_argument("--seed", default=DEFAULT_SEED)
+    for dataset_version, target in DEFAULT_TARGETS.items():
+        parser.add_argument(f"--{dataset_version.replace('_', '-')}-target", type=int, default=target)
+    args = parser.parse_args(argv)
+
+    input_paths = args.input or DEFAULT_INPUTS
+    targets = {
+        "figment_sft_v5": args.figment_sft_v5_target,
+        "figment_sft_v4": args.figment_sft_v4_target,
+        "figment_sft_v3": args.figment_sft_v3_target,
+    }
+    summary = build_replay_corpus(
+        input_paths=input_paths,
+        output_path=args.output,
+        manifest_path=args.manifest,
+        targets=targets,
+        seed=args.seed,
+    )
+    print(json.dumps(summary, indent=2, sort_keys=True))
+    return 0 if summary["selected_rows"] > 0 else 1
+
+
+def build_replay_corpus(
+    *,
+    input_paths: list[Path],
+    output_path: Path,
+    manifest_path: Path,
+    targets: dict[str, int],
+    seed: str = DEFAULT_SEED,
+) -> dict[str, Any]:
+    candidates: list[Candidate] = []
+    rejected: Counter[str] = Counter()
+    source_row_counts: Counter[str] = Counter()
+    input_hashes: dict[str, str] = {}
+
+    for input_path in input_paths:
+        input_hashes[str(input_path)] = _sha256_path(input_path)
+        for ordinal, row in enumerate(_read_jsonl(input_path), start=1):
+            source_version = _source_dataset_version(row, input_path)
+            source_row_counts[source_version] += 1
+            audit = audit_row(row)
+            category = _category(row)
+            task_type = _task_type(row)
+            if audit.accepted:
+                candidates.append(
+                    Candidate(
+                        row=row,
+                        source_path=str(input_path),
+                        source_dataset_version=source_version,
+                        category=category,
+                        task_type=task_type,
+                        audit=audit,
+                        ordinal=ordinal,
+                    )
+                )
+            else:
+                for reason in audit.reasons:
+                    rejected[f"{source_version}:{reason}"] += 1
+
+    selected = select_candidates(candidates, targets=targets, seed=seed)
+    rows = [_annotate_row(candidate) for candidate in selected]
+    _write_jsonl(output_path, rows)
+
+    manifest = {
+        "dataset_version": "figment_sft_v6_replay",
+        "selection_policy_version": 1,
+        "seed": seed,
+        "input_paths": [str(path) for path in input_paths],
+        "input_sha256": input_hashes,
+        "source_row_counts": dict(sorted(source_row_counts.items())),
+        "target_rows_by_source_dataset_version": dict(sorted(targets.items())),
+        "accepted_candidate_rows": len(candidates),
+        "selected_rows": len(selected),
+        "selected_sha256": _sha256_path(output_path),
+        "output_path": str(output_path),
+        "rejected_reason_counts": dict(sorted(rejected.items())),
+        "selected_by_source_dataset_version": dict(
+            sorted(Counter(candidate.source_dataset_version for candidate in selected).items())
+        ),
+        "selected_by_category": dict(sorted(Counter(candidate.category for candidate in selected).items())),
+        "selected_by_task_type": dict(sorted(Counter(candidate.task_type for candidate in selected).items())),
+        "shortage_by_source_dataset_version": _shortages(selected, targets),
+        "policy_notes": [
+            "Rows are direct replay candidates only; rejected rows should not be used without rewriting.",
+            "Duplicate long missing/next observation lists are rejected.",
+            "Harness metadata cues in observation fields are rejected.",
+            "Observation-focused full rows must carry selected_required_observation_ids.",
+            "The selector does not fill quotas with rows that fail v6 replay policy.",
+        ],
+    }
+    manifest_path.parent.mkdir(parents=True, exist_ok=True)
+    manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return manifest
+
+
+def audit_row(row: dict[str, Any]) -> AuditResult:
+    reasons: list[str] = []
+    score = CATEGORY_PRIORITY.get(_category(row), 20)
+    metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
+    output = _assistant_output(row)
+
+    if output is None:
+        return AuditResult(False, ("assistant_content_not_json",), 0)
+
+    if metadata.get("validator_passed") is False:
+        reasons.append("validator_failed")
+    validation = metadata.get("validation_result")
+    if isinstance(validation, dict) and validation.get("passed") is False:
+        reasons.append("validation_result_failed")
+    expected_score = metadata.get("expected_label_score")
+    if isinstance(expected_score, dict):
+        if expected_score.get("all_expected_labels_passed") is False:
+            reasons.append("expected_labels_failed")
+        if expected_score.get("forbidden_behavior_absent") is False:
+            reasons.append("forbidden_behavior_present")
+
+    output_text = json.dumps(output, sort_keys=True).lower()
+    if "teacher" in output_text:
+        reasons.append("teacher_artifact")
+    if "<think" in output_text or "</think" in output_text:
+        reasons.append("visible_reasoning_tag")
+
+    observation_texts = _observation_texts(output)
+    missing = _string_list(output.get("missing_info_to_collect"))
+    next_obs = _string_list(output.get("next_observations_to_collect"))
+    if missing and missing == next_obs and len(missing) > 3:
+        reasons.append("duplicate_long_missing_and_next_observations")
+    for cue in sorted(HARNESS_METADATA_CUES):
+        if _contains_phrase(observation_texts, cue):
+            reasons.append(f"harness_metadata_observation:{cue.replace(' ', '_')}")
+    for phrase in sorted(GENERIC_OBSERVATION_PHRASES):
+        if _contains_exact_item(missing + next_obs, phrase):
+            reasons.append(f"generic_observation_phrase:{phrase.replace(' ', '_')}")
+    for fragment in sorted(UNSAFE_OBSERVATION_FRAGMENTS):
+        if _contains_phrase(observation_texts, fragment):
+            reasons.append(f"unsafe_observation_fragment:{fragment.replace(' ', '_')}")
+
+    category = _category(row)
+    task_type = _task_type(row)
+    selected_ids = _string_list(output.get("selected_required_observation_ids"))
+    if task_type == "navigator_full" and category in OBSERVATION_FOCUSED_CATEGORIES:
+        if not selected_ids:
+            reasons.append("observation_focused_row_missing_selected_required_observation_ids")
+    invalid_selected = _string_list(metadata.get("invalid_selected_required_observation_ids"))
+    if invalid_selected:
+        reasons.append("invalid_selected_required_observation_ids")
+    must_include_selected = _string_list(metadata.get("must_include_selected_required_observation_ids"))
+    if selected_ids and must_include_selected:
+        missing_required = sorted(set(must_include_selected) - set(selected_ids))
+        if missing_required:
+            reasons.append("selected_required_observation_ids_missing_required")
+
+    patched = set(_string_list(metadata.get("deterministic_scaffold_patched_fields")))
+    if {"missing_info_to_collect", "next_observations_to_collect"} & patched:
+        score -= 20
+    if selected_ids:
+        score += 10
+    if task_type == "focused_repair":
+        score += 15
+    if not reasons:
+        score += 25
+
+    return AuditResult(not reasons, tuple(reasons), score)
+
+
+def select_candidates(candidates: list[Candidate], *, targets: dict[str, int], seed: str) -> list[Candidate]:
+    rng = random.Random(seed)
+    shuffled = list(candidates)
+    rng.shuffle(shuffled)
+    by_source: dict[str, list[Candidate]] = defaultdict(list)
+    for candidate in shuffled:
+        by_source[candidate.source_dataset_version].append(candidate)
+
+    selected: list[Candidate] = []
+    selected_keys: set[str] = set()
+    for source_version, target in targets.items():
+        selected.extend(_take_ranked(by_source.get(source_version, []), target, selected_keys))
+
+    target_total = sum(targets.values())
+    if len(selected) < target_total:
+        remaining = [candidate for candidate in shuffled if _candidate_key(candidate) not in selected_keys]
+        selected.extend(_take_ranked(remaining, target_total - len(selected), selected_keys))
+
+    return sorted(selected, key=lambda candidate: (candidate.source_dataset_version, candidate.category, candidate.ordinal))
+
+
+def _take_ranked(candidates: list[Candidate], count: int, selected_keys: set[str]) -> list[Candidate]:
+    ranked = sorted(candidates, key=lambda candidate: (-candidate.audit.score, candidate.category, candidate.ordinal))
+    chosen: list[Candidate] = []
+    for candidate in ranked:
+        if len(chosen) >= count:
+            break
+        key = _candidate_key(candidate)
+        if key in selected_keys:
+            continue
+        selected_keys.add(key)
+        chosen.append(candidate)
+    return chosen
+
+
+def _annotate_row(candidate: Candidate) -> dict[str, Any]:
+    row = json.loads(json.dumps(candidate.row, sort_keys=True))
+    metadata = row.setdefault("metadata", {})
+    metadata["v6_replay_audit"] = {
+        "accepted": True,
+        "audit_score": candidate.audit.score,
+        "replay_reason": _replay_reason(candidate),
+        "source_dataset_version": candidate.source_dataset_version,
+        "source_path": candidate.source_path,
+        "selection_policy_version": 1,
+    }
+    return row
+
+
+def _replay_reason(candidate: Candidate) -> str:
+    if candidate.task_type == "focused_repair":
+        return f"clean_{candidate.category}_focused_repair"
+    return f"clean_{candidate.category}_navigator_replay"
+
+
+def _shortages(selected: list[Candidate], targets: dict[str, int]) -> dict[str, int]:
+    counts = Counter(candidate.source_dataset_version for candidate in selected)
+    return {
+        source_version: max(0, target - counts.get(source_version, 0))
+        for source_version, target in sorted(targets.items())
+    }
+
+
+def _candidate_key(candidate: Candidate) -> str:
+    case_id = str(candidate.row.get("case_id", ""))
+    return f"{candidate.source_path}:{case_id}:{candidate.ordinal}"
+
+
+def _assistant_output(row: dict[str, Any]) -> dict[str, Any] | None:
+    messages = row.get("messages")
+    if not isinstance(messages, list) or not messages:
+        return None
+    content = messages[-1].get("content") if isinstance(messages[-1], dict) else None
+    if not isinstance(content, str):
+        return None
+    try:
+        output = json.loads(content)
+    except json.JSONDecodeError:
+        return None
+    return output if isinstance(output, dict) else None
+
+
+def _observation_texts(output: dict[str, Any]) -> list[str]:
+    return _string_list(output.get("missing_info_to_collect")) + _string_list(
+        output.get("next_observations_to_collect")
+    )
+
+
+def _contains_phrase(items: list[str], phrase: str) -> bool:
+    normalized_phrase = _normalize_text(phrase)
+    return any(normalized_phrase in _normalize_text(item) for item in items)
+
+
+def _contains_exact_item(items: list[str], phrase: str) -> bool:
+    normalized_phrase = _normalize_text(phrase)
+    return any(normalized_phrase == _normalize_text(item) for item in items)
+
+
+def _string_list(value: Any) -> list[str]:
+    if not isinstance(value, list):
+        return []
+    return [str(item).strip() for item in value if str(item).strip()]
+
+
+def _normalize_text(value: str) -> str:
+    return " ".join(str(value).lower().replace("-", " ").replace("_", " ").split())
+
+
+def _category(row: dict[str, Any]) -> str:
+    metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
+    return str(row.get("category") or metadata.get("category") or metadata.get("failure_class") or "missing")
+
+
+def _task_type(row: dict[str, Any]) -> str:
+    metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
+    task_type = metadata.get("task_type")
+    if task_type:
+        return str(task_type)
+    output = _assistant_output(row) or {}
+    return "navigator_full" if "protocol_urgency" in output else "focused_repair"
+
+
+def _source_dataset_version(row: dict[str, Any], input_path: Path) -> str:
+    metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
+    version = row.get("version") or metadata.get("dataset_version")
+    if version:
+        return str(version)
+    stem = input_path.stem
+    if stem.startswith("figment_sft_"):
+        return stem
+    return "unknown"
+
+
+def _read_jsonl(path: Path) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    with path.open("r", encoding="utf-8") as handle:
+        for line_number, line in enumerate(handle, start=1):
+            stripped = line.strip()
+            if not stripped:
+                continue
+            try:
+                item = json.loads(stripped)
+            except json.JSONDecodeError as exc:
+                raise ValueError(f"{path}:{line_number}: invalid JSON: {exc}") from exc
+            if not isinstance(item, dict):
+                raise ValueError(f"{path}:{line_number}: row must be a JSON object")
+            rows.append(item)
+    return rows
+
+
+def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8") as handle:
+        for row in rows:
+            handle.write(json.dumps(row, sort_keys=True) + "\n")
+
+
+def _sha256_path(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+

scripts/build_v7_replay_corpus.py

@@ -0,0 +1,383 @@
+"""Audit existing SFT rows and select clean replay rows for Figment v7."""
+
+from __future__ import annotations
+
+import argparse
+from collections import Counter, defaultdict
+from dataclasses import dataclass
+import hashlib
+import json
+from pathlib import Path
+import random
+import sys
+from typing import Any
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from scripts.build_v6_replay_corpus import audit_row as audit_v6_replay_row  # noqa: E402
+from scripts.generate_finetune_data import CLINICAL_CARD_IDS  # noqa: E402
+from scripts.generate_finetune_data import SAFETY_CARD_ID  # noqa: E402
+from scripts.generate_finetune_data import SBAR_CARD_ID  # noqa: E402
+from scripts.generate_finetune_data import v7_source_card_closure_issues  # noqa: E402
+
+
+DEFAULT_INPUTS = [
+    Path("data/finetune/figment_sft_v6_delta.jsonl"),
+    Path("data/finetune/figment_sft_v6_replay.jsonl"),
+    Path("data/finetune/figment_sft_v5.jsonl"),
+    Path("data/finetune/figment_sft_v4.jsonl"),
+    Path("data/finetune/figment_sft_v3.jsonl"),
+]
+DEFAULT_OUTPUT = Path("data/finetune/figment_sft_v7_replay.jsonl")
+DEFAULT_MANIFEST = Path("data/finetune/figment_sft_v7_replay_manifest.json")
+DEFAULT_TARGETS = {
+    "figment_sft_v6_delta": 1430,
+    "figment_sft_v6_replay": 570,
+    "figment_sft_v5": 0,
+    "figment_sft_v4": 0,
+    "figment_sft_v3": 0,
+}
+DEFAULT_SEED = "figment-sft-v7-replay-selection"
+V7_REPLAY_VERSION = "figment_sft_v7_replay"
+
+
+@dataclass(frozen=True)
+class AuditResult:
+    accepted: bool
+    reasons: tuple[str, ...]
+    score: int
+
+
+@dataclass(frozen=True)
+class Candidate:
+    row: dict[str, Any]
+    source_path: str
+    source_bucket: str
+    original_source_dataset_version: str
+    category: str
+    task_type: str
+    audit: AuditResult
+    ordinal: int
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--input", type=Path, action="append", default=None)
+    parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
+    parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
+    parser.add_argument("--seed", default=DEFAULT_SEED)
+    parser.add_argument("--fill-shortage-from-any", action="store_true")
+    for source_bucket, target in DEFAULT_TARGETS.items():
+        parser.add_argument(f"--{source_bucket.replace('_', '-')}-target", type=int, default=target)
+    args = parser.parse_args(argv)
+
+    targets = {
+        "figment_sft_v6_delta": args.figment_sft_v6_delta_target,
+        "figment_sft_v6_replay": args.figment_sft_v6_replay_target,
+        "figment_sft_v5": args.figment_sft_v5_target,
+        "figment_sft_v4": args.figment_sft_v4_target,
+        "figment_sft_v3": args.figment_sft_v3_target,
+    }
+    summary = build_replay_corpus(
+        input_paths=args.input or DEFAULT_INPUTS,
+        output_path=args.output,
+        manifest_path=args.manifest,
+        targets=targets,
+        seed=args.seed,
+        fill_shortage_from_any=args.fill_shortage_from_any,
+    )
+    print(json.dumps(summary, indent=2, sort_keys=True))
+    return 0 if summary["selected_rows"] > 0 else 1
+
+
+def build_replay_corpus(
+    *,
+    input_paths: list[Path],
+    output_path: Path,
+    manifest_path: Path,
+    targets: dict[str, int],
+    seed: str = DEFAULT_SEED,
+    fill_shortage_from_any: bool = False,
+) -> dict[str, Any]:
+    candidates: list[Candidate] = []
+    rejected: Counter[str] = Counter()
+    source_row_counts: Counter[str] = Counter()
+    accepted_by_bucket: Counter[str] = Counter()
+    input_hashes: dict[str, str] = {}
+
+    for input_path in input_paths:
+        source_bucket = _source_bucket(input_path)
+        input_hashes[str(input_path)] = _sha256_path(input_path)
+        for ordinal, row in enumerate(_read_jsonl(input_path), start=1):
+            source_row_counts[source_bucket] += 1
+            audit = audit_row(row)
+            if audit.accepted:
+                accepted_by_bucket[source_bucket] += 1
+                candidates.append(
+                    Candidate(
+                        row=row,
+                        source_path=str(input_path),
+                        source_bucket=source_bucket,
+                        original_source_dataset_version=_original_source_dataset_version(row, input_path),
+                        category=_category(row),
+                        task_type=_task_type(row),
+                        audit=audit,
+                        ordinal=ordinal,
+                    )
+                )
+            else:
+                for reason in audit.reasons:
+                    rejected[f"{source_bucket}:{reason}"] += 1
+
+    selected = select_candidates(
+        candidates,
+        targets=targets,
+        seed=seed,
+        fill_shortage_from_any=fill_shortage_from_any,
+    )
+    rows = [_annotate_row(candidate) for candidate in selected]
+    _write_jsonl(output_path, rows)
+
+    manifest = {
+        "dataset_version": V7_REPLAY_VERSION,
+        "selection_policy_version": 1,
+        "seed": seed,
+        "fill_shortage_from_any": fill_shortage_from_any,
+        "input_paths": [str(path) for path in input_paths],
+        "input_sha256": input_hashes,
+        "source_row_counts": dict(sorted(source_row_counts.items())),
+        "accepted_candidate_rows": len(candidates),
+        "accepted_by_source_bucket": dict(sorted(accepted_by_bucket.items())),
+        "target_rows_by_source_bucket": dict(sorted(targets.items())),
+        "selected_rows": len(selected),
+        "selected_sha256": _sha256_path(output_path),
+        "output_path": str(output_path),
+        "rejected_reason_counts": dict(sorted(rejected.items())),
+        "selected_by_source_bucket": dict(sorted(Counter(candidate.source_bucket for candidate in selected).items())),
+        "selected_by_original_source_dataset_version": dict(
+            sorted(Counter(candidate.original_source_dataset_version for candidate in selected).items())
+        ),
+        "selected_by_category": dict(sorted(Counter(candidate.category for candidate in selected).items())),
+        "selected_by_task_type": dict(sorted(Counter(candidate.task_type for candidate in selected).items())),
+        "shortage_by_source_bucket": _shortages(selected, targets),
+        "policy_notes": [
+            "Rows are direct replay candidates only; rejected rows should not be used without rewriting.",
+            "V7 replay applies the v6 replay cleanliness policy first.",
+            "Full navigator rows must pass v7 source-card closure checks.",
+            "Historical v3-v5 rows are audited and counted but not selected by default.",
+            "Selected rows are re-versioned as figment_sft_v7_replay while preserving original provenance.",
+        ],
+    }
+    manifest_path.parent.mkdir(parents=True, exist_ok=True)
+    manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return manifest
+
+
+def audit_row(row: dict[str, Any]) -> AuditResult:
+    v6_audit = audit_v6_replay_row(row)
+    reasons = list(v6_audit.reasons)
+    output = _assistant_output(row)
+    if output is None:
+        return AuditResult(False, tuple(reasons or ["assistant_content_not_json"]), 0)
+
+    if _task_type(row) == "navigator_full":
+        for issue in v7_source_card_closure_issues(
+            output,
+            target_protocol_card_id=_target_protocol_card_id(row, output),
+        ):
+            reasons.append(issue)
+
+    score = int(v6_audit.score)
+    source_cards = _string_list(output.get("source_cards"))
+    source_card_set = set(source_cards)
+    if {SAFETY_CARD_ID, SBAR_CARD_ID} <= source_card_set:
+        score += 15
+    if 3 <= len(source_cards) <= 5:
+        score += 8
+    if _task_type(row) == "navigator_full":
+        score += 5
+    if not reasons:
+        score += 25
+    return AuditResult(not reasons, tuple(_dedupe(reasons)), score)
+
+
+def select_candidates(
+    candidates: list[Candidate],
+    *,
+    targets: dict[str, int],
+    seed: str,
+    fill_shortage_from_any: bool,
+) -> list[Candidate]:
+    rng = random.Random(seed)
+    shuffled = list(candidates)
+    rng.shuffle(shuffled)
+    by_bucket: dict[str, list[Candidate]] = defaultdict(list)
+    for candidate in shuffled:
+        by_bucket[candidate.source_bucket].append(candidate)
+
+    selected: list[Candidate] = []
+    selected_keys: set[str] = set()
+    for source_bucket, target in targets.items():
+        selected.extend(_take_ranked(by_bucket.get(source_bucket, []), target, selected_keys))
+
+    target_total = sum(targets.values())
+    if fill_shortage_from_any and len(selected) < target_total:
+        remaining = [candidate for candidate in shuffled if _candidate_key(candidate) not in selected_keys]
+        selected.extend(_take_ranked(remaining, target_total - len(selected), selected_keys))
+
+    return sorted(selected, key=lambda candidate: (candidate.source_bucket, candidate.category, candidate.ordinal))
+
+
+def _take_ranked(candidates: list[Candidate], count: int, selected_keys: set[str]) -> list[Candidate]:
+    ranked = sorted(candidates, key=lambda candidate: (-candidate.audit.score, candidate.category, candidate.ordinal))
+    chosen: list[Candidate] = []
+    for candidate in ranked:
+        if len(chosen) >= count:
+            break
+        key = _candidate_key(candidate)
+        if key in selected_keys:
+            continue
+        selected_keys.add(key)
+        chosen.append(candidate)
+    return chosen
+
+
+def _annotate_row(candidate: Candidate) -> dict[str, Any]:
+    row = json.loads(json.dumps(candidate.row, sort_keys=True))
+    metadata = row.setdefault("metadata", {})
+    metadata["dataset_version"] = V7_REPLAY_VERSION
+    metadata["v7_replay_audit"] = {
+        "accepted": True,
+        "audit_score": candidate.audit.score,
+        "original_source_dataset_version": candidate.original_source_dataset_version,
+        "replay_reason": _replay_reason(candidate),
+        "selection_policy_version": 1,
+        "source_bucket": candidate.source_bucket,
+        "source_path": candidate.source_path,
+    }
+    row["version"] = V7_REPLAY_VERSION
+    return row
+
+
+def _replay_reason(candidate: Candidate) -> str:
+    if candidate.task_type == "focused_repair":
+        return f"clean_{candidate.category}_focused_repair"
+    return f"clean_{candidate.category}_navigator_replay"
+
+
+def _shortages(selected: list[Candidate], targets: dict[str, int]) -> dict[str, int]:
+    counts = Counter(candidate.source_bucket for candidate in selected)
+    return {source_bucket: max(0, target - counts.get(source_bucket, 0)) for source_bucket, target in sorted(targets.items())}
+
+
+def _candidate_key(candidate: Candidate) -> str:
+    case_id = str(candidate.row.get("case_id", ""))
+    return f"{candidate.source_path}:{case_id}:{candidate.ordinal}"
+
+
+def _target_protocol_card_id(row: dict[str, Any], output: dict[str, Any]) -> str:
+    metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
+    for card_id in _string_list(metadata.get("must_include_source_cards")):
+        if card_id in CLINICAL_CARD_IDS:
+            return card_id
+    for card_id in _string_list(output.get("source_cards")):
+        if card_id in CLINICAL_CARD_IDS:
+            return card_id
+    return ""
+
+
+def _assistant_output(row: dict[str, Any]) -> dict[str, Any] | None:
+    messages = row.get("messages")
+    if not isinstance(messages, list) or not messages:
+        return None
+    content = messages[-1].get("content") if isinstance(messages[-1], dict) else None
+    if not isinstance(content, str):
+        return None
+    try:
+        output = json.loads(content)
+    except json.JSONDecodeError:
+        return None
+    return output if isinstance(output, dict) else None
+
+
+def _source_bucket(input_path: Path) -> str:
+    stem = input_path.stem
+    if stem.startswith("figment_sft_"):
+        return stem
+    return "unknown"
+
+
+def _original_source_dataset_version(row: dict[str, Any], input_path: Path) -> str:
+    metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
+    replay_audit = metadata.get("v6_replay_audit")
+    if isinstance(replay_audit, dict) and replay_audit.get("source_dataset_version"):
+        return str(replay_audit["source_dataset_version"])
+    version = row.get("version") or metadata.get("dataset_version")
+    if version:
+        return str(version)
+    return _source_bucket(input_path)
+
+
+def _category(row: dict[str, Any]) -> str:
+    metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
+    return str(row.get("category") or metadata.get("category") or metadata.get("failure_class") or "missing")
+
+
+def _task_type(row: dict[str, Any]) -> str:
+    metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
+    if metadata.get("task_type"):
+        return str(metadata["task_type"])
+    output = _assistant_output(row) or {}
+    return "navigator_full" if "protocol_urgency" in output else "focused_repair"
+
+
+def _string_list(value: Any) -> list[str]:
+    if not isinstance(value, list):
+        return []
+    return [str(item).strip() for item in value if str(item).strip()]
+
+
+def _dedupe(values: list[str]) -> list[str]:
+    result: list[str] = []
+    for value in values:
+        if value not in result:
+            result.append(value)
+    return result
+
+
+def _read_jsonl(path: Path) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    with path.open("r", encoding="utf-8") as handle:
+        for line_number, line in enumerate(handle, start=1):
+            stripped = line.strip()
+            if not stripped:
+                continue
+            try:
+                item = json.loads(stripped)
+            except json.JSONDecodeError as exc:
+                raise ValueError(f"{path}:{line_number}: invalid JSON: {exc}") from exc
+            if not isinstance(item, dict):
+                raise ValueError(f"{path}:{line_number}: row must be a JSON object")
+            rows.append(item)
+    return rows
+
+
+def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8") as handle:
+        for row in rows:
+            handle.write(json.dumps(row, sort_keys=True) + "\n")
+
+
+def _sha256_path(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/evidence_gate_status.py

@@ -0,0 +1,284 @@
+#!/usr/bin/env python3
+"""Report Figment evidence gates without upgrading unsupported claims."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+import sys
+from typing import Any
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from scripts import audit_submission_claims  # noqa: E402
+
+
+REPO_ROOT = PROJECT_ROOT
+
+
+def build_report(repo_root: Path = REPO_ROOT) -> dict[str, Any]:
+    repo_root = repo_root.resolve()
+    claim_audit = audit_submission_claims.audit_claims(repo_root)
+    gate_status = claim_audit["gate_status"]
+    gates = {
+        "public_space_no_secret": _public_space_gate(repo_root),
+        "hosted_omni_eval": _hosted_eval_gate(repo_root),
+        "local_4b_50_case_eval": _local_4b_eval_gate(repo_root),
+        "no_cloud_route": _no_cloud_route_gate(repo_root),
+        "llama_champion_route": _llama_champion_gate(repo_root),
+        "local_asr_provider_proof": _local_asr_gate(repo_root),
+        "trained_responder_user_test": _simple_gate(
+            passed=gate_status.get("backyard_user_use", False),
+            label="Trained-responder user test",
+            required_evidence="Completed user-test notes from a real trained responder.",
+            evidence_paths=_existing_paths(repo_root, [Path("docs/user_test_notes.md")]),
+            next_action="Fill docs/user_test_notes.md from a real trained-responder session.",
+        ),
+        "demo_video": _simple_gate(
+            passed=gate_status.get("demo_video", False),
+            label="Demo video",
+            required_evidence="Final demo video link.",
+            evidence_paths=_existing_paths(repo_root, [Path("docs/submission_checklist.md")]),
+            next_action="Add the final demo video link after recording a route-supported demo.",
+        ),
+        "social_post": _simple_gate(
+            passed=gate_status.get("social_post", False),
+            label="Social post",
+            required_evidence="Final social post link with achieved-versus-targeted wording.",
+            evidence_paths=_existing_paths(repo_root, [Path("docs/submission_checklist.md")]),
+            next_action="Add the final social post link after proof-sensitive copy is ready.",
+        ),
+        "well_tuned_adapter": _simple_gate(
+            passed=gate_status.get("well_tuned", False),
+            label="Well-Tuned adapter",
+            required_evidence="Published tuned model or adapter used by the app and measured.",
+            evidence_paths=_existing_paths(repo_root, [Path("docs/model_parameter_evidence_ledger.md")]),
+            next_action="Leave Well-Tuned as stretch until a published measured adapter exists.",
+        ),
+        "claim_audit": _simple_gate(
+            passed=claim_audit["status"] == "passed",
+            label="Submission claim audit",
+            required_evidence="No premature achieved/proven/used/tested claims in submission-facing copy.",
+            evidence_paths=_existing_paths(repo_root, audit_submission_claims.AUDITED_FILES),
+            next_action="Run make audit-claims and fix any overclaiming lines.",
+            extra={"violation_count": len(claim_audit["violations"])},
+        ),
+    }
+    missing_gate_keys = [key for key, gate in gates.items() if not gate["passed"]]
+    return {
+        "status": "complete" if not missing_gate_keys else "incomplete",
+        "ready_for_badge_claims": not missing_gate_keys,
+        "repo_root": str(repo_root),
+        "gates": gates,
+        "missing_gate_keys": missing_gate_keys,
+    }
+
+
+def _public_space_gate(repo_root: Path) -> dict[str, Any]:
+    checklist = _read_text(repo_root / "docs/submission_checklist.md")
+    passed = "Public Hugging Face Space | Runnable" in checklist and "Space cold boot with app files present | Verified" in checklist
+    return _simple_gate(
+        passed=passed,
+        label="Public Hugging Face Space",
+        required_evidence="Public Space URL plus cold-boot evidence with app files present.",
+        evidence_paths=_existing_paths(repo_root, [Path("docs/submission_checklist.md")]),
+        next_action="Re-verify public Space cold boot and record the current Space commit.",
+    )
+
+
+def _hosted_eval_gate(repo_root: Path) -> dict[str, Any]:
+    traces = sorted(repo_root.glob("traces/hosted_omni_eval*.jsonl"))
+    return _simple_gate(
+        passed=bool(traces),
+        label="Hosted Omni eval",
+        required_evidence="Hosted Omni eval JSONL trace and scorecard.",
+        evidence_paths=[str(path) for path in traces]
+        + _existing_paths(repo_root, [Path("docs/hosted_omni_eval_results.md")]),
+        next_action="Run or refresh the hosted Omni eval and update docs/hosted_omni_eval_results.md.",
+    )
+
+
+def _local_4b_eval_gate(repo_root: Path) -> dict[str, Any]:
+    summaries = _local_4b_summaries(repo_root)
+    passing = [
+        (path, summary)
+        for path, summary in summaries
+        if summary.get("counts_as_50_case_local_llm_competence") is True
+        and int(summary.get("total_cases") or 0) >= 50
+    ]
+    evidence_paths = _local_4b_evidence_paths([path for path, _summary in passing] or [path for path, _summary in summaries])
+    return _simple_gate(
+        passed=bool(passing),
+        label="Local 4B 50-case eval",
+        required_evidence="50-case local OpenAI-compatible eval with configured-model competence.",
+        evidence_paths=evidence_paths,
+        next_action="Run scripts/run_local_4b_evidence.py against the local full-weight endpoint.",
+    )
+
+
+def _no_cloud_route_gate(repo_root: Path) -> dict[str, Any]:
+    summaries = _local_4b_summaries(repo_root)
+    passing = [
+        (path, summary)
+        for path, summary in summaries
+        if summary.get("counts_as_no_cloud_route_proof") is True
+    ]
+    return _simple_gate(
+        passed=bool(passing),
+        label="No-cloud/off-grid route",
+        required_evidence="Recorded no-cloud route proof from a local or self-hosted endpoint.",
+        evidence_paths=_local_4b_evidence_paths([path for path, _summary in passing] or [path for path, _summary in summaries]),
+        next_action="Capture a no-cloud local route smoke or eval bundle.",
+    )
+
+
+def _llama_champion_gate(repo_root: Path) -> dict[str, Any]:
+    summaries = _local_4b_summaries(repo_root)
+    passing = [
+        (path, summary)
+        for path, summary in summaries
+        if summary.get("counts_as_50_case_local_llm_competence") is True
+        and int(summary.get("total_cases") or 0) >= 50
+    ]
+    return _simple_gate(
+        passed=bool(passing),
+        label="Llama Champion route",
+        required_evidence="Eligible local llama.cpp/OpenAI-compatible route with trace or eval evidence.",
+        evidence_paths=_local_4b_evidence_paths([path for path, _summary in passing] or [path for path, _summary in summaries]),
+        next_action="Record a qualifying local model route before claiming Llama Champion.",
+    )
+
+
+def _local_asr_gate(repo_root: Path) -> dict[str, Any]:
+    summaries = _local_asr_summaries(repo_root)
+    passing = [
+        (path, summary)
+        for path, summary in summaries
+        if summary.get("counts_as_local_asr_proof") is True
+    ]
+    evidence_paths = _local_asr_evidence_paths([path for path, _summary in passing] or [path for path, _summary in summaries])
+    return _simple_gate(
+        passed=bool(passing),
+        label="Local Parakeet ASR provider proof",
+        required_evidence="Real local ASR provider payload with counts_as_local_asr_proof=true.",
+        evidence_paths=evidence_paths,
+        next_action="Run scripts/run_local_asr_evidence.py with a real local Parakeet provider payload.",
+    )
+
+
+def _simple_gate(
+    *,
+    passed: bool,
+    label: str,
+    required_evidence: str,
+    evidence_paths: list[str],
+    next_action: str,
+    extra: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    gate = {
+        "passed": bool(passed),
+        "label": label,
+        "required_evidence": required_evidence,
+        "evidence_paths": evidence_paths,
+        "next_action": "" if passed else next_action,
+    }
+    if extra:
+        gate.update(extra)
+    return gate
+
+
+def _local_4b_summaries(repo_root: Path) -> list[tuple[Path, dict[str, Any]]]:
+    return [
+        (path, _read_json(path))
+        for path in sorted(repo_root.glob("traces/local_4b_evidence_*/summary.json"))
+    ]
+
+
+def _local_asr_summaries(repo_root: Path) -> list[tuple[Path, dict[str, Any]]]:
+    return [
+        (path, _read_json(path))
+        for path in sorted(repo_root.glob("traces/local_asr_parakeet_evidence_*/summary.json"))
+    ]
+
+
+def _local_4b_evidence_paths(summary_paths: list[Path]) -> list[str]:
+    paths: list[str] = []
+    for summary_path in summary_paths:
+        paths.append(str(summary_path))
+        manifest_path = summary_path.parent / "eval_evidence_manifest.json"
+        if manifest_path.exists():
+            paths.append(str(manifest_path))
+    return paths
+
+
+def _local_asr_evidence_paths(summary_paths: list[Path]) -> list[str]:
+    paths: list[str] = []
+    for summary_path in summary_paths:
+        paths.append(str(summary_path))
+        manifest_path = summary_path.parent / "asr_evidence_manifest.json"
+        if manifest_path.exists():
+            paths.append(str(manifest_path))
+    return paths
+
+
+def _existing_paths(repo_root: Path, relative_paths: tuple[Path, ...] | list[Path]) -> list[str]:
+    return [str(repo_root / path) for path in relative_paths if (repo_root / path).exists()]
+
+
+def _read_json(path: Path) -> dict[str, Any]:
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return {}
+    return payload if isinstance(payload, dict) else {}
+
+
+def _read_text(path: Path) -> str:
+    try:
+        return path.read_text(encoding="utf-8")
+    except OSError:
+        return ""
+
+
+def _markdown_report(report: dict[str, Any]) -> str:
+    lines = [
+        "# Figment Evidence Gate Status",
+        "",
+        f"- Status: `{report['status']}`",
+        f"- Ready for badge claims: `{str(report['ready_for_badge_claims']).lower()}`",
+        "",
+        "| Gate | Passed | Next action |",
+        "| ---- | ------ | ----------- |",
+    ]
+    for key, gate in report["gates"].items():
+        next_action = gate["next_action"] or "Evidence recorded."
+        lines.append(f"| `{key}` | `{str(gate['passed']).lower()}` | {next_action} |")
+    lines.append("")
+    return "\n".join(lines)
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--repo-root", type=Path, default=REPO_ROOT)
+    parser.add_argument("--json", action="store_true")
+    parser.add_argument("--markdown", action="store_true")
+    args = parser.parse_args(argv)
+
+    report = build_report(args.repo_root)
+    if args.json:
+        print(json.dumps(report, indent=2, sort_keys=True))
+    elif args.markdown:
+        print(_markdown_report(report))
+    else:
+        print(f"evidence gate status: {report['status']}")
+        for key in report["missing_gate_keys"]:
+            gate = report["gates"][key]
+            print(f"- {key}: {gate['next_action']}")
+    return 0 if report["status"] == "complete" else 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/export_v4_training_seeds.py

@@ -0,0 +1,270 @@
+"""Export v4 teacher/repair seeds from updated Figment eval failures."""
+
+from __future__ import annotations
+
+import argparse
+from datetime import UTC, datetime
+import json
+from pathlib import Path
+import sys
+from typing import Any
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from figment.eval_metrics import score_expected_labels  # noqa: E402
+
+
+DEFAULT_OUTPUT = Path("data/finetune/v4_seed_exports/figment_sft_v4_failure_seeds.jsonl")
+MODEL_TRAINING_CHECKS = (
+    "red_flags_match",
+    "min_urgency_met",
+    "target_card_in_source_cards",
+    "expected_source_cards_present",
+    "target_card_in_candidate_pathways",
+    "expected_candidate_pathways_present",
+    "missing_observation_cues_present",
+    "model_observation_cues_present",
+    "handoff_cues_present",
+    "handoff_readiness_passed",
+    "forbidden_behavior_absent",
+)
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--eval", type=Path, required=True, help="Scored eval JSONL to export from.")
+    parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
+    parser.add_argument("--include-passing", action="store_true", help="Also emit high-quality replay candidates.")
+    args = parser.parse_args(argv)
+
+    manifest = export_v4_training_seeds(
+        eval_path=args.eval,
+        output_path=args.output,
+        include_passing=args.include_passing,
+    )
+    print(json.dumps(manifest, indent=2, sort_keys=True))
+    return 0
+
+
+def export_v4_training_seeds(*, eval_path: Path, output_path: Path, include_passing: bool = False) -> dict[str, Any]:
+    records = _read_jsonl(eval_path)
+    case_cache: dict[str, list[dict[str, Any]]] = {}
+    seeds = []
+    for record in records:
+        score = score_expected_labels(record)
+        failed = _model_training_failed(score, record)
+        if not failed and not include_passing:
+            continue
+        source_case = _source_case_for_record(record, case_cache)
+        seed = _seed_from_record(record, score, source_case, failed=failed)
+        if seed:
+            seeds.append(seed)
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text("".join(json.dumps(seed, sort_keys=True) + "\n" for seed in seeds), encoding="utf-8")
+    manifest = {
+        "source_eval_path": str(eval_path),
+        "output_path": str(output_path),
+        "source_records": len(records),
+        "seed_count": len(seeds),
+        "failure_seed_count": sum(1 for seed in seeds if seed["seed_type"] == "v4_failure_seed"),
+        "replay_seed_count": sum(1 for seed in seeds if seed["seed_type"] == "v4_replay_candidate"),
+        "harness_only_score_failure_count": sum(1 for seed in seeds if seed.get("harness_only_score_failure")),
+        "repair_scope_counts": _scope_counts(seeds),
+        "generated_at": datetime.now(UTC).isoformat(),
+        "holdout_policy": {
+            "holdout_rows_are_not_training_rows": True,
+            "teacher_must_generate_synthetic_siblings_or_repairs": True,
+            "copying_source_case_or_close_paraphrase_allowed": False,
+        },
+    }
+    output_path.with_suffix(".manifest.json").write_text(
+        json.dumps(manifest, indent=2, sort_keys=True) + "\n",
+        encoding="utf-8",
+    )
+    return manifest
+
+
+def _seed_from_record(
+    record: dict[str, Any],
+    score: dict[str, Any],
+    source_case: dict[str, Any],
+    *,
+    failed: bool,
+) -> dict[str, Any] | None:
+    repair_scopes = _repair_scopes_for_score(score, record)
+    if failed and not repair_scopes:
+        repair_scopes = ["responder_checklist"]
+    if not failed and not _high_quality_replay(record, score):
+        return None
+    case_id = str(record.get("case_id") or source_case.get("case_id") or "")
+    dataset_version = str(source_case.get("dataset_version") or "")
+    direct_training_allowed = dataset_version not in {"field_workflow_holdout_v1"} and not str(
+        record.get("case_path") or ""
+    ).endswith("field_workflow_holdout_v1.jsonl")
+    return {
+        "seed_id": f"v4-seed-{case_id}",
+        "seed_type": "v4_failure_seed" if failed else "v4_replay_candidate",
+        "source_case_id": case_id,
+        "source_case_path": record.get("case_path"),
+        "source_case_line": record.get("case_line"),
+        "source_trace_hash": record.get("trace_hash"),
+        "workflow_category": _workflow_category(record, source_case),
+        "target_protocol_card_id": record.get("target_protocol_card_id") or source_case.get("target_protocol_card_id"),
+        "repair_scopes": repair_scopes,
+        "score_failed_checks": _score_failed_checks(score),
+        "model_training_failed": failed,
+        "harness_only_score_failure": _harness_only_score_failure(score, record),
+        "expected_label_score": score,
+        "final_validation": record.get("final_validation") or record.get("validation_result"),
+        "field_provenance": record.get("field_provenance"),
+        "model_route": record.get("model_route"),
+        "harness_evidence": record.get("harness_evidence") or record.get("final_output", {}).get("harness_evidence"),
+        "structured_intake": source_case.get("structured_intake"),
+        "expected_labels": {
+            "expected_red_flag_rule_ids": source_case.get("expected_red_flag_rule_ids")
+            or record.get("expected_red_flag_rule_ids", []),
+            "expected_min_protocol_urgency": source_case.get("expected_min_protocol_urgency")
+            or record.get("expected_min_protocol_urgency"),
+            "expected_source_card_ids": source_case.get("expected_source_card_ids")
+            or record.get("expected_source_card_ids", []),
+            "expected_candidate_pathway_card_ids": source_case.get("expected_candidate_pathway_card_ids")
+            or record.get("expected_candidate_pathway_card_ids", []),
+            "expected_model_observation_cues": score.get("expected_model_observation_cues", []),
+            "expected_handoff_cues": score.get("expected_handoff_cues", []),
+            "expected_harness_evidence_cues": score.get("expected_harness_evidence_cues", []),
+        },
+        "previous_output": record.get("final_output"),
+        "teacher_instruction": _teacher_instruction(repair_scopes, direct_training_allowed),
+        "direct_training_allowed": direct_training_allowed,
+        "anti_overfit_policy": {
+            "do_not_copy_source_case": True,
+            "do_not_create_close_paraphrase": True,
+            "use_as_failure_pattern_or_repair_seed": True,
+        },
+    }
+
+
+def _repair_scopes_for_score(score: dict[str, Any], record: dict[str, Any]) -> list[str]:
+    scopes: list[str] = []
+    if score.get("red_flags_match") is False or score.get("min_urgency_met") is False:
+        scopes.append("safety_boundary")
+    if score.get("handoff_readiness_passed") is False or score.get("handoff_cues_present") is False:
+        scopes.append("handoff_note_sbar")
+    if score.get("expected_source_cards_present") is False or score.get("target_card_in_source_cards") is False:
+        scopes.append("source_cards")
+    if (
+        score.get("expected_candidate_pathways_present") is False
+        or score.get("target_card_in_candidate_pathways") is False
+    ):
+        scopes.append("candidate_protocol_pathways")
+    if score.get("model_observation_cues_present") is False or score.get("missing_observation_cues_present") is False:
+        scopes.append("missing_observations")
+    if score.get("forbidden_behavior_absent") is False:
+        scopes.append("safety_boundary")
+    validation = record.get("final_validation") or record.get("validation_result")
+    if isinstance(validation, dict) and validation.get("passed") is False:
+        scopes.append("validation_failure")
+    return _ordered_unique(scopes)
+
+
+def _teacher_instruction(repair_scopes: list[str], direct_training_allowed: bool) -> str:
+    if direct_training_allowed:
+        return (
+            "Generate a JSON-only Figment navigator target or focused repair row matching the current harness. "
+            "Improve only the listed repair scopes while preserving deterministic red flags, urgency floor, "
+            "retrieved-card discipline, and protocol-navigation safety."
+        )
+    return (
+        "This source is an eval/holdout seed. Do not copy it or make a close paraphrase. Generate a synthetic "
+        "sibling or repair pattern that exercises the same failure scopes: "
+        f"{', '.join(repair_scopes) or 'replay'}."
+    )
+
+
+def _high_quality_replay(record: dict[str, Any], score: dict[str, Any]) -> bool:
+    validation = record.get("final_validation") or record.get("validation_result")
+    return (
+        isinstance(validation, dict)
+        and validation.get("passed") is True
+        and _model_training_passed(score, record)
+        and not record.get("canned_fallback_used")
+        and not record.get("fallback_reason")
+    )
+
+
+def _model_training_failed(score: dict[str, Any], record: dict[str, Any]) -> bool:
+    validation = record.get("final_validation") or record.get("validation_result")
+    if isinstance(validation, dict) and validation.get("passed") is False:
+        return True
+    return any(score.get(check) is False for check in MODEL_TRAINING_CHECKS)
+
+
+def _model_training_passed(score: dict[str, Any], record: dict[str, Any]) -> bool:
+    validation = record.get("final_validation") or record.get("validation_result")
+    if isinstance(validation, dict) and validation.get("passed") is not True:
+        return False
+    return not _model_training_failed(score, record)
+
+
+def _score_failed_checks(score: dict[str, Any]) -> list[str]:
+    return [key for key, value in score.items() if isinstance(value, bool) and value is False]
+
+
+def _harness_only_score_failure(score: dict[str, Any], record: dict[str, Any]) -> bool:
+    return (
+        score.get("all_expected_labels_passed") is False
+        and not _model_training_failed(score, record)
+        and score.get("harness_evidence_cues_visible") is False
+    )
+
+
+def _workflow_category(record: dict[str, Any], source_case: dict[str, Any]) -> str | None:
+    structured_intake = source_case.get("structured_intake")
+    if isinstance(structured_intake, dict) and structured_intake.get("workflow_category"):
+        return str(structured_intake["workflow_category"])
+    for payload in (source_case, record):
+        if payload.get("workflow_category"):
+            return str(payload["workflow_category"])
+    return None
+
+
+def _source_case_for_record(record: dict[str, Any], case_cache: dict[str, list[dict[str, Any]]]) -> dict[str, Any]:
+    path = record.get("case_path")
+    line = record.get("case_line")
+    if not path or not line:
+        return {}
+    path_text = str(path)
+    if path_text not in case_cache:
+        case_cache[path_text] = _read_jsonl(Path(path_text))
+    index = int(line) - 1
+    cases = case_cache[path_text]
+    if index < 0 or index >= len(cases):
+        return {}
+    return cases[index]
+
+
+def _read_jsonl(path: Path) -> list[dict[str, Any]]:
+    return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
+
+
+def _ordered_unique(values: list[str]) -> list[str]:
+    out: list[str] = []
+    for value in values:
+        if value and value not in out:
+            out.append(value)
+    return out
+
+
+def _scope_counts(seeds: list[dict[str, Any]]) -> dict[str, int]:
+    counts: dict[str, int] = {}
+    for seed in seeds:
+        for scope in seed.get("repair_scopes", []):
+            counts[scope] = counts.get(scope, 0) + 1
+    return dict(sorted(counts.items()))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/generate_field_workflow_holdout.py

@@ -0,0 +1,150 @@
+"""Generate the frozen v3 field-workflow holdout eval cases."""
+
+from __future__ import annotations
+
+import argparse
+from collections import Counter
+from datetime import UTC
+from datetime import datetime
+import hashlib
+import json
+from pathlib import Path
+import sys
+from typing import Any
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from scripts.generate_finetune_data import case_spec_record  # noqa: E402
+from scripts.generate_finetune_data import forbidden_behavior_for_version  # noqa: E402
+from scripts.generate_finetune_data import generate_case_spec  # noqa: E402
+from scripts.generate_finetune_data import prepare_case  # noqa: E402
+from scripts.generate_finetune_data import safety_boundary_for_version  # noqa: E402
+from scripts.generate_finetune_data import stable_hash  # noqa: E402
+from figment.eval_metrics import bucket_expected_observation_cues  # noqa: E402
+from figment.retrieval import load_protocol_cards  # noqa: E402
+
+
+HOLDOUT_VERSION = "field_workflow_holdout_v1"
+OUTPUT_PATH = Path("data/eval/field_workflow_holdout_v1.jsonl")
+MANIFEST_PATH = Path("data/eval/field_workflow_holdout_v1_manifest.json")
+SOURCE_DATASET_VERSION = "figment_sft_v3_holdout_source"
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--count", type=int, default=150)
+    parser.add_argument("--output", type=Path, default=OUTPUT_PATH)
+    parser.add_argument("--manifest", type=Path, default=MANIFEST_PATH)
+    args = parser.parse_args(argv)
+
+    manifest = generate_holdout(count=args.count, output_path=args.output, manifest_path=args.manifest)
+    print(json.dumps(manifest, indent=2, sort_keys=True))
+    return 0
+
+
+def generate_holdout(*, count: int, output_path: Path, manifest_path: Path) -> dict[str, Any]:
+    if count <= 0:
+        raise ValueError("count must be positive")
+
+    cards_by_id = {str(card["card_id"]): card for card in load_protocol_cards()}
+    rows: list[dict[str, Any]] = []
+    row_hashes: list[dict[str, str]] = []
+    attempts = 0
+    max_attempts = count * 8
+
+    while len(rows) < count and attempts < max_attempts:
+        source_spec = generate_case_spec(attempts, cards_by_id, dataset_version=SOURCE_DATASET_VERSION)
+        attempts += 1
+        prepared = prepare_case(source_spec, cards_by_id)
+        record = case_spec_record(prepared)
+        holdout_case_id = f"{HOLDOUT_VERSION}-{len(rows):06d}"
+        row = _holdout_row(
+            record,
+            holdout_case_id=holdout_case_id,
+            source_case_id=source_spec.case_id,
+            prompt_hash=stable_hash(prepared.prompt),
+            prompt_template_hash=prepared.prompt_hash,
+        )
+        row_hashes.append({"case_id": holdout_case_id, "sha256": _json_sha256(row)})
+        rows.append(row)
+
+    if len(rows) < count:
+        raise RuntimeError(f"could only generate {len(rows)} holdout rows after {attempts} attempts")
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text("".join(json.dumps(row, sort_keys=True) + "\n" for row in rows), encoding="utf-8")
+
+    manifest = {
+        "dataset_version": HOLDOUT_VERSION,
+        "source_dataset_version": SOURCE_DATASET_VERSION,
+        "row_count": len(rows),
+        "output_path": str(output_path),
+        "output_sha256": _file_sha256(output_path),
+        "generated_at": datetime.now(UTC).isoformat(),
+        "source_generator": "scripts/generate_field_workflow_holdout.py",
+        "source_generator_prompt_family": "figment_sft_v3_field_workflow",
+        "category_counts": dict(sorted(Counter(row["workflow_category"] for row in rows).items())),
+        "row_hashes": row_hashes,
+        "holdout_policy": {
+            "never_train_on_this_file": True,
+            "never_copy_close_paraphrases_into_training": True,
+            "freeze_case_ids": True,
+            "primary_v3_success_surface": True,
+        },
+    }
+    manifest_path.parent.mkdir(parents=True, exist_ok=True)
+    manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return manifest
+
+
+def _holdout_row(
+    record: dict[str, Any],
+    *,
+    holdout_case_id: str,
+    source_case_id: str,
+    prompt_hash: str,
+    prompt_template_hash: str,
+) -> dict[str, Any]:
+    workflow_category = str(record.get("workflow_category") or record.get("failure_class") or "field_workflow")
+    cue_buckets = bucket_expected_observation_cues(record["expected_missing_observations"])
+    return {
+        "case_id": holdout_case_id,
+        "dataset_version": HOLDOUT_VERSION,
+        "source_generator_case_id": source_case_id,
+        "workflow_category": workflow_category,
+        "target_protocol_card_id": record["target_protocol_card_id"],
+        "structured_intake": record["structured_intake"],
+        "expected_red_flag_rule_ids": record["expected_red_flag_rule_ids"],
+        "expected_min_protocol_urgency": record["expected_min_protocol_urgency"],
+        "expected_source_card_ids": record["expected_source_card_ids"],
+        "expected_candidate_pathway_card_ids": record["expected_candidate_pathway_card_ids"],
+        "expected_missing_observations": record["expected_missing_observations"],
+        "expected_model_observation_cues": cue_buckets["model"],
+        "expected_handoff_cues": cue_buckets["handoff"],
+        "expected_harness_evidence_cues": cue_buckets["harness"],
+        "workflow_priority_observations": record.get("workflow_priority_observations", []),
+        "retrieved_card_ids": record["retrieved_card_ids"],
+        "tags": record.get("tags", []),
+        "safety_notes": safety_boundary_for_version(HOLDOUT_VERSION),
+        "forbidden_behavior": forbidden_behavior_for_version(HOLDOUT_VERSION),
+        "prompt_hash": prompt_hash,
+        "prompt_template_hash": prompt_template_hash,
+    }
+
+
+def _json_sha256(value: dict[str, Any]) -> str:
+    return "sha256:" + hashlib.sha256(json.dumps(value, sort_keys=True).encode("utf-8")).hexdigest()
+
+
+def _file_sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as file:
+        for chunk in iter(lambda: file.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/generate_v10_full_corpus.py

@@ -0,0 +1,66 @@
+"""Generate, verify, and Modal-prep the Figment v10 delta SFT corpus."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import sys
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from scripts.generate_finetune_data import V10_NAVIGATOR_COUNTS  # noqa: E402
+from scripts.generate_v3_full_corpus import main as _run_full_corpus  # noqa: E402
+
+
+DEFAULT_OUTPUT_VERSION = "figment_sft_v10_delta"
+DEFAULT_TEACHER_MODEL_ID = "nvidia/nemotron-3-ultra-550b-a55b:free"
+DEFAULT_NAVIGATOR_COUNT = sum(V10_NAVIGATOR_COUNTS.values())
+DEFAULT_REPAIR_COUNT = 0
+DEFAULT_ARGS = [
+    "--dataset-version",
+    DEFAULT_OUTPUT_VERSION,
+    "--navigator-count",
+    str(DEFAULT_NAVIGATOR_COUNT),
+    "--repair-count",
+    str(DEFAULT_REPAIR_COUNT),
+    "--teacher-model-id",
+    DEFAULT_TEACHER_MODEL_ID,
+    "--base-start-index",
+    "120000",
+    "--rows-per-shard",
+    "40",
+    "--shard-prefix",
+    "data/finetune/shards/figment_sft_v10_delta_full_shard",
+    "--output",
+    "data/finetune/figment_sft_v10_delta.jsonl",
+    "--case-specs",
+    "data/finetune/figment_sft_v10_delta_case_specs.jsonl",
+    "--manifest",
+    "data/finetune/figment_sft_v10_delta_manifest.json",
+    "--modal-output-dir",
+    "data/finetune/modal/figment_sft_v10_delta",
+    "--seed",
+    "figment-modal-sft-v10-delta",
+]
+
+
+def build_corpus_args(argv: list[str] | None = None) -> list[str]:
+    raw_args = list(sys.argv[1:] if argv is None else argv)
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument("--navigator-count", type=int, default=DEFAULT_NAVIGATOR_COUNT)
+    parser.add_argument("--repair-count", type=int, default=DEFAULT_REPAIR_COUNT)
+    parsed, remaining = parser.parse_known_args(raw_args)
+    defaults = list(DEFAULT_ARGS)
+    defaults[defaults.index("--navigator-count") + 1] = str(parsed.navigator_count)
+    defaults[defaults.index("--repair-count") + 1] = str(parsed.repair_count)
+    return defaults + remaining
+
+
+def main(argv: list[str] | None = None) -> int:
+    return _run_full_corpus(build_corpus_args(argv))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/generate_v11_full_corpus.py

@@ -0,0 +1,66 @@
+"""Generate, verify, and Modal-prep the Figment v11 delta SFT corpus."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import sys
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from scripts.generate_finetune_data import V11_NAVIGATOR_COUNTS  # noqa: E402
+from scripts.generate_v3_full_corpus import main as _run_full_corpus  # noqa: E402
+
+
+DEFAULT_OUTPUT_VERSION = "figment_sft_v11_delta"
+DEFAULT_TEACHER_MODEL_ID = "nvidia/nemotron-3-ultra-550b-a55b:free"
+DEFAULT_NAVIGATOR_COUNT = sum(V11_NAVIGATOR_COUNTS.values())
+DEFAULT_REPAIR_COUNT = 0
+DEFAULT_ARGS = [
+    "--dataset-version",
+    DEFAULT_OUTPUT_VERSION,
+    "--navigator-count",
+    str(DEFAULT_NAVIGATOR_COUNT),
+    "--repair-count",
+    str(DEFAULT_REPAIR_COUNT),
+    "--teacher-model-id",
+    DEFAULT_TEACHER_MODEL_ID,
+    "--base-start-index",
+    "140000",
+    "--rows-per-shard",
+    "40",
+    "--shard-prefix",
+    "data/finetune/shards/figment_sft_v11_delta_full_shard",
+    "--output",
+    "data/finetune/figment_sft_v11_delta.jsonl",
+    "--case-specs",
+    "data/finetune/figment_sft_v11_delta_case_specs.jsonl",
+    "--manifest",
+    "data/finetune/figment_sft_v11_delta_manifest.json",
+    "--modal-output-dir",
+    "data/finetune/modal/figment_sft_v11_delta",
+    "--seed",
+    "figment-modal-sft-v11-delta",
+]
+
+
+def build_corpus_args(argv: list[str] | None = None) -> list[str]:
+    raw_args = list(sys.argv[1:] if argv is None else argv)
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument("--navigator-count", type=int, default=DEFAULT_NAVIGATOR_COUNT)
+    parser.add_argument("--repair-count", type=int, default=DEFAULT_REPAIR_COUNT)
+    parsed, remaining = parser.parse_known_args(raw_args)
+    defaults = list(DEFAULT_ARGS)
+    defaults[defaults.index("--navigator-count") + 1] = str(parsed.navigator_count)
+    defaults[defaults.index("--repair-count") + 1] = str(parsed.repair_count)
+    return defaults + remaining
+
+
+def main(argv: list[str] | None = None) -> int:
+    return _run_full_corpus(build_corpus_args(argv))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/generate_v12_full_corpus.py

@@ -0,0 +1,66 @@
+"""Generate, verify, and Modal-prep the Figment v12 delta SFT corpus."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import sys
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from scripts.generate_finetune_data import V12_NAVIGATOR_COUNTS  # noqa: E402
+from scripts.generate_v3_full_corpus import main as _run_full_corpus  # noqa: E402
+
+
+DEFAULT_OUTPUT_VERSION = "figment_sft_v12_delta"
+DEFAULT_TEACHER_MODEL_ID = "nvidia/nemotron-3-ultra-550b-a55b:free"
+DEFAULT_NAVIGATOR_COUNT = sum(V12_NAVIGATOR_COUNTS.values())
+DEFAULT_REPAIR_COUNT = 0
+DEFAULT_ARGS = [
+    "--dataset-version",
+    DEFAULT_OUTPUT_VERSION,
+    "--navigator-count",
+    str(DEFAULT_NAVIGATOR_COUNT),
+    "--repair-count",
+    str(DEFAULT_REPAIR_COUNT),
+    "--teacher-model-id",
+    DEFAULT_TEACHER_MODEL_ID,
+    "--base-start-index",
+    "160000",
+    "--rows-per-shard",
+    "40",
+    "--shard-prefix",
+    "data/finetune/shards/figment_sft_v12_delta_full_shard",
+    "--output",
+    "data/finetune/figment_sft_v12_delta.jsonl",
+    "--case-specs",
+    "data/finetune/figment_sft_v12_delta_case_specs.jsonl",
+    "--manifest",
+    "data/finetune/figment_sft_v12_delta_manifest.json",
+    "--modal-output-dir",
+    "data/finetune/modal/figment_sft_v12_delta",
+    "--seed",
+    "figment-modal-sft-v12-delta",
+]
+
+
+def build_corpus_args(argv: list[str] | None = None) -> list[str]:
+    raw_args = list(sys.argv[1:] if argv is None else argv)
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument("--navigator-count", type=int, default=DEFAULT_NAVIGATOR_COUNT)
+    parser.add_argument("--repair-count", type=int, default=DEFAULT_REPAIR_COUNT)
+    parsed, remaining = parser.parse_known_args(raw_args)
+    defaults = list(DEFAULT_ARGS)
+    defaults[defaults.index("--navigator-count") + 1] = str(parsed.navigator_count)
+    defaults[defaults.index("--repair-count") + 1] = str(parsed.repair_count)
+    return defaults + remaining
+
+
+def main(argv: list[str] | None = None) -> int:
+    return _run_full_corpus(build_corpus_args(argv))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/generate_v13_full_corpus.py

@@ -0,0 +1,75 @@
+"""Generate, verify, and Modal-prep the Figment v13 delta SFT corpus."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import sys
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from scripts.generate_finetune_data import V13_NAVIGATOR_COUNTS  # noqa: E402
+from scripts.generate_v3_full_corpus import main as _run_full_corpus  # noqa: E402
+
+
+DEFAULT_OUTPUT_VERSION = "figment_sft_v13_delta"
+DEFAULT_TEACHER_MODEL_ID = "nvidia/nemotron-3-ultra-550b-a55b"
+DEFAULT_NAVIGATOR_COUNT = sum(V13_NAVIGATOR_COUNTS.values())
+DEFAULT_REPAIR_COUNT = 0
+DEFAULT_ARGS = [
+    "--dataset-version",
+    DEFAULT_OUTPUT_VERSION,
+    "--navigator-count",
+    str(DEFAULT_NAVIGATOR_COUNT),
+    "--repair-count",
+    str(DEFAULT_REPAIR_COUNT),
+    "--teacher-model-id",
+    DEFAULT_TEACHER_MODEL_ID,
+    "--no-teacher-worker",
+    "--base-start-index",
+    "180000",
+    "--rows-per-shard",
+    "10",
+    "--parallelism",
+    "2",
+    "--timeout-seconds",
+    "120",
+    "--teacher-error-retries",
+    "12",
+    "--teacher-error-sleep-seconds",
+    "20",
+    "--shard-prefix",
+    "data/finetune/shards/figment_sft_v13_delta_full_shard",
+    "--output",
+    "data/finetune/figment_sft_v13_delta.jsonl",
+    "--case-specs",
+    "data/finetune/figment_sft_v13_delta_case_specs.jsonl",
+    "--manifest",
+    "data/finetune/figment_sft_v13_delta_manifest.json",
+    "--modal-output-dir",
+    "data/finetune/modal/figment_sft_v13_delta",
+    "--seed",
+    "figment-modal-sft-v13-delta",
+]
+
+
+def build_corpus_args(argv: list[str] | None = None) -> list[str]:
+    raw_args = list(sys.argv[1:] if argv is None else argv)
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument("--navigator-count", type=int, default=DEFAULT_NAVIGATOR_COUNT)
+    parser.add_argument("--repair-count", type=int, default=DEFAULT_REPAIR_COUNT)
+    parsed, remaining = parser.parse_known_args(raw_args)
+    defaults = list(DEFAULT_ARGS)
+    defaults[defaults.index("--navigator-count") + 1] = str(parsed.navigator_count)
+    defaults[defaults.index("--repair-count") + 1] = str(parsed.repair_count)
+    return defaults + remaining
+
+
+def main(argv: list[str] | None = None) -> int:
+    return _run_full_corpus(build_corpus_args(argv))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/generate_v14_full_corpus.py

@@ -0,0 +1,75 @@
+"""Generate, verify, and Modal-prep the Figment v14 delta SFT corpus."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import sys
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from scripts.generate_finetune_data import V14_NAVIGATOR_COUNTS  # noqa: E402
+from scripts.generate_v3_full_corpus import main as _run_full_corpus  # noqa: E402
+
+
+DEFAULT_OUTPUT_VERSION = "figment_sft_v14_delta"
+DEFAULT_TEACHER_MODEL_ID = "nvidia/nemotron-3-ultra-550b-a55b"
+DEFAULT_NAVIGATOR_COUNT = sum(V14_NAVIGATOR_COUNTS.values())
+DEFAULT_REPAIR_COUNT = 0
+DEFAULT_ARGS = [
+    "--dataset-version",
+    DEFAULT_OUTPUT_VERSION,
+    "--navigator-count",
+    str(DEFAULT_NAVIGATOR_COUNT),
+    "--repair-count",
+    str(DEFAULT_REPAIR_COUNT),
+    "--teacher-model-id",
+    DEFAULT_TEACHER_MODEL_ID,
+    "--no-teacher-worker",
+    "--base-start-index",
+    "240000",
+    "--rows-per-shard",
+    "10",
+    "--parallelism",
+    "2",
+    "--timeout-seconds",
+    "120",
+    "--teacher-error-retries",
+    "20",
+    "--teacher-error-sleep-seconds",
+    "20",
+    "--shard-prefix",
+    "data/finetune/shards/figment_sft_v14_delta_full_shard",
+    "--output",
+    "data/finetune/figment_sft_v14_delta.jsonl",
+    "--case-specs",
+    "data/finetune/figment_sft_v14_delta_case_specs.jsonl",
+    "--manifest",
+    "data/finetune/figment_sft_v14_delta_manifest.json",
+    "--modal-output-dir",
+    "data/finetune/modal/figment_sft_v14_delta",
+    "--seed",
+    "figment-modal-sft-v14-delta",
+]
+
+
+def build_corpus_args(argv: list[str] | None = None) -> list[str]:
+    raw_args = list(sys.argv[1:] if argv is None else argv)
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument("--navigator-count", type=int, default=DEFAULT_NAVIGATOR_COUNT)
+    parser.add_argument("--repair-count", type=int, default=DEFAULT_REPAIR_COUNT)
+    parsed, remaining = parser.parse_known_args(raw_args)
+    defaults = list(DEFAULT_ARGS)
+    defaults[defaults.index("--navigator-count") + 1] = str(parsed.navigator_count)
+    defaults[defaults.index("--repair-count") + 1] = str(parsed.repair_count)
+    return defaults + remaining
+
+
+def main(argv: list[str] | None = None) -> int:
+    return _run_full_corpus(build_corpus_args(argv))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/generate_v3_full_corpus.py

@@ -0,0 +1,303 @@
+"""Generate, merge, verify, and Modal-prep the full Figment v3 SFT corpus."""
+
+from __future__ import annotations
+
+import argparse
+from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import as_completed
+from dataclasses import dataclass
+import json
+import math
+from pathlib import Path
+import subprocess
+import sys
+from typing import Any
+
+
+DATASET_VERSION = "figment_sft_v3"
+DEFAULT_SHARD_PREFIX = Path("data/finetune/shards/figment_sft_v3_full_shard")
+DEFAULT_OUTPUT = Path("data/finetune/figment_sft_v3.jsonl")
+DEFAULT_CASE_SPECS = Path("data/finetune/figment_sft_v3_case_specs.jsonl")
+DEFAULT_MANIFEST = Path("data/finetune/figment_sft_v3_manifest.json")
+DEFAULT_MODAL_DIR = Path("data/finetune/modal/figment_sft_v3")
+DEFAULT_TEACHER_MODEL = "nvidia/nemotron-3-ultra-550b-a55b"
+
+
+@dataclass(frozen=True)
+class ShardSpec:
+    index: int
+    row_count: int
+    start_index: int
+    index_stride: int
+    output: Path
+    case_specs: Path
+    manifest: Path
+    log: Path
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--dataset-version", default=DATASET_VERSION)
+    parser.add_argument("--navigator-count", type=int, default=2500)
+    parser.add_argument("--repair-count", type=int, default=500)
+    parser.add_argument("--rows-per-shard", type=int, default=50)
+    parser.add_argument("--parallelism", type=int, default=4)
+    parser.add_argument("--base-start-index", type=int, default=20000)
+    parser.add_argument("--shard-prefix", type=Path, default=DEFAULT_SHARD_PREFIX)
+    parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
+    parser.add_argument("--case-specs", type=Path, default=DEFAULT_CASE_SPECS)
+    parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
+    parser.add_argument("--modal-output-dir", type=Path, default=DEFAULT_MODAL_DIR)
+    parser.add_argument("--teacher-model-id", default=DEFAULT_TEACHER_MODEL)
+    parser.add_argument("--timeout-seconds", type=float, default=60.0)
+    parser.add_argument("--teacher-max-tokens", type=int, default=700)
+    parser.add_argument("--teacher-error-retries", type=int, default=2)
+    parser.add_argument("--teacher-error-sleep-seconds", type=float, default=2.0)
+    parser.add_argument("--no-teacher-worker", action="store_true")
+    parser.add_argument("--max-attempts-multiplier", type=int, default=8)
+    parser.add_argument("--validation-fraction", type=float, default=0.1)
+    parser.add_argument("--seed", default="figment-modal-sft-v3")
+    parser.add_argument("--min-validation-group-size", type=int, default=5)
+    parser.add_argument("--skip-generation", action="store_true")
+    parser.add_argument("--skip-repair", action="store_true")
+    parser.add_argument("--only-generate", action="store_true")
+    parser.add_argument("--log-rejections", action="store_true")
+    parser.add_argument("--dry-run", action="store_true", help="Use deterministic fallback rows instead of teacher calls.")
+    args = parser.parse_args(argv)
+
+    if args.navigator_count <= 0:
+        raise SystemExit("--navigator-count must be positive")
+    if args.repair_count < 0:
+        raise SystemExit("--repair-count must be non-negative")
+    if args.rows_per_shard <= 0:
+        raise SystemExit("--rows-per-shard must be positive")
+    if args.parallelism <= 0:
+        raise SystemExit("--parallelism must be positive")
+    if args.max_attempts_multiplier < 1:
+        raise SystemExit("--max-attempts-multiplier must be at least 1")
+
+    shard_specs = build_shard_specs(
+        navigator_count=args.navigator_count,
+        rows_per_shard=args.rows_per_shard,
+        base_start_index=args.base_start_index,
+        shard_prefix=args.shard_prefix,
+    )
+    generation_results = []
+    if not args.skip_generation:
+        generation_results = generate_shards(args, shard_specs)
+
+    if args.only_generate:
+        print(json.dumps({"shards": [spec.__dict__ | {"output": str(spec.output)} for spec in shard_specs]}, default=str))
+        return 0
+
+    merge_cmd = [
+        sys.executable,
+        "scripts/merge_finetune_shards.py",
+        "--dataset-version",
+        args.dataset_version,
+        "--shard-prefix",
+        str(args.shard_prefix),
+        "--shard-count",
+        str(len(shard_specs)),
+        "--output",
+        str(args.output),
+        "--case-specs",
+        str(args.case_specs),
+        "--manifest",
+        str(args.manifest),
+    ]
+    merge_summary = _run_json_command(merge_cmd)
+
+    repair_summary = None
+    if not args.skip_repair and args.repair_count:
+        repair_summary = _run_json_command(build_repair_command(args))
+
+    verify_summary = _run_json_command(
+        [
+            sys.executable,
+            "scripts/verify_finetune_harness_alignment.py",
+            "--dataset",
+            str(args.output),
+            "--case-specs",
+            str(args.case_specs),
+        ]
+    )
+    if verify_summary.get("passed") is not True:
+        raise SystemExit(f"harness verification failed: {json.dumps(verify_summary, sort_keys=True)}")
+
+    modal_summary = _run_json_command(
+        [
+            sys.executable,
+            "scripts/prepare_modal_finetune_dataset.py",
+            "--dataset",
+            str(args.output),
+            "--dataset-version",
+            args.dataset_version,
+            "--output-dir",
+            str(args.modal_output_dir),
+            "--validation-fraction",
+            str(args.validation_fraction),
+            "--seed",
+            args.seed,
+            "--min-validation-group-size",
+            str(args.min_validation_group_size),
+        ]
+    )
+
+    summary = {
+        "dataset_version": args.dataset_version,
+        "navigator_count": args.navigator_count,
+        "repair_count": args.repair_count,
+        "shard_count": len(shard_specs),
+        "generation": generation_results,
+        "merge": merge_summary,
+        "repair": repair_summary,
+        "verify": verify_summary,
+        "modal": modal_summary,
+    }
+    print(json.dumps(summary, indent=2, sort_keys=True, default=str))
+    return 0
+
+
+def build_shard_specs(
+    *,
+    navigator_count: int,
+    rows_per_shard: int,
+    base_start_index: int,
+    shard_prefix: Path,
+) -> list[ShardSpec]:
+    shard_count = math.ceil(navigator_count / rows_per_shard)
+    specs: list[ShardSpec] = []
+    remaining = navigator_count
+    for index in range(shard_count):
+        row_count = min(rows_per_shard, remaining)
+        base = Path(f"{shard_prefix}{index}")
+        specs.append(
+            ShardSpec(
+                index=index,
+                row_count=row_count,
+                start_index=base_start_index + index,
+                index_stride=shard_count,
+                output=Path(f"{base}.jsonl"),
+                case_specs=Path(f"{base}_case_specs.jsonl"),
+                manifest=Path(f"{base}_manifest.json"),
+                log=Path(f"{base}.log"),
+            )
+        )
+        remaining -= row_count
+    return specs
+
+
+def generate_shards(args: argparse.Namespace, shard_specs: list[ShardSpec]) -> list[dict[str, Any]]:
+    for spec in shard_specs:
+        spec.output.parent.mkdir(parents=True, exist_ok=True)
+    results: list[dict[str, Any]] = []
+    with ThreadPoolExecutor(max_workers=args.parallelism) as executor:
+        futures = [executor.submit(generate_one_shard, args, spec) for spec in shard_specs]
+        for future in as_completed(futures):
+            result = future.result()
+            results.append(result)
+            print(json.dumps({"shard_complete": result}, sort_keys=True), flush=True)
+    return sorted(results, key=lambda item: int(item["index"]))
+
+
+def generate_one_shard(args: argparse.Namespace, spec: ShardSpec) -> dict[str, Any]:
+    existing = _read_manifest(spec.manifest)
+    if existing and int(existing.get("row_count") or 0) >= spec.row_count:
+        return {"index": spec.index, "status": "skipped_existing", "rows": int(existing.get("row_count") or 0)}
+
+    cmd = [
+        sys.executable,
+        "scripts/generate_finetune_data.py",
+        "--dataset-version",
+        args.dataset_version,
+        "--count",
+        str(spec.row_count),
+        "--output",
+        str(spec.output),
+        "--case-specs",
+        str(spec.case_specs),
+        "--manifest",
+        str(spec.manifest),
+        "--teacher-model-id",
+        args.teacher_model_id,
+        "--timeout-seconds",
+        str(args.timeout_seconds),
+        "--teacher-max-tokens",
+        str(args.teacher_max_tokens),
+        "--teacher-error-retries",
+        str(args.teacher_error_retries),
+        "--teacher-error-sleep-seconds",
+        str(args.teacher_error_sleep_seconds),
+        "--candidate-count",
+        "1",
+        "--high-risk-candidate-count",
+        "1",
+        "--max-attempts",
+        str(spec.row_count * args.max_attempts_multiplier),
+        "--start-index",
+        str(spec.start_index),
+        "--index-stride",
+        str(spec.index_stride),
+        "--resume",
+    ]
+    if args.log_rejections:
+        cmd.append("--log-rejections")
+    if args.dry_run:
+        cmd.append("--dry-run")
+    if args.no_teacher_worker:
+        cmd.append("--no-teacher-worker")
+
+    spec.log.parent.mkdir(parents=True, exist_ok=True)
+    with spec.log.open("a", encoding="utf-8") as log:
+        log.write("\n=== command ===\n")
+        log.write(" ".join(cmd) + "\n")
+        log.flush()
+        completed = subprocess.run(cmd, stdout=log, stderr=subprocess.STDOUT, text=True)
+    if completed.returncode != 0:
+        raise RuntimeError(f"shard {spec.index} failed with exit {completed.returncode}; see {spec.log}")
+    manifest = _read_manifest(spec.manifest)
+    return {
+        "index": spec.index,
+        "status": "generated",
+        "rows": int(manifest.get("row_count") or 0),
+        "attempts": int(manifest.get("attempts") or 0),
+        "manifest": str(spec.manifest),
+        "log": str(spec.log),
+    }
+
+
+def build_repair_command(args: argparse.Namespace) -> list[str]:
+    return [
+        sys.executable,
+        "scripts/augment_finetune_repair_rows.py",
+        "--dataset-version",
+        args.dataset_version,
+        "--dataset",
+        str(args.output),
+        "--case-specs",
+        str(args.case_specs),
+        "--manifest",
+        str(args.manifest),
+        "--repair-count",
+        str(args.repair_count),
+    ]
+
+
+def _read_manifest(path: Path) -> dict[str, Any]:
+    if not path.exists():
+        return {}
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError:
+        return {}
+    return data if isinstance(data, dict) else {}
+
+
+def _run_json_command(cmd: list[str]) -> dict[str, Any]:
+    completed = subprocess.run(cmd, check=True, text=True, capture_output=True)
+    return json.loads(completed.stdout)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/generate_v4_full_corpus.py

@@ -0,0 +1,48 @@
+"""Generate, verify, and Modal-prep the full Figment v4 SFT corpus."""
+
+from __future__ import annotations
+
+from pathlib import Path
+import sys
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from scripts.generate_v3_full_corpus import main as _run_full_corpus  # noqa: E402
+
+
+DEFAULT_ARGS = [
+    "--dataset-version",
+    "figment_sft_v4",
+    "--navigator-count",
+    "1500",
+    "--repair-count",
+    "150",
+    "--base-start-index",
+    "40000",
+    "--shard-prefix",
+    "data/finetune/shards/figment_sft_v4_full_shard",
+    "--output",
+    "data/finetune/figment_sft_v4.jsonl",
+    "--case-specs",
+    "data/finetune/figment_sft_v4_case_specs.jsonl",
+    "--manifest",
+    "data/finetune/figment_sft_v4_manifest.json",
+    "--modal-output-dir",
+    "data/finetune/modal/figment_sft_v4",
+    "--seed",
+    "figment-modal-sft-v4",
+]
+
+
+def build_corpus_args(argv: list[str] | None = None) -> list[str]:
+    return DEFAULT_ARGS + list(sys.argv[1:] if argv is None else argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    return _run_full_corpus(build_corpus_args(argv))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/generate_v5_full_corpus.py

@@ -0,0 +1,56 @@
+"""Generate, verify, and Modal-prep the full Figment v5 SFT corpus."""
+
+from __future__ import annotations
+
+from pathlib import Path
+import sys
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from scripts.generate_finetune_data import V5_FOCUSED_COUNTS  # noqa: E402
+from scripts.generate_v3_full_corpus import main as _run_full_corpus  # noqa: E402
+
+
+DEFAULT_OUTPUT_VERSION = "figment_sft_v5"
+DEFAULT_TEACHER_MODEL_ID = "nvidia/nemotron-3-ultra-550b-a55b:free"
+DEFAULT_COUNTS = dict(V5_FOCUSED_COUNTS)
+DEFAULT_NAVIGATOR_COUNT = sum(DEFAULT_COUNTS.values())
+
+DEFAULT_ARGS = [
+    "--dataset-version",
+    DEFAULT_OUTPUT_VERSION,
+    "--navigator-count",
+    str(DEFAULT_NAVIGATOR_COUNT),
+    "--repair-count",
+    "200",
+    "--teacher-model-id",
+    DEFAULT_TEACHER_MODEL_ID,
+    "--base-start-index",
+    "60000",
+    "--shard-prefix",
+    "data/finetune/shards/figment_sft_v5_full_shard",
+    "--output",
+    "data/finetune/figment_sft_v5.jsonl",
+    "--case-specs",
+    "data/finetune/figment_sft_v5_case_specs.jsonl",
+    "--manifest",
+    "data/finetune/figment_sft_v5_manifest.json",
+    "--modal-output-dir",
+    "data/finetune/modal/figment_sft_v5",
+    "--seed",
+    "figment-modal-sft-v5",
+]
+
+
+def build_corpus_args(argv: list[str] | None = None) -> list[str]:
+    return DEFAULT_ARGS + list(sys.argv[1:] if argv is None else argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    return _run_full_corpus(build_corpus_args(argv))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/generate_v6_full_corpus.py

@@ -0,0 +1,66 @@
+"""Generate, verify, and Modal-prep the Figment v6 delta SFT corpus."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import sys
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from scripts.generate_finetune_data import V6_NAVIGATOR_COUNTS  # noqa: E402
+from scripts.generate_v3_full_corpus import main as _run_full_corpus  # noqa: E402
+
+
+DEFAULT_OUTPUT_VERSION = "figment_sft_v6_delta"
+DEFAULT_TEACHER_MODEL_ID = "nvidia/nemotron-3-ultra-550b-a55b:free"
+DEFAULT_NAVIGATOR_COUNT = sum(V6_NAVIGATOR_COUNTS.values())
+DEFAULT_REPAIR_COUNT = 250
+DEFAULT_ARGS = [
+    "--dataset-version",
+    DEFAULT_OUTPUT_VERSION,
+    "--navigator-count",
+    str(DEFAULT_NAVIGATOR_COUNT),
+    "--repair-count",
+    str(DEFAULT_REPAIR_COUNT),
+    "--teacher-model-id",
+    DEFAULT_TEACHER_MODEL_ID,
+    "--base-start-index",
+    "70000",
+    "--shard-prefix",
+    "data/finetune/shards/figment_sft_v6_delta_full_shard",
+    "--output",
+    "data/finetune/figment_sft_v6_delta.jsonl",
+    "--case-specs",
+    "data/finetune/figment_sft_v6_delta_case_specs.jsonl",
+    "--manifest",
+    "data/finetune/figment_sft_v6_delta_manifest.json",
+    "--modal-output-dir",
+    "data/finetune/modal/figment_sft_v6_delta",
+    "--seed",
+    "figment-modal-sft-v6-delta",
+]
+
+
+def build_corpus_args(argv: list[str] | None = None) -> list[str]:
+    raw_args = list(sys.argv[1:] if argv is None else argv)
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument("--new-delta-count", type=int, default=1000)
+    parser.add_argument("--correction-count", type=int, default=180)
+    parser.add_argument("--repair-count", type=int, default=DEFAULT_REPAIR_COUNT)
+    parsed, remaining = parser.parse_known_args(raw_args)
+    navigator_count = parsed.new_delta_count + parsed.correction_count
+    defaults = list(DEFAULT_ARGS)
+    defaults[defaults.index("--navigator-count") + 1] = str(navigator_count)
+    defaults[defaults.index("--repair-count") + 1] = str(parsed.repair_count)
+    return defaults + remaining
+
+
+def main(argv: list[str] | None = None) -> int:
+    return _run_full_corpus(build_corpus_args(argv))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/generate_v7_full_corpus.py

@@ -0,0 +1,64 @@
+"""Generate, verify, and Modal-prep the Figment v7 delta SFT corpus."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import sys
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from scripts.generate_finetune_data import V7_NAVIGATOR_COUNTS  # noqa: E402
+from scripts.generate_v3_full_corpus import main as _run_full_corpus  # noqa: E402
+
+
+DEFAULT_OUTPUT_VERSION = "figment_sft_v7_delta"
+DEFAULT_TEACHER_MODEL_ID = "nvidia/nemotron-3-ultra-550b-a55b:free"
+DEFAULT_NAVIGATOR_COUNT = sum(V7_NAVIGATOR_COUNTS.values())
+DEFAULT_REPAIR_COUNT = 240
+DEFAULT_ARGS = [
+    "--dataset-version",
+    DEFAULT_OUTPUT_VERSION,
+    "--navigator-count",
+    str(DEFAULT_NAVIGATOR_COUNT),
+    "--repair-count",
+    str(DEFAULT_REPAIR_COUNT),
+    "--teacher-model-id",
+    DEFAULT_TEACHER_MODEL_ID,
+    "--base-start-index",
+    "80000",
+    "--shard-prefix",
+    "data/finetune/shards/figment_sft_v7_delta_full_shard",
+    "--output",
+    "data/finetune/figment_sft_v7_delta.jsonl",
+    "--case-specs",
+    "data/finetune/figment_sft_v7_delta_case_specs.jsonl",
+    "--manifest",
+    "data/finetune/figment_sft_v7_delta_manifest.json",
+    "--modal-output-dir",
+    "data/finetune/modal/figment_sft_v7_delta",
+    "--seed",
+    "figment-modal-sft-v7-delta",
+]
+
+
+def build_corpus_args(argv: list[str] | None = None) -> list[str]:
+    raw_args = list(sys.argv[1:] if argv is None else argv)
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument("--navigator-count", type=int, default=DEFAULT_NAVIGATOR_COUNT)
+    parser.add_argument("--repair-count", type=int, default=DEFAULT_REPAIR_COUNT)
+    parsed, remaining = parser.parse_known_args(raw_args)
+    defaults = list(DEFAULT_ARGS)
+    defaults[defaults.index("--navigator-count") + 1] = str(parsed.navigator_count)
+    defaults[defaults.index("--repair-count") + 1] = str(parsed.repair_count)
+    return defaults + remaining
+
+
+def main(argv: list[str] | None = None) -> int:
+    return _run_full_corpus(build_corpus_args(argv))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/generate_v8_full_corpus.py

@@ -0,0 +1,66 @@
+"""Generate, verify, and Modal-prep the Figment v8 delta SFT corpus."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import sys
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from scripts.generate_finetune_data import V8_NAVIGATOR_COUNTS  # noqa: E402
+from scripts.generate_v3_full_corpus import main as _run_full_corpus  # noqa: E402
+
+
+DEFAULT_OUTPUT_VERSION = "figment_sft_v8_delta"
+DEFAULT_TEACHER_MODEL_ID = "nvidia/nemotron-3-ultra-550b-a55b:free"
+DEFAULT_NAVIGATOR_COUNT = sum(V8_NAVIGATOR_COUNTS.values())
+DEFAULT_REPAIR_COUNT = 0
+DEFAULT_ARGS = [
+    "--dataset-version",
+    DEFAULT_OUTPUT_VERSION,
+    "--navigator-count",
+    str(DEFAULT_NAVIGATOR_COUNT),
+    "--repair-count",
+    str(DEFAULT_REPAIR_COUNT),
+    "--teacher-model-id",
+    DEFAULT_TEACHER_MODEL_ID,
+    "--base-start-index",
+    "90000",
+    "--rows-per-shard",
+    "40",
+    "--shard-prefix",
+    "data/finetune/shards/figment_sft_v8_delta_full_shard",
+    "--output",
+    "data/finetune/figment_sft_v8_delta.jsonl",
+    "--case-specs",
+    "data/finetune/figment_sft_v8_delta_case_specs.jsonl",
+    "--manifest",
+    "data/finetune/figment_sft_v8_delta_manifest.json",
+    "--modal-output-dir",
+    "data/finetune/modal/figment_sft_v8_delta",
+    "--seed",
+    "figment-modal-sft-v8-delta",
+]
+
+
+def build_corpus_args(argv: list[str] | None = None) -> list[str]:
+    raw_args = list(sys.argv[1:] if argv is None else argv)
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument("--navigator-count", type=int, default=DEFAULT_NAVIGATOR_COUNT)
+    parser.add_argument("--repair-count", type=int, default=DEFAULT_REPAIR_COUNT)
+    parsed, remaining = parser.parse_known_args(raw_args)
+    defaults = list(DEFAULT_ARGS)
+    defaults[defaults.index("--navigator-count") + 1] = str(parsed.navigator_count)
+    defaults[defaults.index("--repair-count") + 1] = str(parsed.repair_count)
+    return defaults + remaining
+
+
+def main(argv: list[str] | None = None) -> int:
+    return _run_full_corpus(build_corpus_args(argv))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/generate_v9_full_corpus.py

@@ -0,0 +1,66 @@
+"""Generate, verify, and Modal-prep the Figment v9 delta SFT corpus."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import sys
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from scripts.generate_finetune_data import V9_NAVIGATOR_COUNTS  # noqa: E402
+from scripts.generate_v3_full_corpus import main as _run_full_corpus  # noqa: E402
+
+
+DEFAULT_OUTPUT_VERSION = "figment_sft_v9_delta"
+DEFAULT_TEACHER_MODEL_ID = "nvidia/nemotron-3-ultra-550b-a55b:free"
+DEFAULT_NAVIGATOR_COUNT = sum(V9_NAVIGATOR_COUNTS.values())
+DEFAULT_REPAIR_COUNT = 0
+DEFAULT_ARGS = [
+    "--dataset-version",
+    DEFAULT_OUTPUT_VERSION,
+    "--navigator-count",
+    str(DEFAULT_NAVIGATOR_COUNT),
+    "--repair-count",
+    str(DEFAULT_REPAIR_COUNT),
+    "--teacher-model-id",
+    DEFAULT_TEACHER_MODEL_ID,
+    "--base-start-index",
+    "100000",
+    "--rows-per-shard",
+    "40",
+    "--shard-prefix",
+    "data/finetune/shards/figment_sft_v9_delta_full_shard",
+    "--output",
+    "data/finetune/figment_sft_v9_delta.jsonl",
+    "--case-specs",
+    "data/finetune/figment_sft_v9_delta_case_specs.jsonl",
+    "--manifest",
+    "data/finetune/figment_sft_v9_delta_manifest.json",
+    "--modal-output-dir",
+    "data/finetune/modal/figment_sft_v9_delta",
+    "--seed",
+    "figment-modal-sft-v9-delta",
+]
+
+
+def build_corpus_args(argv: list[str] | None = None) -> list[str]:
+    raw_args = list(sys.argv[1:] if argv is None else argv)
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument("--navigator-count", type=int, default=DEFAULT_NAVIGATOR_COUNT)
+    parser.add_argument("--repair-count", type=int, default=DEFAULT_REPAIR_COUNT)
+    parsed, remaining = parser.parse_known_args(raw_args)
+    defaults = list(DEFAULT_ARGS)
+    defaults[defaults.index("--navigator-count") + 1] = str(parsed.navigator_count)
+    defaults[defaults.index("--repair-count") + 1] = str(parsed.repair_count)
+    return defaults + remaining
+
+
+def main(argv: list[str] | None = None) -> int:
+    return _run_full_corpus(build_corpus_args(argv))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/merge_finetune_shards.py

@@ -0,0 +1,196 @@
+"""Merge disjoint Figment SFT teacher-generation shards."""
+
+from __future__ import annotations
+
+import argparse
+from collections import Counter
+from datetime import UTC
+from datetime import datetime
+import hashlib
+import json
+from pathlib import Path
+from typing import Any
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--dataset-version", required=True)
+    parser.add_argument("--shard-prefix", type=Path, required=True)
+    parser.add_argument("--shard-count", type=int, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--case-specs", type=Path, required=True)
+    parser.add_argument("--manifest", type=Path, required=True)
+    args = parser.parse_args(argv)
+
+    manifest = merge_shards(
+        dataset_version=args.dataset_version,
+        shard_prefix=args.shard_prefix,
+        shard_count=args.shard_count,
+        output_path=args.output,
+        case_specs_path=args.case_specs,
+        manifest_path=args.manifest,
+    )
+    print(json.dumps(manifest, indent=2, sort_keys=True))
+    return 0
+
+
+def merge_shards(
+    *,
+    dataset_version: str,
+    shard_prefix: Path,
+    shard_count: int,
+    output_path: Path,
+    case_specs_path: Path,
+    manifest_path: Path,
+) -> dict[str, Any]:
+    if shard_count <= 0:
+        raise ValueError("shard_count must be positive")
+
+    rows_by_id: dict[str, dict[str, Any]] = {}
+    specs_by_id: dict[str, dict[str, Any]] = {}
+    shard_summaries: list[dict[str, Any]] = []
+    source_attempts = 0
+    anti_overfit_eval_paths: set[str] = set()
+    anti_overfit_signature_count = 0
+    anti_overfit_enabled = False
+
+    for shard_index in range(shard_count):
+        dataset_shard, spec_shard, manifest_shard = shard_paths(shard_prefix, shard_index)
+        rows = _read_jsonl(dataset_shard)
+        specs = _read_jsonl(spec_shard)
+        source_manifest = _read_json(manifest_shard) if manifest_shard.exists() else {}
+        source_attempts += int(source_manifest.get("attempts") or 0)
+        source_exclusions = source_manifest.get("anti_overfit_exclusions")
+        if isinstance(source_exclusions, dict):
+            anti_overfit_enabled = anti_overfit_enabled or bool(source_exclusions.get("enabled"))
+            anti_overfit_eval_paths.update(str(path) for path in source_exclusions.get("eval_paths", []) if str(path))
+            anti_overfit_signature_count = max(
+                anti_overfit_signature_count,
+                int(source_exclusions.get("signature_count") or 0),
+            )
+
+        for row in rows:
+            row_id = str(row.get("case_id") or row.get("uuid") or "")
+            if not row_id:
+                raise ValueError(f"{dataset_shard}: row missing case_id")
+            if str(row.get("version")) != dataset_version:
+                raise ValueError(f"{dataset_shard}: {row_id} version does not match {dataset_version}")
+            if row_id in rows_by_id:
+                raise ValueError(f"duplicate row id across shards: {row_id}")
+            rows_by_id[row_id] = row
+
+        for spec in specs:
+            spec_id = str(spec.get("case_id") or "")
+            if not spec_id:
+                raise ValueError(f"{spec_shard}: spec missing case_id")
+            if str(spec.get("dataset_version") or dataset_version) != dataset_version:
+                raise ValueError(f"{spec_shard}: {spec_id} dataset_version does not match {dataset_version}")
+            if spec_id in specs_by_id:
+                raise ValueError(f"duplicate case spec id across shards: {spec_id}")
+            specs_by_id[spec_id] = spec
+
+        shard_summaries.append(
+            {
+                "index": shard_index,
+                "dataset_path": str(dataset_shard),
+                "case_specs_path": str(spec_shard),
+                "manifest_path": str(manifest_shard),
+                "row_count": len(rows),
+                "case_spec_count": len(specs),
+                "attempts": int(source_manifest.get("attempts") or 0),
+                "accepted_by_failure_class": source_manifest.get("accepted_by_failure_class", {}),
+                "rejection_reasons": source_manifest.get("rejection_reasons", {}),
+            }
+        )
+
+    missing_specs = sorted(set(rows_by_id) - set(specs_by_id))
+    orphan_specs = sorted(set(specs_by_id) - set(rows_by_id))
+    if missing_specs:
+        raise ValueError(f"missing case specs for rows: {', '.join(missing_specs[:10])}")
+    if orphan_specs:
+        raise ValueError(f"case specs without rows: {', '.join(orphan_specs[:10])}")
+
+    rows = [rows_by_id[row_id] for row_id in sorted(rows_by_id)]
+    specs = [specs_by_id[case_id] for case_id in sorted(specs_by_id)]
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    case_specs_path.parent.mkdir(parents=True, exist_ok=True)
+    manifest_path.parent.mkdir(parents=True, exist_ok=True)
+    _write_jsonl(output_path, rows)
+    _write_jsonl(case_specs_path, specs)
+
+    task_counts = Counter(_task_type(row) for row in rows)
+    category_counts = Counter(str(row.get("category") or "unknown") for row in rows)
+    summary = {
+        "dataset_version": dataset_version,
+        "merged_at": datetime.now(UTC).isoformat(),
+        "row_count": len(rows),
+        "case_spec_count": len(specs),
+        "shard_count": shard_count,
+        "source_attempts": source_attempts,
+        "output_path": str(output_path),
+        "case_specs_path": str(case_specs_path),
+        "output_sha256": _sha256_path(output_path),
+        "case_specs_sha256": _sha256_path(case_specs_path),
+        "task_type_counts": dict(sorted(task_counts.items())),
+        "category_counts": dict(sorted(category_counts.items())),
+        "anti_overfit_exclusions": {
+            "enabled": anti_overfit_enabled,
+            "eval_paths": sorted(anti_overfit_eval_paths),
+            "signature_count": anti_overfit_signature_count,
+        },
+        "shards": shard_summaries,
+    }
+    manifest_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return summary
+
+
+def shard_paths(shard_prefix: Path, shard_index: int) -> tuple[Path, Path, Path]:
+    base = f"{shard_prefix}{shard_index}"
+    return (
+        Path(f"{base}.jsonl"),
+        Path(f"{base}_case_specs.jsonl"),
+        Path(f"{base}_manifest.json"),
+    )
+
+
+def _read_jsonl(path: Path) -> list[dict[str, Any]]:
+    if not path.exists():
+        raise FileNotFoundError(path)
+    rows: list[dict[str, Any]] = []
+    for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
+        if not line.strip():
+            continue
+        item = json.loads(line)
+        if not isinstance(item, dict):
+            raise ValueError(f"{path}:{line_number}: expected JSON object")
+        rows.append(item)
+    return rows
+
+
+def _read_json(path: Path) -> dict[str, Any]:
+    item = json.loads(path.read_text(encoding="utf-8"))
+    if not isinstance(item, dict):
+        raise ValueError(f"{path}: expected JSON object")
+    return item
+
+
+def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
+    path.write_text("".join(json.dumps(row, sort_keys=True) + "\n" for row in rows), encoding="utf-8")
+
+
+def _task_type(row: dict[str, Any]) -> str:
+    metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
+    return str(metadata.get("task_type") or "navigator_full")
+
+
+def _sha256_path(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as file:
+        for chunk in iter(lambda: file.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/merge_v10_training_corpus.py

@@ -0,0 +1,67 @@
+"""Merge Figment v10 delta rows with the v9 corpus and prepare Modal splits."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import sys
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from scripts.merge_v8_training_corpus import main as _merge_main  # noqa: E402
+
+
+DEFAULT_BASE = "data/finetune/figment_sft_v9.jsonl"
+DEFAULT_BASE_CASE_SPECS = "data/finetune/figment_sft_v9_case_specs.jsonl"
+DEFAULT_DELTA = "data/finetune/figment_sft_v10_delta.jsonl"
+DEFAULT_DELTA_CASE_SPECS = "data/finetune/figment_sft_v10_delta_case_specs.jsonl"
+DEFAULT_OUTPUT = "data/finetune/figment_sft_v10.jsonl"
+DEFAULT_CASE_SPECS = "data/finetune/figment_sft_v10_case_specs.jsonl"
+DEFAULT_MANIFEST = "data/finetune/figment_sft_v10_manifest.json"
+DEFAULT_MODAL_DIR = "data/finetune/modal/figment_sft_v10"
+
+
+def build_merge_args(argv: list[str] | None = None) -> list[str]:
+    raw_args = list(sys.argv[1:] if argv is None else argv)
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument("--base", default=DEFAULT_BASE)
+    parser.add_argument("--base-case-specs", default=DEFAULT_BASE_CASE_SPECS)
+    parser.add_argument("--delta", default=DEFAULT_DELTA)
+    parser.add_argument("--delta-case-specs", default=DEFAULT_DELTA_CASE_SPECS)
+    parser.add_argument("--output", default=DEFAULT_OUTPUT)
+    parser.add_argument("--case-specs", default=DEFAULT_CASE_SPECS)
+    parser.add_argument("--manifest", default=DEFAULT_MANIFEST)
+    parser.add_argument("--modal-output-dir", default=DEFAULT_MODAL_DIR)
+    parser.add_argument("--dataset-version", default="figment_sft_v10")
+    parsed, remaining = parser.parse_known_args(raw_args)
+    return [
+        "--base",
+        parsed.base,
+        "--base-case-specs",
+        parsed.base_case_specs,
+        "--delta",
+        parsed.delta,
+        "--delta-case-specs",
+        parsed.delta_case_specs,
+        "--output",
+        parsed.output,
+        "--case-specs",
+        parsed.case_specs,
+        "--manifest",
+        parsed.manifest,
+        "--modal-output-dir",
+        parsed.modal_output_dir,
+        "--dataset-version",
+        parsed.dataset_version,
+        *remaining,
+    ]
+
+
+def main(argv: list[str] | None = None) -> int:
+    return _merge_main(build_merge_args(argv))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/merge_v11_training_corpus.py

@@ -0,0 +1,67 @@
+"""Merge Figment v11 delta rows with the v10 corpus and prepare Modal splits."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import sys
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from scripts.merge_v8_training_corpus import main as _merge_main  # noqa: E402
+
+
+DEFAULT_BASE = "data/finetune/figment_sft_v10.jsonl"
+DEFAULT_BASE_CASE_SPECS = "data/finetune/figment_sft_v10_case_specs.jsonl"
+DEFAULT_DELTA = "data/finetune/figment_sft_v11_delta.jsonl"
+DEFAULT_DELTA_CASE_SPECS = "data/finetune/figment_sft_v11_delta_case_specs.jsonl"
+DEFAULT_OUTPUT = "data/finetune/figment_sft_v11.jsonl"
+DEFAULT_CASE_SPECS = "data/finetune/figment_sft_v11_case_specs.jsonl"
+DEFAULT_MANIFEST = "data/finetune/figment_sft_v11_manifest.json"
+DEFAULT_MODAL_DIR = "data/finetune/modal/figment_sft_v11"
+
+
+def build_merge_args(argv: list[str] | None = None) -> list[str]:
+    raw_args = list(sys.argv[1:] if argv is None else argv)
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument("--base", default=DEFAULT_BASE)
+    parser.add_argument("--base-case-specs", default=DEFAULT_BASE_CASE_SPECS)
+    parser.add_argument("--delta", default=DEFAULT_DELTA)
+    parser.add_argument("--delta-case-specs", default=DEFAULT_DELTA_CASE_SPECS)
+    parser.add_argument("--output", default=DEFAULT_OUTPUT)
+    parser.add_argument("--case-specs", default=DEFAULT_CASE_SPECS)
+    parser.add_argument("--manifest", default=DEFAULT_MANIFEST)
+    parser.add_argument("--modal-output-dir", default=DEFAULT_MODAL_DIR)
+    parser.add_argument("--dataset-version", default="figment_sft_v11")
+    parsed, remaining = parser.parse_known_args(raw_args)
+    return [
+        "--base",
+        parsed.base,
+        "--base-case-specs",
+        parsed.base_case_specs,
+        "--delta",
+        parsed.delta,
+        "--delta-case-specs",
+        parsed.delta_case_specs,
+        "--output",
+        parsed.output,
+        "--case-specs",
+        parsed.case_specs,
+        "--manifest",
+        parsed.manifest,
+        "--modal-output-dir",
+        parsed.modal_output_dir,
+        "--dataset-version",
+        parsed.dataset_version,
+        *remaining,
+    ]
+
+
+def main(argv: list[str] | None = None) -> int:
+    return _merge_main(build_merge_args(argv))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/merge_v12_training_corpus.py

@@ -0,0 +1,67 @@
+"""Merge Figment v12 delta rows with the v10 corpus and prepare Modal splits."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import sys
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from scripts.merge_v8_training_corpus import main as _merge_main  # noqa: E402
+
+
+DEFAULT_BASE = "data/finetune/figment_sft_v10.jsonl"
+DEFAULT_BASE_CASE_SPECS = "data/finetune/figment_sft_v10_case_specs.jsonl"
+DEFAULT_DELTA = "data/finetune/figment_sft_v12_delta.jsonl"
+DEFAULT_DELTA_CASE_SPECS = "data/finetune/figment_sft_v12_delta_case_specs.jsonl"
+DEFAULT_OUTPUT = "data/finetune/figment_sft_v12.jsonl"
+DEFAULT_CASE_SPECS = "data/finetune/figment_sft_v12_case_specs.jsonl"
+DEFAULT_MANIFEST = "data/finetune/figment_sft_v12_manifest.json"
+DEFAULT_MODAL_DIR = "data/finetune/modal/figment_sft_v12"
+
+
+def build_merge_args(argv: list[str] | None = None) -> list[str]:
+    raw_args = list(sys.argv[1:] if argv is None else argv)
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument("--base", default=DEFAULT_BASE)
+    parser.add_argument("--base-case-specs", default=DEFAULT_BASE_CASE_SPECS)
+    parser.add_argument("--delta", default=DEFAULT_DELTA)
+    parser.add_argument("--delta-case-specs", default=DEFAULT_DELTA_CASE_SPECS)
+    parser.add_argument("--output", default=DEFAULT_OUTPUT)
+    parser.add_argument("--case-specs", default=DEFAULT_CASE_SPECS)
+    parser.add_argument("--manifest", default=DEFAULT_MANIFEST)
+    parser.add_argument("--modal-output-dir", default=DEFAULT_MODAL_DIR)
+    parser.add_argument("--dataset-version", default="figment_sft_v12")
+    parsed, remaining = parser.parse_known_args(raw_args)
+    return [
+        "--base",
+        parsed.base,
+        "--base-case-specs",
+        parsed.base_case_specs,
+        "--delta",
+        parsed.delta,
+        "--delta-case-specs",
+        parsed.delta_case_specs,
+        "--output",
+        parsed.output,
+        "--case-specs",
+        parsed.case_specs,
+        "--manifest",
+        parsed.manifest,
+        "--modal-output-dir",
+        parsed.modal_output_dir,
+        "--dataset-version",
+        parsed.dataset_version,
+        *remaining,
+    ]
+
+
+def main(argv: list[str] | None = None) -> int:
+    return _merge_main(build_merge_args(argv))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())