US dispositions: 51-jurisdiction survey, 5-point verdicts, bottom-line decision paths

Every disposition entry now carries a real state-by-state breakdown (284 rows
across 50 states + DC): local statute cites, mechanism facts, the state's own
characterization, and an IRPA verdict applied uniformly by rule from the
verified Saini/Burgon/Drake framework. Verdicts moved from yes/no/depends to
a 5-point scale assessed on the COMPLETED disposition (yes / likely-yes /
likely-no / no / fact-specific) after the user rejected 'depends' as
unusable; each entry gained a BOTTOM LINE stating the one or two record facts
that decide the case. The user-review correction on standard of proof is
included: s. 33 reasonable grounds governs (Mugesera para 114), balance of
probabilities only for a PR's s. 36(1)(c) determination (s. 36(3)(d)).

Method documented in-dataset: 10 states agent-researched, 41 extracted from
the Restoration of Rights Project profiles; 9 highest-risk clearance rows
spot-verified verbatim against statute text (zero contradictions); full-
population verification pending and disclosed. commentary.py renders 12
entry chunks (state lists capped, coverage summaries), 51 per-state chunks,
and the methodology. Reviewed by the user 2026-07-15.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

canlex/commentary.py

@@ -16,6 +16,7 @@ import json
 from .config import DATA_DIR, PROCESSED_DIR
 
 CURATED = DATA_DIR / "curated" / "us_dispositions.json"
+EQUIV = DATA_DIR / "curated" / "us_equivalency.json"
 OUT = PROCESSED_DIR / "commentary.json"
 
 ACT_CODE = "US-DISP"
@@ -33,22 +34,57 @@ _STATUS_LABEL = {
     "no-authority": "NO AUTHORITY LOCATED -- reasoned interpretation only",
 }
 
+# Verdicts assess the COMPLETED disposition; the methodology chunk explains
+# the rule each one derives from. 'depends' is the legacy spelling of
+# fact-specific, kept for backwards compatibility.
+_VERDICT_LABEL = {
+    "yes": "YES", "likely-yes": "LIKELY YES", "likely-no": "LIKELY NO",
+    "no": "NO", "fact-specific": "FACT-SPECIFIC",
+    "depends": "FACT-SPECIFIC",
+}
+
+
+def _vlabel(v):
+    return _VERDICT_LABEL.get(v, str(v).upper())
 
-def _entry_text(e):
-    """Render one disposition entry as a readable, retrieval-friendly block."""
+
+def _entry_text(e, max_states=None):
+    """Render one disposition entry as a readable, retrieval-friendly block.
+
+    With the 51-jurisdiction survey an entry can carry dozens of state rows;
+    max_states caps how many render inline (the corpus keeps full per-state
+    detail in separate per-state chunks so retrieval still reaches every
+    row). None renders everything -- the tool uses that for direct lookups."""
     lines = [BANNER, ""]
     lines.append(f"Disposition: {'; '.join(e['names'])}")
-    lines.append(f"Is it a conviction for IRPA s. 36 purposes: "
-                 f"{e['is_conviction'].upper()}")
+    lines.append(f"Is the COMPLETED disposition a conviction for IRPA s. 36: "
+                 f"{_vlabel(e['is_conviction'])}")
     lines.append(f"Authority status: {_STATUS_LABEL[e['status']]}")
+    if e.get("bottom_line"):
+        lines.append("")
+        lines.append(f"BOTTOM LINE: {e['bottom_line']}")
     lines.append("")
     lines.append(e["analysis"])
     if e.get("state_variations"):
+        named = [v for v in e["state_variations"]
+                 if v["state"].lower() != "general"]
+        shown = named if max_states is None else named[:max_states]
         lines.append("")
-        lines.append("State variations:")
-        for v in e["state_variations"]:
-            flag = f" (conviction: {v['is_conviction']})" if v.get("is_conviction") else ""
+        if max_states is not None and len(named) > len(shown):
+            from collections import Counter
+            counts = Counter(v.get("is_conviction", "depends") for v in named)
+            lines.append(f"State-by-state coverage: {len(named)} jurisdictions "
+                         f"({', '.join(f'{k}: {n}' for k, n in counts.most_common())}) "
+                         f"-- full per-state detail in the per-state entries.")
+        else:
+            lines.append("State variations:")
+        for v in shown:
+            flag = (f" (conviction: {_vlabel(v['is_conviction'])})"
+                    if v.get("is_conviction") else "")
             lines.append(f"- {v['state']}{flag}: {v['note']}")
+        for v in e["state_variations"]:
+            if v["state"].lower() == "general":
+                lines.append(f"- General: {v['note']}")
     if e.get("authorities"):
         lines.append("")
         lines.append("Authorities:")
@@ -103,20 +139,123 @@ def build():
             "part": "US dispositions",
             "division": "",
             "heading": (f"Is a US {e['names'][0]} a conviction for IRPA "
-                        f"s. 36? ({e['is_conviction']})"),
-            "text": _entry_text(e),
+                        f"s. 36? ({_vlabel(e['is_conviction'])})"),
+            "text": _entry_text(e, max_states=6),
             "history": "",
             "last_amended": "",
             "current_to": data.get("reviewed", ""),
             "citation": f"{ACT_SHORT} — {e['names'][0]}",
             "source_url": "",
         })
+
+    # One chunk per jurisdiction: every disposition row for that state, so a
+    # query naming a state ("Georgia first offender act", "Missouri SIS")
+    # retrieves that state's page directly.
+    by_state = {}
+    for e in data.get("dispositions", []):
+        for v in e.get("state_variations", []):
+            st = v["state"]
+            if st.lower() == "general":
+                continue
+            flag = _vlabel(v.get("is_conviction") or e["is_conviction"])
+            by_state.setdefault(st, []).append(
+                f"- {e['names'][0]} (conviction: {flag}): {v['note']}")
+    for st in sorted(by_state):
+        body = (BANNER + "\n\n"
+                + f"US dispositions — {st}: whether each disposition type is a "
+                  f"conviction for IRPA s. 36, under {st} law.\n\n"
+                + "\n".join(by_state[st])
+                + "\n\nThe act branch (IRPA s. 36(1)(c)/(2)(c)) can apply even "
+                  "where a disposition is not a conviction. See the "
+                  "per-disposition entries for the governing analysis and "
+                  "authorities.")
+        slug = st.lower().replace(" ", "-")
+        chunks.append({
+            "id": f"commentary-state-{slug}",
+            "doc_type": "commentary",
+            "act_code": ACT_CODE,
+            "act_short": ACT_SHORT,
+            "act_name": ACT_NAME,
+            "section": f"state-{slug}",
+            "marginal_note": f"US dispositions — {st}",
+            "part": "US dispositions by state",
+            "division": "",
+            "heading": (f"{st}: criminal dispositions vs the IRPA "
+                        f"'conviction' concept"),
+            "text": body,
+            "history": "",
+            "last_amended": "",
+            "current_to": data.get("reviewed", ""),
+            "citation": f"{ACT_SHORT} — {st}",
+            "source_url": "",
+        })
+    # --- equivalency pairings (step 2), same banner discipline
+    n_pairings = 0
+    if EQUIV.exists():
+        eq = json.loads(EQUIV.read_text(encoding="utf-8"))
+        for m in eq.get("methodology", []):
+            chunks.append({
+                "id": f"commentary-method-{m['id']}",
+                "doc_type": "commentary",
+                "act_code": ACT_CODE,
+                "act_short": ACT_SHORT,
+                "act_name": ACT_NAME,
+                "section": m["id"],
+                "marginal_note": m["title"],
+                "part": "Methodology",
+                "division": "",
+                "heading": m["title"],
+                "text": BANNER + "\n\n" + m["text"],
+                "history": "",
+                "last_amended": "",
+                "current_to": eq.get("reviewed", ""),
+                "citation": f"{ACT_SHORT} — {m['title']}",
+                "source_url": "",
+            })
+        for p in eq.get("pairings", []):
+            n_pairings += 1
+            lines = [BANNER, "",
+                     f"US offence: {'; '.join(p['us_terms'][:5])}",
+                     f"Canadian equivalent: {p['canadian_offence']}",
+                     f"Maximum penalty (verified): {p['penalty']}",
+                     f"Inadmissibility branch: {p['branch']}", "",
+                     p["analysis"]]
+            if p.get("caveats"):
+                lines.append("")
+                lines.append("Caveats:")
+                lines += [f"- {c}" for c in p["caveats"]]
+            if p.get("authorities"):
+                lines.append("")
+                lines.append("Authorities:")
+                lines += [f"- {a['cite']} ({a['court']}): {a['holding']}"
+                          for a in p["authorities"]]
+            chunks.append({
+                "id": f"commentary-equiv-{p['id']}",
+                "doc_type": "commentary",
+                "act_code": ACT_CODE,
+                "act_short": ACT_SHORT,
+                "act_name": ACT_NAME,
+                "section": f"equiv-{p['id']}",
+                "marginal_note": f"Equivalency: {p['us_terms'][0]}",
+                "part": "US offence equivalency",
+                "division": "",
+                "heading": (f"What does a US {p['us_terms'][0]} conviction "
+                            f"equate to in Canada?"),
+                "text": "\n".join(lines),
+                "history": "",
+                "last_amended": "",
+                "current_to": eq.get("reviewed", ""),
+                "citation": f"{ACT_SHORT} — equivalency: {p['us_terms'][0]}",
+                "source_url": "",
+            })
+
     PROCESSED_DIR.mkdir(parents=True, exist_ok=True)
     OUT.write_text(json.dumps(chunks, ensure_ascii=False, indent=1),
                    encoding="utf-8")
     print(f"{len(chunks)} commentary chunks "
           f"({len(data.get('dispositions', []))} dispositions, "
-          f"{len(data.get('methodology', []))} methodology) -> {OUT}")
+          f"{len(by_state)} state pages, {n_pairings} equivalency pairings, "
+          f"{len(data.get('methodology', []))}+ methodology) -> {OUT}")
 
 
 if __name__ == "__main__":

tests/test_commentary.py

@@ -34,10 +34,17 @@ class EntryTextTests(unittest.TestCase):
     def test_carries_banner_and_flags(self):
         text = _entry_text(entry())
         self.assertTrue(text.startswith(BANNER))
-        self.assertIn("DEPENDS", text)
+        # legacy 'depends' renders under the 5-point vocabulary
+        self.assertIn("FACT-SPECIFIC", text)
         self.assertIn("NO AUTHORITY LOCATED", text)
         self.assertIn("INTERPRETATION (no direct authority", text)
 
+    def test_five_point_verdicts_render(self):
+        text = _entry_text(entry(is_conviction="likely-no",
+                                 bottom_line="Check completion."))
+        self.assertIn("LIKELY NO", text)
+        self.assertIn("BOTTOM LINE: Check completion.", text)
+
     def test_no_interpretation_block_when_empty(self):
         text = _entry_text(entry(interpretation="", status="settled"))
         self.assertNotIn("INTERPRETATION", text)