Filter-aware recall, accent folding + FR bridges, repealed penalty, eval slices, curated staleness watch

canlex/eval.py

@@ -6,10 +6,26 @@ retrieval index and reports Hit@k and MRR. Re-run it after any retrieval change
 -- a new reranker, different embeddings, a chunking tweak -- to see whether
 quality moved, and read the "Misses" list to see exactly what to fix.
 
+Question schema (per item):
+    query      the question text
+    answers    [[act_short, section], ...]; empty section matches any chunk
+    style      optional tag ('standard' when absent): 'fact-pattern',
+               'keyword', 'typo', 'french', 'no-answer', ... -- metrics are
+               reported per style so a hard new slice cannot silently drag
+               the comparable legacy numbers
+    holdout    optional bool; holdout questions are excluded from parameter
+               sweeps (sweep.py) so knob choices cannot fit their noise --
+               eval.py reports them separately
+    'no-answer' style items carry no answers: they exist to calibrate the
+    confidence signal (a good corpus-coverage score should be LOW on them),
+    reported as a separate table, never as misses.
+
     py -m canlex.eval
 """
 import json
+import os
 import sys
+from collections import defaultdict
 
 from .config import ROOT
 from .index import LegislationIndex
@@ -17,6 +33,29 @@ from .index import LegislationIndex
 QUESTIONS = ROOT / "data" / "eval" / "questions.json"
 EVAL_TOP_K = 20      # search depth, so ranks past the usual 6 are still visible
 
+# The historical baseline slice: styles that existed before the 2026-07
+# expansion. Numbers over this slice stay comparable across commits.
+_LEGACY_STYLES = {"standard"}
+
+
+def load_questions():
+    """The question set with defaults applied; None if the file is absent."""
+    if not QUESTIONS.exists():
+        return None
+    items = json.loads(QUESTIONS.read_text(encoding="utf-8"))
+    for item in items:
+        item.setdefault("style", "standard")
+        item.setdefault("holdout", False)
+    return items
+
+
+def sweep_questions():
+    """The slice parameter sweeps may see (sweep.py calls this): everything
+    except holdout items and the no-answer calibration set."""
+    items = load_questions() or []
+    return [i for i in items
+            if not i["holdout"] and i["style"] != "no-answer"]
+
 
 def _matches(result, answers):
     """True if a search result is one of the gold answers (act + section).
@@ -33,15 +72,41 @@ def _matches(result, answers):
     return False
 
 
+def _metrics(ranks):
+    n = len(ranks) or 1
+    hit = lambda k: sum(1 for r in ranks if 0 < r <= k) / n
+    mrr = sum(1.0 / r for r in ranks if r) / n
+    return hit(1), hit(3), hit(5), hit(10), mrr
+
+
+def _report_line(label, ranks):
+    h1, h3, h5, h10, mrr = _metrics(ranks)
+    print(f"  {label:24} n={len(ranks):<4} Hit@1 {h1:.2f}  Hit@3 {h3:.2f}  "
+          f"Hit@5 {h5:.2f}  Hit@10 {h10:.2f}  MRR {mrr:.2f}")
+
+
 def run():
-    if not QUESTIONS.exists():
+    items = load_questions()
+    if items is None:
         print(f"No question set at {QUESTIONS}.", file=sys.stderr)
         return
-    items = json.loads(QUESTIONS.read_text(encoding="utf-8"))
+    # Sweep mode (CANLEX_EVAL_SWEEP=1, set by sweep.py): score only the
+    # sweep-visible slice and print the legacy vertical metric block its
+    # parser expects -- holdout questions stay blind to every knob choice.
+    sweep_mode = bool(os.environ.get("CANLEX_EVAL_SWEEP"))
+    if sweep_mode:
+        items = [i for i in items
+                 if not i["holdout"] and i["style"] != "no-answer"]
     index = LegislationIndex()
-    ranks = []          # rank of the first gold hit per question (0 = miss)
+    ranks = []          # (item, rank); rank 0 = miss
     misses = []
+    no_answer = []      # (query, top confidence) for the calibration table
     for item in items:
+        if item["style"] == "no-answer":
+            results = index.search(item["query"], top_k=3)
+            conf = results[0].get("confidence") if results else None
+            no_answer.append((item["query"], conf))
+            continue
         answers = [tuple(a) for a in item["answers"]]
         results = index.search(item["query"], top_k=EVAL_TOP_K)
         rank = 0
@@ -49,29 +114,55 @@ def run():
             if _matches(result, answers):
                 rank = i
                 break
-        ranks.append(rank)
+        ranks.append((item, rank))
         if rank == 0 or rank > 5:
             top = results[0] if results else None
-            misses.append((item["query"], answers, rank, top))
+            misses.append((item, answers, rank, top))
 
-    n = len(ranks) or 1
-    hit = lambda k: sum(1 for r in ranks if 0 < r <= k) / n
-    mrr = sum(1.0 / r for r in ranks if r) / n
-    print(f"CanLex retrieval evaluation -- {len(ranks)} questions\n")
-    print(f"  Hit@1:   {hit(1):.2f}")
-    print(f"  Hit@3:   {hit(3):.2f}")
-    print(f"  Hit@5:   {hit(5):.2f}")
-    print(f"  Hit@10:  {hit(10):.2f}")
-    print(f"  MRR:     {mrr:.2f}")
+    all_ranks = [r for _, r in ranks]
+    if sweep_mode:
+        h1, h3, h5, h10, mrr = _metrics(all_ranks)
+        print(f"CanLex retrieval evaluation -- {len(all_ranks)} questions "
+              f"(sweep slice)\n")
+        print(f"  Hit@1:   {h1:.2f}")
+        print(f"  Hit@3:   {h3:.2f}")
+        print(f"  Hit@5:   {h5:.2f}")
+        print(f"  Hit@10:  {h10:.2f}")
+        print(f"  MRR:     {mrr:.2f}")
+        return
+    legacy = [r for it, r in ranks if it["style"] in _LEGACY_STYLES]
+    held = [r for it, r in ranks if it["holdout"]]
+    swept = [r for it, r in ranks if not it["holdout"]]
+    by_style = defaultdict(list)
+    for it, r in ranks:
+        by_style[it["style"]].append(r)
+
+    print(f"CanLex retrieval evaluation -- {len(all_ranks)} scored questions\n")
+    _report_line("ALL", all_ranks)
+    _report_line("legacy (comparable)", legacy)
+    for style in sorted(by_style):
+        if style not in _LEGACY_STYLES:
+            _report_line(f"style: {style}", by_style[style])
+    if held:
+        _report_line("holdout (sweep-blind)", held)
+        _report_line("sweep-visible", swept)
+
+    if no_answer:
+        print("\nNo-answer calibration (confidence should be LOW here; the "
+              "weak-match hedge fires below 0.72):")
+        for query, conf in no_answer:
+            shown = f"{conf:.3f}" if conf is not None else "n/a"
+            flag = "" if (conf is None or conf < 0.72) else "  <-- OVER-CONFIDENT"
+            print(f"  {shown}  {query[:70]}{flag}")
 
     if misses:
         print(f"\n{len(misses)} miss(es) -- gold answer ranked >5 or absent:")
-        for query, answers, rank, top in misses:
+        for item, answers, rank, top in misses:
             gold = ", ".join(f"{a} s.{s}".rstrip(" s.") for a, s in answers)
             where = f"ranked #{rank}" if rank else f"absent (searched {EVAL_TOP_K})"
             got = (f"{top.get('act_short', '')} s.{top.get('section', '')}".rstrip(" s.")
                    if top else "nothing")
-            print(f"  [{where}] {query}")
+            print(f"  [{where}] ({item['style']}) {item['query']}")
             print(f"      gold: {gold}   |   top result: {got}")
     print()
 

canlex/index.py

@@ -4,6 +4,7 @@ import math
 import os
 import re
 import sys
+import unicodedata
 from collections import Counter, defaultdict
 
 import snowballstemmer
@@ -37,6 +38,11 @@ REG_PENALTY = float(os.environ.get("CANLEX_REG_PENALTY", "0.004"))
 NIF_PENALTY = float(os.environ.get("CANLEX_NIF_PENALTY", "0.012"))
                     # fusion penalty on not-in-force amendment chunks, so
                     # pending law never outranks the live provision it amends
+REPEALED_PENALTY = float(os.environ.get("CANLEX_REPEALED_PENALTY", "0.008"))
+                    # likewise for repealed provisions (kept so lookups still
+                    # inform): a repealed section's on-topic title must not
+                    # beat live law in a close contest -- the CDSA-to-Cannabis
+                    # -Act migration is the live risk zone
 BACKMATTER_PENALTY = float(os.environ.get("CANLEX_BACKMATTER_PENALTY", "0.004"))
                     # likewise for a collective agreement's back-matter
                     # (memoranda, letters of understanding) vs its numbered articles
@@ -129,11 +135,22 @@ def _stem(word):
     return stemmed
 
 
+def _fold_accents(text):
+    """ASCII-fold accented characters so a French or mixed query tokenizes
+    usefully instead of shattering: the [a-z0-9] token pattern used to split
+    'détention' into ['d','tention']. NFKD + ASCII-encode drops the combining
+    marks ('détention' -> 'detention'), which both matches the English corpus
+    directly and lets the FR->EN synonym bridges fire."""
+    return unicodedata.normalize("NFKD", text).encode(
+        "ascii", "ignore").decode("ascii")
+
+
 def tokenize(text):
-    """Lower-case, split on word characters, and Snowball-stem each token, so a
-    query matches a provision even when their word forms differ -- 'possession'
-    vs 'possess', 'reporting' vs 'report', 'importation' vs 'import'."""
-    return [_stem(w) for w in _TOKEN.findall(text.lower())]
+    """Lower-case, accent-fold, split on word characters, and Snowball-stem
+    each token, so a query matches a provision even when their word forms
+    differ -- 'possession' vs 'possess', 'reporting' vs 'report',
+    'importation' vs 'import', 'détention' vs 'detention'."""
+    return [_stem(w) for w in _TOKEN.findall(_fold_accents(text.lower()))]
 
 
 def _section_refs(query):
@@ -240,6 +257,7 @@ class LegislationIndex:
         self._is_regulation = []
         self._is_backmatter = []
         self._is_nif = []
+        self._is_repealed = []
         for c in self.chunks:
             self._note_tokens.append(set(tokenize(topical_title(c))))
             self._is_regulation.append(
@@ -249,6 +267,7 @@ class LegislationIndex:
                 c.get("doc_type") == "agreement"
                 and not str(c["section"])[:1].isdigit())
             self._is_nif.append(c.get("status") == "not-in-force")
+            self._is_repealed.append(c.get("status") == "repealed")
 
     def _build_appendix_index(self):
         """Index directive appendices by (act_code, letter), so a directive
@@ -313,21 +332,32 @@ class LegislationIndex:
             print(f"CanLex index: reranker disabled ({type(exc).__name__}: {exc}); "
                   f"using hybrid fusion order.", file=sys.stderr)
 
-    def _bm25_scores(self, query):
+    def _bm25_scores(self, query, allowed=None):
+        """allowed: optional boolean list by chunk index. Filtered searches
+        pass it so recall competes only within scope -- filtering after
+        recall meant a doc_type='caselaw' search drew from whatever case law
+        happened to survive the open-corpus top-N competition."""
         scores = defaultdict(float)
         for term in set(tokenize(query)):
             idf = self.idf.get(term)
             if idf is None:
                 continue
             for idx, tf in self.postings[term]:
+                if allowed is not None and not allowed[idx]:
+                    continue
                 dl = self.doc_len[idx]
                 denom = tf + K1 * (1 - B + B * dl / self.avgdl)
                 scores[idx] += idf * tf * (K1 + 1) / denom
         return scores
 
-    def _semantic_ranking(self, query):
+    def _semantic_ranking(self, query, allowed=None):
         qv = self.embedder.encode_query(query)
         sims = self.vectors @ qv
+        if allowed is not None:
+            # Mask out-of-scope chunks so a filtered search ranks (and reports
+            # confidence) within its own scope, not corpus-wide.
+            sims = self._np.where(
+                self._np.asarray(allowed, dtype=bool), sims, -1.0)
         # Over-fetch 4x: the recall-stage source cap (_capped_top) drops the
         # surplus chunks of any one deep source, and the extra depth is what
         # lets other sources backfill the freed CANDIDATES slots.
@@ -523,19 +553,38 @@ class LegislationIndex:
         return {pos: (label, " ".join(snippet[:240].split()))
                 for pos, (score, label, snippet) in best.items()}
 
+    def _filter_ok(self, c, act, doc_type):
+        """One predicate for both recall masking and the late result filter.
+        The act filter matches short name, code, or full name -- an agent
+        passing 'Immigration and Refugee Protection Act' should not get
+        silence."""
+        if act:
+            a = act.lower()
+            if a not in (c["act_short"].lower(), c["act_code"].lower(),
+                         c.get("act_name", "").lower()):
+                return False
+        if doc_type and c.get("doc_type", "legislation") != doc_type:
+            return False
+        return True
+
     def search(self, query, top_k=6, act=None, doc_type=None):
         """Hybrid candidate fusion (BM25 + semantic), then cross-encoder rerank."""
         # Expand legal abbreviations (PRRA, H&C, ...) into statutory wording for
-        # the recall stages; the reranker still sees the user's original query.
+        # the recall stages; the reranker sees the original AND expanded forms.
         expanded = expand_query(query)
         confidence = None
         fused = defaultdict(float)
-        bm25 = self._bm25_scores(expanded)
+        # A filtered search competes only within its scope: the mask reaches
+        # both recall stages, so results come from the best of the whole
+        # filtered corpus rather than whatever survived open recall.
+        mask = ([self._filter_ok(c, act, doc_type) for c in self.chunks]
+                if (act or doc_type) else None)
+        bm25 = self._bm25_scores(expanded, allowed=mask)
         for rank, idx in enumerate(self._capped_top(
                 sorted(bm25, key=bm25.get, reverse=True))):
             fused[idx] += 1.0 / (RRF_K + rank)
         if self.semantic:
-            sem_order, confidence = self._semantic_ranking(expanded)
+            sem_order, confidence = self._semantic_ranking(expanded, allowed=mask)
             for rank, idx in enumerate(self._capped_top(sem_order)):
                 fused[idx] += W_SEM / (RRF_K + rank)
 
@@ -589,20 +638,15 @@ class LegislationIndex:
         for idx in list(fused):
             if self._is_nif[idx]:
                 fused[idx] -= NIF_PENALTY
+            elif self._is_repealed[idx]:
+                fused[idx] -= REPEALED_PENALTY
             elif self._is_regulation[idx]:
                 fused[idx] -= REG_PENALTY
             elif self._is_backmatter[idx]:
                 fused[idx] -= BACKMATTER_PENALTY
 
-        def allowed(idx):
-            c = self.chunks[idx]
-            if act and act.lower() not in (c["act_short"].lower(), c["act_code"].lower()):
-                return False
-            if doc_type and c.get("doc_type", "legislation") != doc_type:
-                return False
-            return True
-
-        candidates = [i for i in sorted(fused, key=fused.get, reverse=True) if allowed(i)]
+        candidates = [i for i in sorted(fused, key=fused.get, reverse=True)
+                      if self._filter_ok(self.chunks[i], act, doc_type)]
         if not candidates:
             return []
         scores = {i: fused[i] for i in candidates}
@@ -615,8 +659,17 @@ class LegislationIndex:
         # negative), so its power to demote a candidate is deliberately removed.
         if self.reranker:
             pool = candidates[:RERANK_POOL]
-            ce = dict(zip(pool, self.reranker.score(
-                query, [self._rerank_doc(i) for i in pool])))
+            docs = [self._rerank_doc(i) for i in pool]
+            ce_orig = self.reranker.score(query, docs)
+            # The cross-encoder has no reason to know Canadian legal
+            # shorthand ('PRRA' vs 'application for protection'), so when
+            # expansion changed the query, score both forms and keep each
+            # candidate's better score -- precision of the original query
+            # preserved, abbreviation blindness closed.
+            if expanded != query:
+                ce_exp = self.reranker.score(expanded, docs)
+                ce_orig = [max(a, b) for a, b in zip(ce_orig, ce_exp)]
+            ce = dict(zip(pool, ce_orig))
             fusion_rank = {idx: r for r, idx in enumerate(pool)}
             rerank_rank = {idx: r for r, idx in enumerate(
                 sorted(pool, key=ce.get, reverse=True))}
@@ -659,7 +712,9 @@ class LegislationIndex:
     def get_section(self, act, section):
         act = act.lower()
         for c in self.chunks:
-            if c["section"] == section and act in (c["act_short"].lower(), c["act_code"].lower()):
+            if c["section"] == section and act in (
+                    c["act_short"].lower(), c["act_code"].lower(),
+                    c.get("act_name", "").lower()):
                 return c
         return None
 

canlex/refresh.py

@@ -65,8 +65,86 @@ _NON_XML = [
     ("benefits", "benefits.json", "py -m canlex.benefits", "Benefit-plan booklets"),
     ("tariff", "tariff_schedule.json", "py -m canlex.tariff_schedule",
      "Customs Tariff Schedule ch. 98-99"),
+    ("enf", "enf.json", "py -m canlex.enf", "ENF operational manuals"),
+    ("amps", "amps.json", "py -m canlex.amps", "AMPS Master Penalty Document"),
+    ("charter", "charter.json", "py -m canlex.charter", "Charter / Constitution"),
+    ("commentary", "commentary.json", "py -m canlex.commentary",
+     "Curated commentary chunks"),
 ]
 
+# Curated datasets age: state legislatures amend deferred-adjudication
+# statutes every session, so a reviewed date older than this many days earns
+# a warning even though nothing upstream can be diffed automatically.
+_CURATED_MAX_AGE_DAYS = 180
+
+
+def check_curated(today):
+    """Age-check the curated datasets and detect edit-without-regen drift.
+
+    Two signals: (1) each data/curated/*.json 'reviewed' date older than
+    _CURATED_MAX_AGE_DAYS -> aging (re-verify against the state sources);
+    (2) the newest commentary.json current_to differing from the newest
+    curated 'reviewed' -> the curated file was edited without rerunning
+    canlex.commentary + canlex.embed, so production serves stale chunks."""
+    from .config import DATA_DIR
+    rows = []
+    newest_reviewed = ""
+    for path in sorted((DATA_DIR / "curated").glob("*.json")):
+        try:
+            reviewed = json.loads(
+                path.read_text(encoding="utf-8")).get("reviewed", "")
+        except Exception as exc:
+            rows.append({"file": path.name, "reviewed": "",
+                         "status": "error", "error": f"{type(exc).__name__}: {exc}"})
+            continue
+        newest_reviewed = max(newest_reviewed, reviewed or "")
+        status = "ok"
+        if reviewed:
+            age = (datetime.date.fromisoformat(today)
+                   - datetime.date.fromisoformat(reviewed)).days
+            if age > _CURATED_MAX_AGE_DAYS:
+                status = "aging"
+        rows.append({"file": path.name, "reviewed": reviewed,
+                     "status": status, "error": ""})
+    commentary = PROCESSED_DIR / "commentary.json"
+    if newest_reviewed and commentary.exists():
+        chunks = json.loads(commentary.read_text(encoding="utf-8"))
+        newest_chunk = max((c.get("current_to", "") for c in chunks), default="")
+        if newest_chunk != newest_reviewed:
+            rows.append({"file": "commentary.json",
+                         "reviewed": newest_chunk, "status": "drift",
+                         "error": f"curated reviewed {newest_reviewed} but "
+                                  f"chunks say {newest_chunk or '?'} -- rerun "
+                                  f"canlex.commentary && canlex.embed"})
+    return rows
+
+
+_DMEMO_INDEX = "https://www.cbsa-asfc.gc.ca/publications/dm-md/d1-d23-eng.html"
+
+
+def check_dmemo_index():
+    """Newest 'Date modified' on the CBSA D-memo index vs stored currency.
+    A newer date upstream means at least one memo changed since ingestion --
+    cheap (one fetch), coarse (does not name the memo)."""
+    stored = ""
+    path = PROCESSED_DIR / "dmemos.json"
+    if path.exists():
+        chunks = json.loads(path.read_text(encoding="utf-8"))
+        stored = max((c.get("current_to", "") for c in chunks), default="")
+    try:
+        req = urllib.request.Request(
+            _DMEMO_INDEX, headers={"User-Agent": BROWSER_UA})
+        with urllib.request.urlopen(req, timeout=60) as resp:
+            html = resp.read().decode("utf-8", "replace")
+        dates = re.findall(r"\b(20\d{2}-\d{2}-\d{2})\b", html)
+        remote = max(dates) if dates else ""
+    except Exception as exc:
+        return {"stored": stored, "remote": "", "status": "error",
+                "error": f"{type(exc).__name__}: {exc}"}
+    status = "stale" if (remote and stored and remote > stored) else \
+        ("ok" if remote else "no-date")
+    return {"stored": stored, "remote": remote, "status": status, "error": ""}
+
 # NJC directive dates as published: the index prints prose ('June  1, 1993' --
 # double spaces occur in the raw markup) while chunks may store either that
 # prose or the ISO fallback directive.py derives from page text, so both sides
@@ -362,18 +440,26 @@ def run(as_json=False):
     deleg = check_delegation()
     dirs = check_directives()
     nonxml = non_xml_currency()
+    curated = check_curated(today)
+    dmemo_idx = check_dmemo_index()
     pending = pending_in_force(today)
     stale = [r for r in leg if r["status"] in ("stale", "missing-local")]
     errors = [r for r in leg if r["status"] == "error"]
     deleg_stale = [r for r in deleg if r["status"] == "stale"]
     dirs_stale = [r for r in dirs if r["status"] == "stale"]
+    curated_flag = [r for r in curated if r["status"] in ("aging", "drift")]
+    dmemo_stale = dmemo_idx["status"] == "stale"
 
     if as_json:
         print(json.dumps({"checked": today, "legislation": leg,
                           "delegation": deleg, "directives": dirs,
-                          "non_xml": nonxml, "pending_in_force": pending,
+                          "non_xml": nonxml, "curated": curated,
+                          "dmemo_index": dmemo_idx,
+                          "pending_in_force": pending,
                           "stale_count": (len(stale) + len(deleg_stale)
-                                          + len(dirs_stale)),
+                                          + len(dirs_stale)
+                                          + len(curated_flag)
+                                          + (1 if dmemo_stale else 0)),
                           "error_count": len(errors)}, indent=2))
     else:
         print(f"CanLex corpus staleness check — {today}\n")
@@ -441,6 +527,29 @@ def run(as_json=False):
             print(f" {mark}{r['label']:14} {str(r['chunks']):>5} chunks  "
                   f"newest {r['newest'] or 'n/a':12}  {r['cmd']}")
 
+        print("\nCurated datasets (age + regen-drift check):")
+        for r in curated:
+            if r["status"] == "aging":
+                print(f"  AGING  {r['file']}: reviewed {r['reviewed']} — older "
+                      f"than {_CURATED_MAX_AGE_DAYS} days; re-verify the state "
+                      f"rows against their statutes and bump 'reviewed'")
+            elif r["status"] == "drift":
+                print(f"  DRIFT  {r['file']}: {r['error']}")
+            elif r["status"] == "error":
+                print(f"  ERROR  {r['file']}: {r['error'][:60]}")
+            else:
+                print(f"  ok     {r['file']}: reviewed {r['reviewed'] or '?'}")
+
+        if dmemo_idx["status"] == "stale":
+            print(f"\nD-memo index: STALE — newest upstream 'Date modified' "
+                  f"{dmemo_idx['remote']} > stored {dmemo_idx['stored']}; "
+                  f"run py -m canlex.dmemo --force && py -m canlex.embed")
+        elif dmemo_idx["status"] == "error":
+            print(f"\nD-memo index: ERROR {dmemo_idx['error'][:60]}")
+        else:
+            print(f"\nD-memo index: ok (upstream {dmemo_idx['remote'] or '?'} "
+                  f"<= stored {dmemo_idx['stored'] or '?'})")
+
         if pending:
             print("\nPending bills now in force (re-ingest the affected Acts):")
             for b in pending:
@@ -449,7 +558,8 @@ def run(as_json=False):
                 print(f"     {b['note']}")
                 print(f"     run: py -m canlex.ingest --force {' '.join(b['affects'])}")
 
-    return 1 if (stale or errors or deleg_stale or dirs_stale) else 0
+    return 1 if (stale or errors or deleg_stale or dirs_stale
+                 or curated_flag or dmemo_stale) else 0
 
 
 def main():

canlex/sweep.py

@@ -41,6 +41,7 @@ _METRIC_RE = re.compile(
 def _run_eval(env_overrides: dict[str, float]) -> dict[str, float]:
     """Run canlex.eval once with the given env overrides; return metrics dict."""
     env = dict(os.environ)
+    env["CANLEX_EVAL_SWEEP"] = "1"   # holdout questions stay sweep-blind
     for k, v in env_overrides.items():
         env[k] = f"{v}"
     proc = subprocess.run(

canlex/synonyms.py

@@ -44,6 +44,21 @@ _SYNONYMS = [
      "prescribed period five years completion of imposed sentence"),
     (r"misrep", "misrepresentation"),
     (r"ircc", "immigration refugees and citizenship canada"),
+    (r"eta", "electronic travel authorization"),
+    (r"trv", "temporary resident visa"),
+    (r"arc", "authorization to return to canada removal order"),
+    (r"danger opinion", "danger to the public refoulement protection"),
+    (r"ministerial relief", "national interest exception inadmissibility"),
+    (r"poe", "port of entry"),
+    (r"flagpol(?:e|ing)", "leaving canada and reentering port of entry"),
+    # The single most common US-screening term: bridge DUI shorthand to the
+    # Criminal Code s. 320.14 vocabulary the equivalency analysis turns on.
+    (r"(?:dwi|dui|drunk driving)",
+     "operation while impaired blood alcohol concentration conveyance"),
+    (r"personal exemption",
+     "returning persons exemption duty free allowance"),
+    (r"(?:nexus|trusted traveller)",
+     "presentation of persons alternative manner authorization"),
     # Border and customs
     (r"cbsa", "canada border services agency"),
     (r"bsos?", "border services officer"),
@@ -51,6 +66,37 @@ _SYNONYMS = [
     # Financial-crime and labour
     (r"fintrac", "financial transactions and reports analysis centre"),
     (r"njc", "national joint council"),
+    # French -> English bridges. The corpus is English-only, but officers work
+    # bilingually and drafting agents emit French terms; with the tokenizer's
+    # accent folding these triggers match either accented or folded forms.
+    # High-precision statutory anchors only.
+    (r"erar", "pre-removal risk assessment application for protection"),
+    (r"asfc", "canada border services agency"),
+    (r"interdiction de territoire", "inadmissibility inadmissible"),
+    (r"grande criminalit[ée]", "serious criminality"),
+    (r"criminalit[ée]", "criminality convicted offence"),
+    (r"renvoi", "removal order"),
+    (r"saisie?s?", "seizure seized goods"),
+    (r"r[ée]sidents? permanents?", "permanent resident"),
+    (r"[ée]trangers?", "foreign national"),
+    (r"d[ée]tention", "detention detained review"),
+    (r"r[ée]fugi[ée]s?", "refugee protection convention"),
+    (r"douanes?", "customs duties"),
+    (r"marchandises?", "goods imported"),
+    (r"d[ée]claration", "report declaration"),
+    (r"armes? [àa] feu", "firearm weapon"),
+    (r"contr[ôo]le", "examination search"),
+    (r"agents? des services frontaliers", "border services officer"),
+    (r"permis de travail", "work permit"),
+    (r"permis d'[ée]tudes", "study permit"),
+    (r"visa de r[ée]sident temporaire", "temporary resident visa"),
+    (r"r[ée]adaptation", "rehabilitation prescribed period"),
+    (r"expulsion", "deportation removal order"),
+    (r"passeurs?", "human smuggling organized entry"),
+    (r"fouille", "search of the person strip search"),
+    (r"griefs?", "grievance adjudication"),
+    (r"convention collective", "collective agreement"),
+    (r"heures suppl[ée]mentaires", "overtime hours of work"),
 ]
 
 _COMPILED = [(re.compile(rf"\b{trigger}\b", re.IGNORECASE), expansion)