Three new tools (whats_new/citing/toc), ENF 2 ATIP ingestion, PDI scraper, duplicate-id fix

canlex_whats_new (amendments since a date), canlex_citing (reverse xrefs),
canlex_toc (Act structure); webapp declares all ten. ENF 2 ingested from
its ATIP release with an UNOFFICIAL banner on every chunk (ENF 14 blocked:
image-scan PDF, needs OCR). pdi.py ingests the PR-card PDI section (ENF 27
successor) and the Tran page. uniquify_ids fixes 161 duplicate chunk ids
(enf TOC/body collisions + 6 pre-existing in delegation) that silently
orphaned 394 chunks from their embeddings.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

canlex/_common.py

@@ -92,3 +92,21 @@ def split_lines(text, target=1800, fold_stubs=False):
             pieces[1] = pieces[0] + "\n" + pieces[1]
             pieces.pop(0)
     return pieces
+
+
+def uniquify_ids(chunks):
+    """Rename duplicate chunk ids in place (second occurrence -> id-b, -c...).
+
+    Duplicate ids silently orphan chunks downstream: the embeddings loader
+    keys rows by id, so one twin gets vectors and the other degrades to
+    BM25-only with nothing but a stderr note. Generators whose source
+    numbering can repeat (a manual's TOC line and body heading both reading
+    '5.1'; delegation items re-using an instrument number) run their output
+    through this before writing."""
+    seen = {}
+    for c in chunks:
+        n = seen.get(c["id"], 0)
+        seen[c["id"]] = n + 1
+        if n:
+            c["id"] = f"{c['id']}-{chr(ord('a') + n)}"
+    return chunks

canlex/delegation.py

@@ -19,7 +19,7 @@ import re
 from bs4 import BeautifulSoup
 from pypdf import PdfReader
 
-from ._common import fetch_cached, norm_ws as _norm
+from ._common import fetch_cached, norm_ws as _norm, uniquify_ids
 from .config import PROCESSED_DIR, RAW_DIR
 
 RAW = RAW_DIR / "delegation"
@@ -465,6 +465,7 @@ def build():
         all_chunks.extend(chunks)
         print(f"  {len(chunks)} chunks")
     PROCESSED_DIR.mkdir(parents=True, exist_ok=True)
+    uniquify_ids(all_chunks)
     OUT.write_text(json.dumps(all_chunks, ensure_ascii=False, indent=1),
                    encoding="utf-8")
     print(f"\n{len(all_chunks)} delegation chunks from {len(SOURCES)} "

canlex/enf.py

@@ -18,7 +18,7 @@ the D-memoranda.
 import json
 import re
 
-from ._common import fetch_cached, norm_ws as _norm, split_lines
+from ._common import fetch_cached, norm_ws as _norm, split_lines, uniquify_ids
 from .config import PROCESSED_DIR, RAW_DIR
 
 BASE = ("https://www.canada.ca/content/dam/ircc/migration/ircc/english/"
@@ -37,6 +37,32 @@ _URL_OVERRIDES = {
          "resources/manuals/enf/enf20a-en.pdf"),
 }
 
+# Chapters IRCC withdrew from public distribution whose only public text is
+# an ATIP-released copy hosted by a third party (meurrensonimmigration.com,
+# verified live 2026-07-22). Ingested at the maintainer's decision
+# (2026-07-23) with an unmistakable UNOFFICIAL banner on every chunk: the
+# text may lag the internal current version, and nothing here is an official
+# publication. ENF 14/OP 19 is the operational manual behind criminal
+# rehabilitation -- directly relevant to canlex_rehabilitation.
+UNOFFICIAL = {
+    2: {"url": "https://meurrensonimmigration.com/wp-content/uploads/2023/06/"
+               "ENF2-3.pdf",
+        "title": "Evaluating Inadmissibility (unofficial ATIP copy)",
+        "note": "ATIP-released copy dated 2021-08-25; retired from canada.ca "
+                "in 2018 with no public successor"},
+    # ENF 14/OP 19 (Criminal Rehabilitation) is NOT here despite the
+    # maintainer authorizing it: the ATIP release at meurrensonimmigration
+    # .com/wp-content/uploads/2025/03/Enforcement-Manual-14-...pdf is a
+    # 56-page image scan with no text layer -- pypdf extracts zero
+    # characters. Ingesting it needs an OCR pass (tesseract or the Windows
+    # OCR API); parked until that toolchain exists.
+}
+_UNOFFICIAL_BANNER = (
+    "[UNOFFICIAL ATIP COPY — this chapter was withdrawn from public "
+    "distribution; this text comes from an access-to-information release "
+    "hosted by a third party and may lag the current internal version. "
+    "Treat as historical guidance; verify against internal sources.] ")
+
 # Curated titles for the core chapters; anything else gets a parsed or
 # generic title. Only used for display -- retrieval works off the text.
 TITLES = {
@@ -141,10 +167,14 @@ def _norm_block(lines):
 def build(chapters=_CHAPTERS):
     all_chunks, skipped = [], []
     for n in chapters:
-        url = _URL_OVERRIDES.get(n, BASE.format(n=n))
+        unofficial = UNOFFICIAL.get(n)
+        url = (unofficial["url"] if unofficial
+               else _URL_OVERRIDES.get(n, BASE.format(n=n)))
         dest = RAW / f"enf{n:02d}.pdf"
         try:
-            fetch_cached(url, dest, powershell=True, pause=1.0)
+            # The ATIP host is an ordinary WordPress site; canada.ca needs
+            # the PowerShell TLS workaround.
+            fetch_cached(url, dest, powershell="canada.ca" in url, pause=1.0)
         except Exception:
             skipped.append(n)
             if dest.exists():
@@ -155,8 +185,16 @@ def build(chapters=_CHAPTERS):
         except Exception as exc:
             print(f"  !! ENF {n}: parse failed: {type(exc).__name__}: {exc}")
             continue
-        print(f"  ENF {n:2d}: {len(chunks)} chunks")
+        if unofficial:
+            for c in chunks:
+                c["act_name"] = f"ENF {n} — {unofficial['title']}"
+                c["text"] = _UNOFFICIAL_BANNER + c["text"]
+                c["citation"] += " (unofficial ATIP copy)"
+                c["history"] = unofficial["note"]
+        print(f"  ENF {n:2d}: {len(chunks)} chunks"
+              + (" [UNOFFICIAL ATIP]" if unofficial else ""))
         all_chunks.extend(chunks)
+    uniquify_ids(all_chunks)
     out = PROCESSED_DIR / "enf.json"
     out.write_text(json.dumps(all_chunks, ensure_ascii=False, indent=2),
                    encoding="utf-8")

canlex/irb_guidelines.py

@@ -17,7 +17,7 @@ import re
 
 from bs4 import BeautifulSoup
 
-from ._common import fetch_cached, norm_ws as _norm, split_lines
+from ._common import fetch_cached, norm_ws as _norm, split_lines, uniquify_ids
 from .config import PROCESSED_DIR, RAW_DIR
 
 RAW = RAW_DIR / "irb_guidelines"
@@ -116,6 +116,7 @@ def build():
                 n += 1
         print(f"  Guideline {g['num']} ({g['title'][:40]}): {n} chunks, "
               f"effective {effective or '?'}")
+    uniquify_ids(chunks)
     out = PROCESSED_DIR / "irb_guidelines.json"
     out.write_text(json.dumps(chunks, ensure_ascii=False, indent=2),
                    encoding="utf-8")

canlex/pdi.py

@@ -0,0 +1,141 @@
+"""Ingest scoped IRCC Program Delivery Instructions (PDI) pages.
+
+The PDIs are the current-policy successor to the legacy chapter manuals.
+This module ingests the specific PDI sets that replace retired ENF chapters
+the corpus needs: the PR-card/status section (successor to ENF 27) and the
+Tran serious-criminality page (partial successor to ENF 2). Each named
+index page is fetched, its same-section sub-pages discovered, and every
+page chunked by heading -- doc_type='memorandum', like all guidance.
+
+    python -m canlex.pdi
+"""
+import json
+import re
+
+from ._common import fetch_cached, norm_ws as _norm, split_lines, uniquify_ids
+from .config import PROCESSED_DIR, RAW_DIR
+
+RAW = RAW_DIR / "pdi"
+_OBM = ("https://www.canada.ca/en/immigration-refugees-citizenship/corporate/"
+        "publications-manuals/operational-bulletins-manuals/")
+
+SETS = [
+    {"code": "pdi-prcard",
+     "short": "PDI PR Card",
+     "name": "PDI — Permanent resident card and PR status (successor to ENF 27)",
+     "index": _OBM + "permanent-residence/card.html",
+     # sub-pages live under these path fragments relative to the index
+     "scope": "/permanent-residence/card"},
+    {"code": "pdi-tran",
+     "short": "PDI Tran",
+     "name": "PDI — Assessing inadmissibility for serious criminality "
+             "following Tran v. Canada",
+     "index": _OBM + "standard-requirements/tran.html",
+     "scope": None},          # single page
+]
+
+_MODIFIED = re.compile(r'<time[^>]*>(\d{4}-\d{2}-\d{2})</time>')
+_TITLE = re.compile(r"<h1[^>]*>(.*?)</h1>", re.S)
+_LINK = re.compile(r'href="([^"#?]*?\.html)"')
+
+
+def _strip_tags(html):
+    html = re.sub(r"</t[dh]>", " | ", html)
+    html = re.sub(r"</(?:p|li|tr|h\d|caption)>", "\n", html)
+    html = re.sub(r"<[^>]+>", " ", html)
+    return [l for l in (_norm(x).strip("| ").strip()
+                        for x in html.split("\n")) if l]
+
+
+def _main(html):
+    m = re.search(r"<main[^>]*>(.*?)</main>", html, re.S)
+    return m.group(1) if m else html
+
+
+def _page_chunks(src, page_url, html, page_no):
+    content = _main(html)
+    title_m = _TITLE.search(content)
+    title = (_norm(re.sub(r"<[^>]+>", " ", title_m.group(1)))
+             if title_m else src["short"])
+    mod = _MODIFIED.search(html)
+    modified = mod.group(1) if mod else ""
+    # Split on h2 headings; each becomes a section-like unit.
+    parts = re.split(r"<h2[^>]*>", content[title_m.end():] if title_m else content)
+    chunks = []
+    for i, part in enumerate(parts):
+        if i == 0:
+            heading, body_html = title, part
+        else:
+            head_end = part.find("</h2>")
+            heading = _norm(re.sub(r"<[^>]+>", " ", part[:head_end]))
+            body_html = part[head_end:]
+        body = "\n".join(_strip_tags(body_html))
+        if len(body) < 120:
+            continue
+        pieces = split_lines(body, 1800) if len(body) > 2400 else [body]
+        total = len(pieces)
+        for k, piece in enumerate(pieces, start=1):
+            suffix = "" if total == 1 else f"-p{k}"
+            note = "" if total == 1 else f" (part {k} of {total})"
+            chunks.append({
+                "id": f"{src['code']}-{page_no}-{i}{suffix}",
+                "doc_type": "memorandum",
+                "act_code": src["code"].upper(),
+                "act_short": src["short"],
+                "act_name": src["name"],
+                "section": f"{title[:60]} — {heading[:60]}" if i else title[:80],
+                "marginal_note": heading,
+                "part": src["name"],
+                "division": "",
+                "heading": "",
+                "text": piece,
+                "history": "",
+                "last_amended": modified,
+                "current_to": modified,
+                "citation": f"{src['short']}: {heading}{note}",
+                "source_url": page_url,
+            })
+    return chunks
+
+
+def build():
+    all_chunks = []
+    for src in SETS:
+        index_html = fetch_cached(
+            src["index"], RAW / f"{src['code']}-index.html",
+            powershell=True, pause=0.8).decode("utf-8", "replace")
+        pages = [(src["index"], index_html)]
+        if src["scope"]:
+            seen = {src["index"]}
+            for href in _LINK.findall(_main(index_html)):
+                if src["scope"] not in href:
+                    continue
+                url = href if href.startswith("http") else \
+                    "https://www.canada.ca" + href
+                if url in seen:
+                    continue
+                seen.add(url)
+                name = url.rsplit("/", 1)[1].replace(".html", "")
+                try:
+                    page = fetch_cached(
+                        url, RAW / f"{src['code']}-{name}.html",
+                        powershell=True, pause=0.8).decode("utf-8", "replace")
+                    pages.append((url, page))
+                except Exception as exc:
+                    print(f"  !! {name}: {type(exc).__name__}: {exc}")
+        n = 0
+        for page_no, (url, html) in enumerate(pages):
+            chunks = _page_chunks(src, url, html, page_no)
+            all_chunks.extend(chunks)
+            n += len(chunks)
+        print(f"  {src['short']}: {len(pages)} pages, {n} chunks")
+    uniquify_ids(all_chunks)
+    out = PROCESSED_DIR / "pdi.json"
+    out.write_text(json.dumps(all_chunks, ensure_ascii=False, indent=2),
+                   encoding="utf-8")
+    print(f"pdi: {len(all_chunks)} chunks -> {out.name}")
+    return all_chunks
+
+
+if __name__ == "__main__":
+    build()

canlex/server.py

@@ -718,7 +718,15 @@ def canlex_list_acts() -> str:
     memo_numbers: set[str] = set()
     memo_chunks = 0
     memo_date = ""
+    fr_chunks = 0
+    fr_acts: set[str] = set()
     for c in index.chunks:
+        if c.get("lang", "en") == "fr":
+            # French twins share their English instrument's identity; count
+            # them once as coverage rather than doubling every act's rows.
+            fr_chunks += 1
+            fr_acts.add(c["act_code"])
+            continue
         doc_type = c.get("doc_type", "legislation")
         if doc_type == "memorandum":
             memo_numbers.add(c["section"])
@@ -765,6 +773,10 @@ def canlex_list_acts() -> str:
             })
             entry["count"] += 1
     lines = ["# CanLex corpus", "", "## Enacted law"]
+    if fr_chunks:
+        lines.insert(1, f"_Bilingual: {len(fr_acts)} instruments also carry "
+                        f"their parallel French text ({fr_chunks} articles); "
+                        f"a French-language query searches both._")
     for a in sorted(acts.values(), key=lambda x: x["short"]):
         lines.append(f"- **{a['short']}** — {a['name']} ({a['code']}): "
                      f"{a['count']} sections, current to {a['current_to'] or 'n/a'}")
@@ -814,6 +826,171 @@ def canlex_list_acts() -> str:
     return "\n".join(lines)
 
 
+@mcp.tool(name="canlex_whats_new",
+          annotations={"title": "What Changed Recently", **_READONLY})
+def canlex_whats_new(
+    since: Annotated[str, Field(
+        description="ISO date (YYYY-MM-DD): report amendments and additions "
+        "on or after this date.", pattern=r"^\d{4}-\d{2}-\d{2}$")],
+) -> str:
+    """List what has changed in the corpus's law since a date: provisions
+    amended or newly in force on or after it, plus each guidance source's
+    newest modification date.
+
+    Use for questions like "has IRPA changed since March?" or "what did the
+    last refresh pick up?". Changes reflect the ingested consolidations --
+    an amendment newer than the corpus's 'current to' dates is invisible
+    here; say so when the date asked about is very recent.
+
+    Returns:
+        str: Markdown -- amended/new provisions grouped by Act, then
+        guidance sources' newest dates.
+    """
+    index = _index()
+    amended: dict[str, list] = {}
+    guidance_newest: dict[str, str] = {}
+    for c in index.chunks:
+        if c.get("lang", "en") == "fr":
+            continue
+        if c.get("doc_type", "legislation") == "legislation":
+            stamp = max(c.get("last_amended") or "", c.get("in_force") or "")
+            if stamp >= since and c.get("section"):
+                amended.setdefault(c["act_short"], []).append(
+                    (c["section"], c.get("marginal_note", ""), stamp))
+        else:
+            key = c.get("act_short") or c.get("doc_type")
+            cur = c.get("current_to") or ""
+            if cur > guidance_newest.get(key, ""):
+                guidance_newest[key] = cur
+    lines = [f"# Changes since {since}", ""]
+    if amended:
+        lines.append("## Provisions amended or newly in force")
+        for act in sorted(amended):
+            secs = sorted({(s, n, d) for s, n, d in amended[act]},
+                          key=lambda t: t[2], reverse=True)
+            lines.append(f"**{act}** ({len(secs)} provisions):")
+            for s, n, d in secs[:25]:
+                lines.append(f"- s. {s} ({n}) — {d}")
+            if len(secs) > 25:
+                lines.append(f"- ...and {len(secs) - 25} more.")
+    else:
+        lines.append("No ingested provision shows an amendment or "
+                     "coming-into-force date on or after that date.")
+    recent_guidance = {k: v for k, v in guidance_newest.items() if v >= since}
+    if recent_guidance:
+        lines += ["", "## Guidance sources modified since then"]
+        for k, v in sorted(recent_guidance.items(), key=lambda kv: -ord(kv[1][0])
+                           if kv[1] else 0):
+            lines.append(f"- {k}: {v}")
+    lines += ["", "_Reflects the ingested consolidations only; verify "
+              "anything newer than the corpus's 'current to' dates upstream._"]
+    return "\n".join(lines)
+
+
+@mcp.tool(name="canlex_citing",
+          annotations={"title": "What Cites This Provision", **_READONLY})
+def canlex_citing(
+    act: Annotated[str, Field(
+        description="Act short name or code, e.g. 'IRPA'.",
+        min_length=1, max_length=60)],
+    section: Annotated[str, Field(
+        description="Section number, e.g. '34' (prefixes accepted).",
+        min_length=1, max_length=40)],
+) -> str:
+    """Reverse cross-reference lookup: everything in the corpus that cites a
+    given provision -- related provisions in the same Act, regulations made
+    under it, and the CBSA D-memoranda/guidance citing the section --
+    without fetching the section's own text first.
+
+    Returns:
+        str: Markdown lists of citing/related material, or a not-found
+        message.
+    """
+    index = _index()
+    result = index.get_section(act, section)
+    if result is None and section != _normalize_section(section):
+        result = index.get_section(act, _normalize_section(section))
+    if result is None:
+        return (f"No section '{section}' in '{act}'. Use canlex_list_acts "
+                f"for the loaded instruments.")
+    related = index.related(result) or {}
+    lines = [f"# Material citing {result['citation']}", ""]
+    memos = related.get("memoranda")
+    if memos:
+        lines.append("**Guidance citing this section** (persuasive, not "
+                     "binding): " + ", ".join(memos))
+    provisions = related.get("provisions")
+    if provisions:
+        lines.append("**Related provisions in this Act:** "
+                     + "; ".join(f"s. {s} ({n})" if n else f"s. {s}"
+                                 for s, n in provisions))
+    regs = related.get("regulations")
+    if regs:
+        lines.append("**Regulations made under this Act:** "
+                     + "; ".join(f"{n} ({s})" for s, n in regs))
+    enabling = related.get("enabling_act")
+    if enabling:
+        lines.append(f"**Made under:** {enabling[1]} ({enabling[0]})")
+    if len(lines) == 2:
+        lines.append("Nothing in the corpus cross-references this section.")
+    lines += ["", "For case law interpreting it, search with "
+              "canlex_search_legislation (doc_type 'caselaw')."]
+    return "\n".join(lines)
+
+
+@mcp.tool(name="canlex_toc",
+          annotations={"title": "Act Table of Contents", **_READONLY})
+def canlex_toc(
+    act: Annotated[str, Field(
+        description="Act short name or code, e.g. 'IRPA' or 'Customs Act'.",
+        min_length=1, max_length=60)],
+) -> str:
+    """Browse an Act's structure: its Parts/Divisions and the sections under
+    each, with their marginal notes. Use when you know the Act but not the
+    section number, or to survey what an Act covers.
+
+    Returns:
+        str: Markdown outline. Very large Acts (e.g. the Criminal Code)
+        return Parts with section ranges rather than every section.
+    """
+    index = _index()
+    a = act.strip().lower()
+    secs = [c for c in index.chunks
+            if c.get("doc_type", "legislation") == "legislation"
+            and c.get("lang", "en") == "en"
+            and a in (c["act_short"].lower(), c["act_code"].lower(),
+                      c.get("act_name", "").lower())
+            and c.get("section") and not c["id"].endswith(tuple(
+                f"-p{k}" for k in range(2, 40)))]
+    if not secs:
+        known = sorted({c["act_short"] for c in index.chunks
+                        if c.get("doc_type", "legislation") == "legislation"})
+        return (f"No legislation matches '{act}'. Loaded: "
+                + ", ".join(known))
+    by_part: dict[str, list] = {}
+    seen = set()
+    for c in secs:
+        if c["section"] in seen:
+            continue
+        seen.add(c["section"])
+        by_part.setdefault(c.get("part") or "(no Part)", []).append(
+            (c["section"], c.get("marginal_note", "")))
+    name = secs[0]["act_name"]
+    total = sum(len(v) for v in by_part.values())
+    lines = [f"# {name} — contents ({total} sections)", ""]
+    detailed = total <= 150
+    for part, entries in by_part.items():
+        lines.append(f"## {part}" if part != "(no Part)" else "## —")
+        if detailed:
+            for s, n in entries:
+                lines.append(f"- s. {s}" + (f" — {n}" if n else ""))
+        else:
+            first, last = entries[0][0], entries[-1][0]
+            lines.append(f"- ss. {first}–{last} ({len(entries)} sections)")
+    lines += ["", "Fetch any section with canlex_get_section."]
+    return "\n".join(lines)
+
+
 _CITATOR = None
 
 

webapp/app.py

@@ -161,6 +161,58 @@ TOOL_DECLARATIONS = [
             "required": ["act", "section"],
         },
     },
+    {
+        "name": "canlex_whats_new",
+        "description": (
+            "List provisions amended or newly in force since a date, plus "
+            "guidance sources modified since then. Use for 'has X changed "
+            "since ...?' questions."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "since": {
+                    "type": "string",
+                    "description": "ISO date YYYY-MM-DD.",
+                },
+            },
+            "required": ["since"],
+        },
+    },
+    {
+        "name": "canlex_citing",
+        "description": (
+            "Reverse cross-reference lookup: the guidance, related "
+            "provisions and regulations that cite a given section, without "
+            "fetching its text."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "act": {"type": "string",
+                        "description": "Act short name or code."},
+                "section": {"type": "string",
+                            "description": "Section number, e.g. '34'."},
+            },
+            "required": ["act", "section"],
+        },
+    },
+    {
+        "name": "canlex_toc",
+        "description": (
+            "Browse an Act's table of contents (Parts and sections with "
+            "marginal notes). Use when the Act is known but the section "
+            "number is not."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "act": {"type": "string",
+                        "description": "Act short name or code."},
+            },
+            "required": ["act"],
+        },
+    },
     {
         "name": "canlex_list_acts",
         "description": (
@@ -546,6 +598,13 @@ def _summarize_call(name: str, args: dict) -> str:
         return f"Looking up case {args.get('case_url', '?')}"
     if name == "canlex_list_acts":
         return "Listing the CanLex corpus"
+    if name == "canlex_whats_new":
+        return f"Checking changes since {args.get('since', '?')}"
+    if name == "canlex_citing":
+        return (f"Finding material citing {args.get('act', '?')} "
+                f"s. {args.get('section', '?')}")
+    if name == "canlex_toc":
+        return f"Browsing the contents of {args.get('act', '?')}"
     if name == "canlex_us_disposition":
         state = f" ({args['state']})" if args.get("state") else ""
         return (f"Checking US disposition "