Parse statutory Schedules; add 10 instruments incl. firearms classification and IAD Rules 2022

The XML parser silently dropped every Schedule except NifProvs, so the CDSA
drug schedules, the AMPS designated-provisions table and IRPA's Refugee
Convention were absent. Schedule-only instruments (Import Control List) now
parse too; the Customs Tariff opts out (tariff_schedule.py serves its
nomenclature). New sources: SOR/98-462, Excise Act 2001, ICL, ECL,
Presentation of Persons Regs, Duty Free Shop Regs, Sufferance Warehouses
Regs, CHRA, GECA, IAD Rules 2022 (the old config comment wrongly treated the
IAD rules as repealed-and-gone).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

canlex/config.py

@@ -27,8 +27,14 @@ SOURCES = {
         "short": "Citizenship Act", "name": "Citizenship Act", "kind": "act",
         "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/C-29.xml",
         "web_base": "https://laws-lois.justice.gc.ca/eng/acts/c-29/"},
-    # IRB tribunal procedural rules. (The Immigration Appeal Division Rules,
-    # SOR-2002-230, are intentionally absent: that instrument has been repealed.)
+    # IRB tribunal procedural rules. (SOR-2002-230 was repealed because the
+    # Immigration Appeal Division Rules, 2022 -- SOR-2022-277, below --
+    # replaced it in January 2023.)
+    "SOR-2022-277": {
+        "short": "IAD Rules", "name": "Immigration Appeal Division Rules, 2022",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-2022-277.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-2022-277/"},
     "SOR-2002-229": {
         "short": "Immigration Division Rules", "name": "Immigration Division Rules",
         "kind": "regulation",
@@ -50,10 +56,47 @@ SOURCES = {
         "short": "Customs Act", "name": "Customs Act", "kind": "act",
         "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/C-52.6.xml",
         "web_base": "https://laws-lois.justice.gc.ca/eng/acts/c-52.6/"},
+    # skip_schedules: the Customs Tariff's schedule is the entire tariff
+    # nomenclature -- tariff_schedule.py serves chapters 98/99 from the CBSA
+    # edition instead, so the XML schedule would only duplicate it at scale.
     "C-54.011": {
         "short": "Customs Tariff", "name": "Customs Tariff", "kind": "act",
+        "skip_schedules": True,
         "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/C-54.011.xml",
         "web_base": "https://laws-lois.justice.gc.ca/eng/acts/c-54.011/"},
+    # Traveller/trade instruments under the Customs Act and EIPA. The two
+    # control lists are almost entirely Schedules, so they depend on the
+    # schedule parser; Presentation of Persons is the legal basis for NEXUS
+    # and every other alternative-presentation program.
+    "E-14.1": {
+        "short": "Excise Act, 2001", "name": "Excise Act, 2001", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/E-14.1.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/e-14.1/"},
+    "CRC-604": {
+        "short": "Import Control List", "name": "Import Control List",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/C.R.C.,_c._604.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/C.R.C.,_c._604/"},
+    "SOR-89-202": {
+        "short": "Export Control List", "name": "Export Control List",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-89-202.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-89-202/"},
+    "SOR-2003-323": {
+        "short": "Presentation of Persons Regs",
+        "name": "Presentation of Persons (2003) Regulations", "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-2003-323.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-2003-323/"},
+    "SOR-86-1072": {
+        "short": "Duty Free Shop Regs", "name": "Duty Free Shop Regulations",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-86-1072.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-86-1072/"},
+    "SOR-86-1065": {
+        "short": "Sufferance Warehouses Regs",
+        "name": "Customs Sufferance Warehouses Regulations", "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-86-1065.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-86-1065/"},
     "S-8.35": {
         "short": "Luxury Tax Act", "name": "Select Luxury Items Tax Act", "kind": "act",
         "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/S-8.35.xml",
@@ -113,6 +156,18 @@ SOURCES = {
         "short": "Firearms Act", "name": "Firearms Act", "kind": "act",
         "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/F-11.6.xml",
         "web_base": "https://laws-lois.justice.gc.ca/eng/acts/f-11.6/"},
+    # Made under Criminal Code s. 117.15: the model-by-model prohibited/
+    # restricted/non-restricted classifications (incl. the May 2020
+    # SOR/2020-96 prohibitions). Nearly all substance is in its Schedules --
+    # this entry is why the schedule parser exists.
+    "SOR-98-462": {
+        "short": "Firearms Classification Regs",
+        "name": ("Regulations Prescribing Certain Firearms and Other Weapons, "
+                 "Components and Parts of Weapons, Accessories, Cartridge "
+                 "Magazines, Ammunition and Projectiles as Prohibited, "
+                 "Restricted or Non-Restricted"), "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-98-462.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-98-462/"},
     "P-24.501": {
         "short": "PCMLTFA",
         "name": "Proceeds of Crime (Money Laundering) and Terrorist Financing Act",
@@ -237,6 +292,18 @@ SOURCES = {
         "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-96-263/"},
 
     # --- Labour & public service ---
+    # CHRA grounds the accommodation/discrimination case law already in the
+    # corpus (Renaud, Hydro-Québec, Elk Valley); GECA governs injury-on-duty
+    # compensation interlocking with the FB agreement's IOD leave.
+    "H-6": {
+        "short": "CHRA", "name": "Canadian Human Rights Act", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/H-6.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/h-6/"},
+    "G-5": {
+        "short": "GECA", "name": "Government Employees Compensation Act",
+        "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/G-5.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/g-5/"},
     "L-2": {
         "short": "Canada Labour Code", "name": "Canada Labour Code", "kind": "act",
         "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/L-2.xml",

canlex/ingest.py

@@ -91,6 +91,115 @@ def _render_block(el, is_section=False):
     return head
 
 
+def _table_text(tablegroup):
+    """Render a CALS <TableGroup> as caption/title lines plus one
+    pipe-separated line per row. Column structure survives as ' | '
+    separators, which is enough for BM25/semantic matching and readable in a
+    result pane; no attempt is made to preserve spanned cells."""
+    out = []
+    for t in (tablegroup.find("Caption"), tablegroup.find("table/title")):
+        if t is not None:
+            txt = _itertext(t)
+            if txt:
+                out.append(txt)
+    for row in tablegroup.iter("row"):
+        cells = [_itertext(e) for e in row.findall("entry")]
+        line = " | ".join(cells).strip(" |")
+        if line:
+            out.append(line)
+    return "\n".join(out)
+
+
+def _sched_body(el):
+    """Render a <Schedule>'s content elements into readable text.
+
+    Observed shapes across the corpus: List/Item trees (CDSA I-IV, IRPR
+    1.1), DocumentInternal/Group/Provision prose (CDSA V-VI, IRPA's Refugee
+    Convention), and TableGroup grids (IRPR Schedule 2 violations, the AMPS
+    designated-provisions tables). Lists, groups and provisions reuse
+    _render_block, whose Label/Text recursion matches their structure;
+    tables get the dedicated renderer above so rows keep their shape."""
+    parts = []
+    for child in el:
+        tag = child.tag
+        if tag in ("ScheduleFormHeading", "HistoricalNote"):
+            continue
+        if tag == "TableGroup":
+            parts.append(_table_text(child))
+        elif tag == "GroupHeading":
+            parts.append(_heading_text(child))
+        elif tag in ("DocumentInternal", "Group", "RegulationPiece", "BillPiece"):
+            parts.append(_sched_body(child))
+        else:
+            rendered = _render_block(child)
+            if rendered:
+                parts.append(rendered)
+    return "\n".join(p for p in parts if p)
+
+
+# Schedule ids with dedicated handling: NifProvs is ingested separately as
+# not-in-force chunks; RelatedProvs (transitional provisions of past
+# amendments) is deliberately excluded -- see _nif_chunks.
+_SCHED_SKIP_IDS = {"NifProvs", "RelatedProvs"}
+
+
+def _schedule_chunks(root, code, src, current_to):
+    """Chunks for an Act's substantive Schedules.
+
+    Previously every Schedule except NifProvs was silently dropped -- which
+    meant the CDSA's controlled-substance lists, the AMPS designated-
+    provisions tables and IRPA's Refugee Convention articles were absent
+    from the corpus even though the sections that depend on them were
+    ingested. A source can opt out with 'skip_schedules': True (the Customs
+    Tariff does: its schedule is the full tariff nomenclature, served
+    separately by tariff_schedule.py from the CBSA edition)."""
+    if src.get("skip_schedules"):
+        return []
+    chunks = []
+    for n, sched in enumerate(root.findall("Schedule"), start=1):
+        if sched.get("id") in _SCHED_SKIP_IDS:
+            continue
+        h = sched.find("ScheduleFormHeading")
+        label = _norm(h.findtext("Label") or "") if h is not None else ""
+        title = _norm(h.findtext("TitleText") or "") if h is not None else ""
+        oref = _norm(h.findtext("OriginatingRef") or "") if h is not None else ""
+        label = label or (f"SCHEDULE ({n})" if not title else title)
+        repealed = sched.find("Repealed") is not None
+        body = "[Repealed]" if repealed else _sched_body(sched)
+        if not body:
+            continue
+        header = " — ".join(p for p in (title, oref) if p)
+        text = (header + "\n" if header else "") + body
+        base = {
+            "act_code": code,
+            "act_short": src["short"],
+            "act_name": src["name"],
+            "section": label.title(),
+            "marginal_note": title or label.title(),
+            "part": "SCHEDULES",
+            "division": "",
+            "heading": label.title(),
+            "history": "",
+            "last_amended": sched.get(f"{LIMS}lastAmendedDate", ""),
+            "in_force": sched.get(f"{LIMS}inforce-start-date", ""),
+            "status": "repealed" if repealed else "in force",
+            "current_to": current_to,
+            "citation": f"{src['short']}, {label.title()}",
+            "source_url": src["web_base"],
+        }
+        pieces = (_split_text(text)
+                  if len(text) > _SPLIT_THRESHOLD else [text])
+        total = len(pieces)
+        for k, piece in enumerate(pieces, start=1):
+            if total == 1:
+                chunks.append({**base, "id": f"{code}-sched{n}", "text": piece})
+            else:
+                chunks.append({
+                    **base, "id": f"{code}-sched{n}-p{k}", "text": piece,
+                    "citation": f"{base['citation']} (part {k} of {total})"})
+    return chunks
+
+
 def _history(section):
     note = section.find("HistoricalNote")
     if note is None:
@@ -107,7 +216,11 @@ def parse_legislation(xml_path, code):
     current_to = root.get(f"{LIMS}current-date", "")
     body = root.find("Body")
     if body is None:
-        raise ValueError(f"{code}: no <Body> element (root <{root.tag}>)")
+        # A schedule-only instrument (e.g. the Import Control List) has no
+        # <Body> at all -- its entire substance is in Schedules.
+        if not root.findall("Schedule"):
+            raise ValueError(f"{code}: no <Body> element (root <{root.tag}>)")
+        body = ()
 
     headings, chunks = {}, []
     for el in body:
@@ -204,6 +317,7 @@ def parse_legislation(xml_path, code):
                 chunks.append({**base, "id": f"{chunk_id}-p{k}", "text": piece,
                                "citation": f"{citation} (part {k} of {n})"})
 
+    chunks.extend(_schedule_chunks(root, code, src, current_to))
     chunks.extend(_nif_chunks(root, code, src, current_to))
     return chunks