Bilingual corpus (phase 3) + multi-vector embeddings + language-routed retrieval

Parallel French legislation for all 59 instruments (same-schema fra XML;
SOR->DORS URL mapping; lang='fr' chunks, art./annexe citations). English
queries never see French chunks; French queries (accent/stopword detection)
search both languages. Long chunks now embed as up to 8 windows scored by
best window -- D-memo tails reachable semantically. Reranker window 1000->
2000 (eval-held, Hit@5 +0.01). devises->espèces bridge.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

canlex/embed.py

@@ -39,8 +39,14 @@ def load_chunks():
     return chunks
 
 
+# A chunk longer than the embed window gets additional vectors for its tail,
+# up to this many per chunk (an 87K-char D-memo section would otherwise need
+# 40+; eight windows cover 16K chars, past which BM25 carries the tail).
+_MAX_WINDOWS = 8
+
+
 def embed_text(chunk):
-    """Compact, retrieval-focused representation of one section."""
+    """Compact, retrieval-focused representation of one section (head only)."""
     # The section title is the strongest topical signal, so it is repeated to
     # emphasise it. Title selection is doc_type-aware (see index.topical_title):
     # a D-memo's marginal_note is a generic banner so its actual subject in
@@ -53,6 +59,27 @@ def embed_text(chunk):
     return " . ".join(p for p in parts if p)
 
 
+def embed_texts(chunk):
+    """One representation per embedding window of a chunk.
+
+    The first is the classic head representation; a chunk whose body exceeds
+    the embed window additionally gets tail windows (same topical-title
+    anchor, successive body slices), so a fact buried mid-memo is reachable
+    through semantic recall instead of BM25 alone. All windows share the
+    chunk's id -- the index scores a chunk by its best window."""
+    from .index import topical_title
+    texts = [embed_text(chunk)]
+    body = chunk["text"]
+    if len(body) > _MAX_BODY:
+        note = topical_title(chunk)
+        anchor = " . ".join(p for p in (chunk["act_short"], note) if p)
+        for start in range(_MAX_BODY, len(body), _MAX_BODY):
+            if len(texts) >= _MAX_WINDOWS:
+                break
+            texts.append(f"{anchor} . {body[start:start + _MAX_BODY]}")
+    return texts
+
+
 class Embedder:
     """Local transformer sentence-embedder: bge-small-en-v1.5 as ONNX on CPU.
 
@@ -122,10 +149,15 @@ def build():
     if not chunks:
         print(f"No processed data in {PROCESSED_DIR}. Run 'canlex.ingest' first.")
         return
-    print(f"Embedding {len(chunks)} sections with {EMB_REPO} ...")
-    vectors = Embedder().encode([embed_text(c) for c in chunks])
-    ids = np.array([c["id"] for c in chunks])
-    np.savez(EMB_PATH, ids=ids, vectors=vectors)
+    texts, ids = [], []
+    for c in chunks:
+        for t in embed_texts(c):
+            texts.append(t)
+            ids.append(c["id"])
+    print(f"Embedding {len(chunks)} sections as {len(texts)} windows "
+          f"with {EMB_REPO} ...")
+    vectors = Embedder().encode(texts)
+    np.savez(EMB_PATH, ids=np.array(ids), vectors=vectors)
     print(f"  {vectors.shape[0]} vectors, dim {vectors.shape[1]} -> {EMB_PATH.name}")
 
 

canlex/index.py

@@ -135,6 +135,28 @@ def _stem(word):
     return stemmed
 
 
+# Query-language detection for the bilingual corpus: accented characters or
+# two-plus French function words mark a French query. French queries may draw
+# on BOTH languages (guidance and case law exist only in English); English
+# queries never surface French chunks (the English corpus is complete, and a
+# French twin would only duplicate results).
+_FR_HINT = re.compile(r"[àâçéèêëîïôùûüÿœæ]", re.IGNORECASE)
+# ASCII-only entries: tokens are matched pre-folding with [a-zA-Z']+, and any
+# accented query already short-circuits through _FR_HINT above.
+_FR_WORDS = frozenset(
+    "le la les de des une du et est dans pour que qui aux au sur par ne pas "
+    "peut doit selon lors quelle quel cette ces son ses leur d'un d'une "
+    "interdiction fouille saisie renvoi territoire".split())
+
+
+def query_lang(query):
+    """'fr' when the query reads as French, else 'en'."""
+    if _FR_HINT.search(query):
+        return "fr"
+    tokens = re.findall(r"[a-zA-Z']+", query.lower())
+    return "fr" if sum(1 for t in tokens if t in _FR_WORDS) >= 2 else "en"
+
+
 def _fold_accents(text):
     """ASCII-fold accented characters so a French or mixed query tokenizes
     usefully instead of shattering: the [a-z0-9] token pattern used to split
@@ -215,6 +237,9 @@ class LegislationIndex:
             raise RuntimeError(
                 f"No processed legislation in {PROCESSED_DIR}. Run 'canlex.ingest' first.")
         self._tri_index = None       # lazy; see _fuzzy_term
+        # Cached recall mask for the common case (English query, no
+        # filters): everything but the French twins.
+        self._en_mask = [c.get("lang", "en") != "fr" for c in self.chunks]
         self._build_bm25()
         self._build_note_tokens()
         self._build_xref()
@@ -299,20 +324,22 @@ class LegislationIndex:
         try:
             import numpy as np
             from .embed import Embedder
+            # Multi-vector layout: rows share a chunk id when the chunk was
+            # embedded as several windows (long D-memo sections etc.); a
+            # chunk's semantic score is the max over its windows. Rows whose
+            # id matches no loaded chunk are dropped; chunks with no rows
+            # count as missing (BM25-only for them).
+            chunk_ordinal = {c["id"]: i for i, c in enumerate(self.chunks)}
             with np.load(emb_path) as data:
-                id_to_vec = dict(zip(data["ids"].tolist(), data["vectors"]))
-                dim = int(data["vectors"].shape[1])
-            missing = 0
-            rows = []
-            for c in self.chunks:
-                vec = id_to_vec.get(c["id"])
-                if vec is None:
-                    missing += 1
-                    rows.append(np.zeros(dim, dtype=np.float32))
-                else:
-                    rows.append(vec)
+                all_ids = data["ids"].tolist()
+                all_vecs = data["vectors"]
+            keep = [(chunk_ordinal[i], r) for r, i in enumerate(all_ids)
+                    if i in chunk_ordinal]
             self._np = np
-            self.vectors = np.vstack(rows)
+            self.vectors = all_vecs[[r for _, r in keep]]
+            self._row_chunk = np.array([o for o, _ in keep], dtype=np.int64)
+            covered = set(self._row_chunk.tolist())
+            missing = len(self.chunks) - len(covered)
             self.embedder = Embedder()
             self.semantic = True
             if missing:
@@ -389,17 +416,21 @@ class LegislationIndex:
         return scores
 
     def _semantic_ranking(self, query, allowed=None):
+        np = self._np
         qv = self.embedder.encode_query(query)
-        sims = self.vectors @ qv
+        row_sims = self.vectors @ qv
+        # Aggregate windows to chunks by max: a long section scores as its
+        # best-matching window, so a fact buried mid-memo is reachable.
+        sims = np.full(len(self.chunks), -1.0, dtype=np.float32)
+        np.maximum.at(sims, self._row_chunk, row_sims)
         if allowed is not None:
             # Mask out-of-scope chunks so a filtered search ranks (and reports
             # confidence) within its own scope, not corpus-wide.
-            sims = self._np.where(
-                self._np.asarray(allowed, dtype=bool), sims, -1.0)
+            sims = np.where(np.asarray(allowed, dtype=bool), sims, -1.0)
         # Over-fetch 4x: the recall-stage source cap (_capped_top) drops the
         # surplus chunks of any one deep source, and the extra depth is what
         # lets other sources backfill the freed CANDIDATES slots.
-        order = self._np.argsort(sims)[::-1][:CANDIDATES * 4]
+        order = np.argsort(sims)[::-1][:CANDIDATES * 4]
         # The top cosine similarity doubles as a corpus-coverage signal: a query
         # the corpus cannot answer has no passage close to it.
         return [int(i) for i in order], float(sims.max())
@@ -591,11 +622,14 @@ class LegislationIndex:
         return {pos: (label, " ".join(snippet[:240].split()))
                 for pos, (score, label, snippet) in best.items()}
 
-    def _filter_ok(self, c, act, doc_type):
+    def _filter_ok(self, c, act, doc_type, qlang="en"):
         """One predicate for both recall masking and the late result filter.
         The act filter matches short name, code, or full name -- an agent
         passing 'Immigration and Refugee Protection Act' should not get
-        silence."""
+        silence. English queries never see French chunks; French queries see
+        both languages (guidance/case law exist only in English)."""
+        if qlang == "en" and c.get("lang", "en") == "fr":
+            return False
         if act:
             a = act.lower()
             if a not in (c["act_short"].lower(), c["act_code"].lower(),
@@ -610,13 +644,21 @@ class LegislationIndex:
         # Expand legal abbreviations (PRRA, H&C, ...) into statutory wording for
         # the recall stages; the reranker sees the original AND expanded forms.
         expanded = expand_query(query)
+        qlang = query_lang(query)
         confidence = None
         fused = defaultdict(float)
         # A filtered search competes only within its scope: the mask reaches
         # both recall stages, so results come from the best of the whole
-        # filtered corpus rather than whatever survived open recall.
-        mask = ([self._filter_ok(c, act, doc_type) for c in self.chunks]
-                if (act or doc_type) else None)
+        # filtered corpus rather than whatever survived open recall. An
+        # unfiltered English query uses the cached English-only mask; a
+        # French query searches both languages unmasked.
+        if act or doc_type:
+            mask = [self._filter_ok(c, act, doc_type, qlang)
+                    for c in self.chunks]
+        elif qlang == "en":
+            mask = self._en_mask
+        else:
+            mask = None
         bm25 = self._bm25_scores(expanded, allowed=mask)
         for rank, idx in enumerate(self._capped_top(
                 sorted(bm25, key=bm25.get, reverse=True))):
@@ -684,7 +726,7 @@ class LegislationIndex:
                 fused[idx] -= BACKMATTER_PENALTY
 
         candidates = [i for i in sorted(fused, key=fused.get, reverse=True)
-                      if self._filter_ok(self.chunks[i], act, doc_type)]
+                      if self._filter_ok(self.chunks[i], act, doc_type, qlang)]
         if not candidates:
             return []
         scores = {i: fused[i] for i in candidates}
@@ -748,14 +790,19 @@ class LegislationIndex:
             results.append(result)
         return results
 
-    def get_section(self, act, section):
+    def get_section(self, act, section, lang="en"):
+        """The section chunk in the requested language, falling back to the
+        other language rather than returning nothing."""
         act = act.lower()
+        fallback = None
         for c in self.chunks:
             if c["section"] == section and act in (
                     c["act_short"].lower(), c["act_code"].lower(),
                     c.get("act_name", "").lower()):
-                return c
-        return None
+                if c.get("lang", "en") == lang:
+                    return c
+                fallback = fallback or c
+        return fallback
 
     def _build_xref(self):
         """Index legislation by (act, section); find each Act's definitions
@@ -767,6 +814,8 @@ class LegislationIndex:
         for c in self.chunks:
             if c.get("doc_type", "legislation") != "legislation":
                 continue
+            if c.get("lang", "en") == "fr":
+                continue    # xref keys are per (act, section); English canonical
             self._by_section[(c["act_code"], c["section"])] = c
             if c["act_code"] not in self._defs_section and (
                     c["marginal_note"].strip().lower() in (

canlex/ingest.py

@@ -399,8 +399,58 @@ def ingest(code, force=False):
     return chunks
 
 
+# --- French corpus (phase 3 of bilingual support) ----------------------------
+# Justice Laws publishes every consolidation as parallel French XML in the
+# SAME schema (English element names, French text, identical section
+# numbering), so the whole parser -- schedules included -- works unchanged.
+# French chunks carry lang='fr', a '-fr-' id infix, French citation style
+# ('art.' / 'annexe') and the French web URL; retrieval filters by query
+# language (index.py), and get_section/xref/list_acts stay English-first.
+
+def _fr_xml_url(src):
+    return (src["xml_url"]
+            .replace("/eng/XML/", "/fra/XML/")
+            .replace("/XML/SOR-", "/XML/DORS-")  # règlements: SOR -> DORS
+            .replace("_c._", "_ch._"))        # C.R.C. chapters: c. -> ch.
+
+
+def _frenchify(chunks, code):
+    out = []
+    for c in chunks:
+        c = dict(c)
+        c["id"] = c["id"].replace(f"{code}-", f"{code}-fr-", 1)
+        c["lang"] = "fr"
+        c["citation"] = (c["citation"]
+                         .replace(", s. ", ", art. ")
+                         .replace(", Schedule", ", annexe"))
+        c["source_url"] = (c["source_url"]
+                           .replace("/eng/acts/", "/fra/lois/")
+                           .replace("/eng/regulations/", "/fra/reglements/"))
+        out.append(c)
+    return out
+
+
+def ingest_fr(code, force=False):
+    src = SOURCES[code]
+    dest = RAW_DIR / f"{code}-fr.xml"
+    if not dest.exists() or force:
+        url = _fr_xml_url(src)
+        print(f"  downloading {url}")
+        req = urllib.request.Request(url, headers={"User-Agent": "CanLex/0.1"})
+        with urllib.request.urlopen(req, timeout=120) as resp:
+            dest.write_bytes(resp.read())
+        time.sleep(1.0)
+    chunks = _frenchify(parse_legislation(dest, code), code)
+    out = PROCESSED_DIR / f"{code}-fr.json"
+    out.write_text(json.dumps(chunks, ensure_ascii=False, indent=2),
+                   encoding="utf-8")
+    print(f"  {len(chunks)} articles -> {out.name}")
+    return chunks
+
+
 def main():
     force = "--force" in sys.argv
+    french = "--fr" in sys.argv
     only = [a for a in sys.argv[1:] if not a.startswith("-")]
     codes = only or list(SOURCES)
     failures = []
@@ -409,7 +459,11 @@ def main():
             print(f"  SKIP {code}: not a known source")
             continue
         try:
-            ingest(code, force=force)
+            if french:
+                print(f"Ingesting {code} (français)...")
+                ingest_fr(code, force=force)
+            else:
+                ingest(code, force=force)
         except Exception as exc:
             failures.append((code, exc))
             print(f"  FAILED {code}: {type(exc).__name__}: {exc}")

canlex/rerank.py

@@ -10,7 +10,11 @@ from tokenizers import Tokenizer
 RERANK_REPO = "Xenova/bge-reranker-base"
 RERANK_ONNX = "onnx/model_quantized.onnx"  # int8: ~3x faster on CPU, negligible quality loss
 MAX_TOKENS = 512
-_MAX_DOC_CHARS = int(os.environ.get("CANLEX_RERANK_DOC_CHARS", "1000"))
+_MAX_DOC_CHARS = int(os.environ.get("CANLEX_RERANK_DOC_CHARS", "2000"))
+                    # 1000 -> 2000 adopted 2026-07-23: the doubled reading
+                    # window held every 203-Q eval metric and lifted Hit@5
+                    # +0.01; the cross-encoder now judges a full-size piece
+                    # rather than 60% of one.
                        # cap doc text before tokenizing. Cross-encoder cost is
                        # ~linear in tokens per doc, so halving this ~halves the
                        # dominant query cost; env-sweepable, eval-gated (a

canlex/synonyms.py

@@ -77,6 +77,9 @@ _SYNONYMS = [
     (r"criminalit[ée]", "criminality convicted offence"),
     (r"renvoi", "removal order"),
     (r"saisie?s?", "seizure seized goods"),
+    # 'devises' is the everyday word for currency; the PCMLTFA's French text
+    # says 'espèces et effets' -- bridge to both statutory vocabularies.
+    (r"devises", "currency monetary instruments espèces effets"),
     (r"r[ée]sidents? permanents?", "permanent resident"),
     (r"[ée]trangers?", "foreign national"),
     (r"d[ée]tention", "detention detained review"),