context_cued.py

"""Context-cued ID detection layer (standalone, multilingual).

The #1 failure of the PII token classifier is the family of *arbitrary
alphanumeric IDs* (passport, tax_id, national_id, drivers_license, medical_record,
health_plan_id, bank_account, case_number, login_credentials, password, username,
device_id). These have NO learnable shape — a passport number, a chart number and a
purchase-order number are byte-for-byte indistinguishable — so the model gets
~0 recall on them. But in real documents they almost always appear immediately
after a *field label / cue* ("Passport No.:", "Reisepass:", "MRN:", "Case No.",
"Versichertennummer:", "n° de compte", "护照号", ...).

This module detects the lead-in CUE, then captures the following VALUE token(s) as
that type. The cue GATES the match -> high precision: it will not fire on a bare
`PO#778231` / `JIRA-4821` / `ISBN ...` because those cues are not PII field labels.

Design:
  * Per type, a list of multilingual cue phrases (regex-escaped, ws-flexible),
    covering the 33 eval languages (en, de, fr, es, it, nl, pt, pl, ru, zh, ja, ko,
    ar, hi, id, vi, th, sv, fi, da, el, ro, hu, cs, sk, bg, lt, lv, et, ga, mt).
  * After a cue, allow <=3 chars of separators (colon/hash/dot/dash/space and a few
    i18n colons),
    then a VALUE: an alphanumeric token allowing internal spaces/hyphens/slashes/dots,
    ``[A-Z0-9][A-Z0-9 ./-]{2,24}[A-Z0-9]``, trimmed, longest plausible.
  * Robust to value being on the same line as the cue, and (for credentials) to an
    inline secondary cue inside the value (e.g. "User: sa Pass: Master#01").

Public API:
  context_cued_spans(text) -> [{start,end,type,text}, ...]
  hybrid_with_context(text, model_spans) -> hybrid_spans + the cued tier (authoritative)

Order in the full decode: AUTH (shape) -> CONTEXT (these IDs, cue-gated) ->
SNAP (phone/date/amount/postal) -> model spans for the rest.
"""
from __future__ import annotations
import re

# --------------------------------------------------------------------------- #
# VALUE: an arbitrary alphanumeric ID token. Must start & end on an alnum char,
# may carry internal spaces / hyphens / slashes / dots (so "1 90 12 33 123 456 78",
# "2:20-cv-09876-ABC", "12.345.678/0001-95", "20-00-00 acc: 55779911" all survive).
# Letters incl. accented Latin so "53219876S", "FK123456", "C01X00T47" survive; we
# stay Latin-only inside the value (CJK IDs are digit/Latin in this eval).
_VAL = r"[A-Za-z0-9@.#][A-Za-z0-9 ./_:@()#\-]{2,40}[A-Za-z0-9)]"
# Credential VALUE: passwords/secret tokens carry special chars (&!#$%@*?) and no
# spaces. Used ONLY for password / login-credential cues so the wider char set never
# leaks into the high-precision ID matching. Single token, no internal whitespace.
_CRED_VAL = re.compile(r"""["']?(?!@)([^\s"'@][^\s"']{3,59})["']?""")
# separators allowed between cue and value (incl. i18n colons / no.-abbreviations /
# quotes for `password = "..."`). Slightly longer cap to span `   = "`.
_SEP = r"""[\s:#=.№　：\-"']{0,6}"""
# secrets embedded in a connection string: scheme://user:PASSWORD(@host | end). The
# password must NOT be all-digits (that's a :PORT, e.g. redis://host:6379) and stops
# before '@host'.
_CONN_PW = re.compile(r"(?i)\b[a-z][a-z0-9+.\-]*://[^\s:@/]+:(?!\d+(?:[@\s\"']|$))([^\s:@/]{3,})(?=@|[\s\"']|$)")

# A trailing value must contain at least one DIGIT *or* be a credential-style token.
# IDs in this schema are numeric or alphanumeric-with-digits; pure-word tails after a
# cue ("Passport: Required") would otherwise false-fire. Credentials/usernames are
# exempt (passwords/usernames can be all-letters: "hunter2"? has a digit; "password"
# does not -> we exempt them, gated tightly by the credential cues).
_HAS_DIGIT = re.compile(r"\d")
_HAS_ALNUM = re.compile(r"[A-Za-z0-9]")

# --------------------------------------------------------------------------- #
# Cue phrases per type. Authored as raw alternation fragments; matched
# case-insensitively, with flexible internal whitespace. Keep them SPECIFIC to PII
# field labels so the precision traps (PO#, WO-, PROJ-, JIRA-, TICKET-, RMA-, INC,
# CHG, EPIC-, BUG-, GH-, ISBN, DOI, PMID, ICD-10, Rule/Section/Chapter/Title,
# port/commit/tag/Flight/Gate/Lane/Building/Channel/tracking/Receipt/invoice/INV-/
# ORD-/REF/PURCHASE) do NOT match.
_CUES: dict[str, list[str]] = {
    "identity.passport": [
        r"passport(?:\s*(?:no|number|num|nr|#))?",
        r"reisepass(?:nr|nummer)?", r"reisepass\s*-?\s*nr",
        r"n[°ºo]\.?\s*(?:de\s*)?passeport", r"passeport\s*n[°ºo]?",
        r"n[.°ºo]*\s*(?:de\s*)?pasaporte", r"pasaporte\s*n[°ºo]?",
        r"passaporto", r"n[°ºo]\.?\s*(?:de\s*)?passaporte", r"paspoort",
        r"numer\s*paszportu", r"paszport",
        r"パスポート(?:番号)?",  # パスポート(番号)
        r"여권(?:번호)?",                    # 여권(번호)
        r"护照号?",                              # 护照(号)
        r"رقم\s*جواز\s*السفر", r"جواز\s*سفر(?:\s*رقم)?",  # رقم جواز السفر / جواز سفر رقم
        r"पासपोर्ट",  # पासपोर्ट (hi)
        r"паспорт",        # паспорт (ru/bg)
        r"диабатирио",  # (filler-safe)
        r"διαβατήριο",  # διαβατήριο (el)
        r"pasul?(?:uri)?",  # ro passport-ish (guarded by digit value)
        r"h[oó]\s*chi[ếe]u", r"so\s*h[oó]\s*chi[ếe]u",        # hộ chiếu (vi)
        r"หนังสือเดินทาง",  # หนังสือเดินทาง (th)
        r"pase", r"pas\s*nr", r"reisedokument", r"uütlevee",  # da/sv/et-ish
    ],
    "identity.tax_id": [
        r"tax\s*(?:id|identification|no|number|#)?", r"\bTIN\b", r"\bEIN\b",
        r"vat\s*(?:id|no|number|reg|registration|#)?", r"\bVAT\b",
        r"steuer\s*-?\s*id", r"steuernummer", r"steuer\s*-?\s*nr", r"ust\s*-?\s*idnr",
        r"umsatzsteuer", r"\bNIF\b", r"\bCIF\b", r"\bCPF\b", r"\bCNPJ\b", r"\bRFC\b",
        r"\bNIP\b", r"\bPAN\b", r"पैन", r"\bΑΦΜ\b", r"αφμ", r"codice\s*fiscale", r"partita\s*iva",
        r"num[ée]ro\s*(?:fiscal|de\s*tva)", r"identifiant\s*fiscal",
        r"momsnr", r"momsregistrerings", r"btw\s*-?\s*nr", r"\bDPH\b",
        r"税号", r"纳税人识别号",  # 税号 / 纳税人识别号
        r"расчётный\s*номер",  # ru tax-ish
        r"ИНН", r"инн",            # ИНН (ru tax id)
        r"マイナンバー",                # マイナンバー (ja)
        r"税務", r"세금", r"사업자등록번호",
        r"رقم\s*ضريبي",  # رقم ضريبي (ar)
        r"adoazonos[ií]t[oó]", r"ad[oó]sz[aá]m",                # hu tax id
        r"daňov[eé]\s*č[ií]slo", r"ičo\s*dph",
        r"mok[ėe]t[oų]jo\s*kodas", r"pvm",                      # lt vat
    ],
    "identity.national_id": [
        r"national\s*(?:id|identity|insurance)\s*(?:no|number|#)?",
        r"\bNIN\b", r"\bNINO\b", r"\bDNI\b", r"\bNIE\b", r"\bNIF\b",
        r"personalausweis(?:nr|nummer)?", r"ausweis\s*-?\s*nr",
        r"identit[ée]\s*nationale", r"carte\s*nationale", r"num[ée]ro\s*national",
        r"s[ée]curit[ée]\s*sociale", r"num[ée]ro\s*de\s*s[ée]curit[ée]\s*sociale",
        r"n[°ºo]\.?\s*(?:de\s*)?s[ée]curit[ée]", r"insee",
        r"documento\s*nacional", r"documento\s*de\s*identidad", r"c[ée]dula",
        r"carta\s*d['’]?identit[aà]", r"codice\s*identit[aà]",
        r"\bBSN\b", r"burgerservicenummer", r"\bPESEL\b", r"\brodn[eé]\s*č[ií]slo\b",
        r"personnummer", r"henkil[öo]tunnus", r"cpr\s*-?\s*nr", r"\bCPR\b",
        r"isikukood", r"personas\s*kods", r"asmens\s*kodas",
        r"身份证(?:号|号码)?",        # 身份证(号)
        r"身份證(?:字號)?",               # 身份證(字號)
        r"주민(?:등록)?번호",         # 주민(등록)번호
        r"マイナンバー",              # マイナンバー
        r"рациональный",  # национальный
        r"номер\s*паспорта",
        r"आधार(?:\s*संख्या|\s*नंबर)?",  # आधार (संख्या) (hi)
        r"رقم\s*وطني", r"الرقم\s*الوطني",  # رقم وطني
        r"บัตรประชาชน",  # บัตรประชาชน (th)
        r"ΑΔΤ", r"αριθμ[όο]ς\s*ταυτ[όο]τητας",  # ΑΔΤ
        r"cnp", r"cod\s*numeric\s*personal",                  # ro
        r"szem[eé]lyi\s*(?:azonos[ií]t[oó]|igazolv[aá]ny)",   # hu
        r"so\s*cmnd", r"can\s*cu[oơ]c", r"cmnd", r"cccd",      # vi
        r"เลขบัตร",        # th id
        r"\bNRIC\b", r"\bKTP\b", r"nomor\s*induk\s*kependudukan", r"\bNIK\b",  # id
    ],
    "identity.drivers_license": [
        r"driver'?s?\s*licen[cs]e\s*(?:no|number|#)?", r"\bDL\b\s*#?",
        r"driving\s*licen[cs]e", r"f[üu]hrerschein(?:nr|nummer)?",
        r"permis\s*de\s*conduire", r"permis\s*conduire",
        r"permiso\s*de\s*conducir", r"carnet\s*de\s*conducir", r"licencia\s*de\s*conducir",
        r"patente\s*(?:di\s*guida|nr)?", r"rijbewijs",
        r"prawo\s*jazdy", r"k[öo]rkort", r"ajokortti", r"f[øo]rerbevis", r"k[øo]rekort",
        r"运转驾驶证", r"驾驶证",      # 驾驶证 (zh)
        r"運転免許", r"免許",                  # 運転免許 (ja)
        r"운전면허(?:증)?",                       # 운전면허(증)
        r"рукавительское",  # filler
        r"водительское\s*удостоверение",  # ru
        r"رخصة\s*(?:ال)?قيادة",  # رخصة القيادة
        r"permis\s*de\s*conducere", r"vezet[őo]i\s*enged[eé]ly",       # ro/hu
        r"ใบขับขี่",            # ใบขับขี่ (th)
        r"gi[aâ]y\s*ph[eé]p\s*l[aá]i\s*xe", r"b[aă]ng\s*l[aá]i",        # vi
    ],
    "healthcare.medical_record": [
        r"\bMRN\b", r"medical\s*record\s*(?:no|number|#)?", r"med\.?\s*rec\.?\s*(?:no|#)?",
        r"chart\s*(?:no|number|#)?", r"patient\s*(?:id|no|number|#)",
        r"\bUR\s*(?:number|no|#)?\b", r"\bNHS\s*(?:no|number)?\b", r"health\s*record",
        r"aktenzeichen", r"patientennummer", r"fallnummer", r"patienten\s*-?\s*id",
        r"n[°ºo]\.?\s*(?:de\s*)?dossier(?:\s*m[ée]dical)?", r"dossier\s*m[ée]dical",
        r"dossiernummer", r"nr\.?\s*dosar(?:\s*medical)?", r"dosar\s*medical",
        r"nr\.?\s*karty", r"medicininės\s*kortelės\s*nr", r"kortelės\s*nr",
        r"n[ºo]\.?\s*(?:de\s*)?historia\s*cl[ií]nica", r"historia\s*cl[ií]nica",
        r"numero\s*de\s*historia", r"n[uú]mero\s*de\s*historia",
        r"cartella\s*clinica", r"numero\s*cartella",
        r"prontu[áa]rio", r"pacientennummer", r"pati[ëe]ntnummer",
        r"病历号", r"病歷號", r"医疗记录",  # 病历号
        r"カルテ番号", r"患者番号",            # カルテ番号 (ja)
        r"차트번호", r"환자번호",                 # 차트번호 (ko)
        r"регистрационный", # filler
        r"номер\s*медицинской\s*карты",  # ru
        r"رقم\s*الملف\s*الطبي",  # رقم الملف الطبي
        r"เวชระเบียน", # th medical record
    ],
    "healthcare.health_plan_id": [
        r"health\s*plan\s*(?:id|no|number|#)?(?:\s*is)?", r"member\s*(?:id|no|number|#)",
        r"id\s*plan\s*de\s*s[aă]n[aă]tate", r"plan\s*de\s*s[aă]n[aă]tate",
        r"convenio", r"conv[êe]nio", r"स्वास्थ्य\s*योजना(?:\s*संख्या)?",
        r"policy\s*(?:no|number|#|id)", r"insurance\s*(?:id|no|number|#)",
        r"plan\s*id", r"subscriber\s*(?:id|no|#)", r"group\s*(?:no|number|#)\s*id",
        r"versichertennummer", r"versicherten\s*-?\s*nr", r"krankenversicherung",
        r"versicherungsnummer", r"\bAOK\b\s*versichert",
        r"n[°ºo]\.?\s*(?:de\s*)?mutuelle", r"num[ée]ro\s*d['’]?assur[ée]", r"\bCPAM\b",
        r"n[ºo]\.?\s*(?:de\s*)?(?:p[óo]liza|seguro)", r"n[uú]mero\s*de\s*afiliaci[óo]n",
        r"tessera\s*sanitaria", r"polizza", r"numero\s*assicurato",
        r"zorgverzekering", r"polisnummer",
        r"保险号", r"医保号", r"医疗保险",  # 保险号
        r"保険証番号", r"被保険者番号",  # 保険証番号 (ja)
        r"보험증번호", r"건강보험",            # 보험증번호 (ko)
        r"номер\s*полиса", r"ОМС",  # номер полиса / ОМС
        r"رقم\s*(?:الت[أا])?مين",  # رقم التأمين
        r"szem[eé]lyi\s*biztos[ií]t",  # hu insurance-ish
    ],
    "financial.bank_account": [
        r"bank\s*account\s*(?:no|number|#)?", r"\baccount\s*(?:no|number|#)",
        r"\bacct\b\.?\s*(?:no|#)?", r"\bacc\b\.?\s*(?:no|#)?", r"\ba/?c\b\s*(?:no|#)?",
        r"checking\s*(?:account|no|#)?", r"savings\s*(?:account|no|#)?",
        r"sort\s*code", r"routing\s*(?:no|number|#)?", r"\bABA\b", r"transit\s*(?:no|#)?",
        r"konto(?:nummer|nr)?", r"konto\s*-?\s*nr", r"bankverbindung",
        r"n[°ºo]\.?\s*(?:de\s*)?compte", r"compte\s*bancaire", r"\bRIB\b",
        r"n[ºo]\.?\s*(?:de\s*)?cuenta", r"cuenta\s*bancaria", r"numero\s*de\s*cuenta",
        r"conto\s*(?:corrente|bancario)?", r"numero\s*di\s*conto",
        r"conta\s*(?:banc[áa]ria|corrente)?", r"n[uú]mero\s*da\s*conta",
        r"rekeningnummer", r"bankrekening",
        r"numer\s*konta", r"nr\s*konta", r"kontonr", r"bankkonto",
        r"kontonummer", r"tilinumero", r"pankkitili", r"konto\s*nr",
        r"银行账号", r"账号", r"帐号", r"銀行口座",  # 银行账号
        r"口座番号", r"銀行口座",                  # 口座番号 (ja)
        r"계좌번호", r"은행계좌",                 # 계좌번호 (ko)
        r"номер\s*счета", r"расчетный\s*счет",  # номер счета
        r"рахм\s*алхисаб",  # رقم الحساب-ish
        r"رقم\s*الحساب",             # رقم الحساب (ar)
        r"cont\s*bancar", r"num[aă]r\s*de\s*cont",                                # ro
        r"banksz[aá]mla", r"sz[aá]mlasz[aá]m",                                    # hu
        r"so\s*t[aà]i\s*kho[aả]n", r"t[aà]i\s*kho[aả]n",                          # vi
        r"เลขที่บัญชี",    # เลขที่บัญชี (th)
    ],
    "legal.case_number": [
        r"case\s*(?:no|number|#)", r"docket\s*(?:no|number|#)?",
        r"cause\s*(?:no|number)", r"indictment\s*(?:no|number)", r"file\s*(?:no|number)",
        r"aktenzeichen", r"gesch[äa]ftsnummer", r"\bAz\.?\s*:",
        r"n[°ºo]\.?\s*(?:de\s*)?(?:r[ôo]le|dossier|affaire)", r"r[ée]f[ée]rence\s*affaire",
        r"\bR\.?\s*G\.?\s*n", r"numero\s*di\s*ruolo", r"procedimento\s*n",
        r"n[ºo]\.?\s*(?:de\s*)?(?:expediente|procedimiento|causa)", r"autos\s*n",
        r"sygnatura(?:\s*akt)?", r"\bsygn\.?\s*akt\b",
        r"zaaknummer", r"rolnummer", r"m[ åa]lnummer", r"sagsnr", r"asianumero",
        r"案件号", r"案号", r"案件编号",       # 案件号 (zh)
        r"事件番号", r"裁判番号",                 # 事件番号 (ja)
        r"사건번호",                                             # 사건번호 (ko)
        r"номер\s*дела", r"дело\s*№",  # номер дела
        r"رقم\s*القضية",             # رقم القضية (ar)
        r"num[aă]r\s*(?:dosar|de\s*[ií]nregistrare)", r"dosar\s*nr",              # ro
        r"[üu]gysz[aá]m", r"\bb[ií]r[oó]s[aá]gi\b",                               # hu
        r"so\s*v[uụ]\s*[aá]n", r"so\s*h[oồ]\s*s[oơ]",                             # vi
        r"αριθμ[όο]ς\s*υπ[οό]θεσης",  # el
        r"spr[aá]vne\s*č[ií]slo", r"č[ií]slo\s*jednac[ií]",            # cs
    ],
    "developer.login_credentials": [
        r"login\s*credentials?", r"credentials?", r"login\s*=?", r"logon",
        r"oauth_token", r"oauth_secret", r"access[_\s]*token", r"auth\s*token",
        r"anmeldedaten", r"zugangsdaten", r"identifiants?\s*de\s*connexion",
        r"credenciales", r"credenziali", r"inloggegevens", r"dane\s*logowania",
        r"登录凭据", r"ログイン情報",
        r"인증정보", r"бианиевые",
        r"учётные\s*данные",  # учётные данные (ru)
    ],
    "credential.password": [
        r"password", r"passwd", r"\bpwd\b", r"pass\b", r"passphrase",
        r"passwort", r"kennwort", r"mot\s*de\s*passe", r"contrase[ñn]a",
        r"senha", r"wachtwoord", r"has[łl]o", r"heslo", r"l[öo]senord", r"salasana", r"adgangskode",
        r"密码", r"パスワード", r"비밀번호",
        r"пароль",                                  # пароль (ru/bg)
        r"كلمة\s*(?:ال)?سر",              # كلمة السر (ar)
        r"รหัสผ่าน",                      # รหัสผ่าน (th)
        r"m[aậ]t\s*kh[aẩ]u", r"jelsz[oó]", r"parol[aă]", r"sl[aā]žvārds",
    ],
    "online.username": [
        r"username", r"user\s*name", r"\buser\b", r"\buserid\b", r"user\s*id",
        r"handle", r"account\s*name", r"\bacct\s*name\b", r"screen\s*name",
        r"login\s*name", r"nick(?:name)?", r"\bid\s*utilisateur\b",
        r"benutzername", r"benutzer\b", r"nom\s*d['’]?utilisateur", r"identifiant",
        r"nombre\s*de\s*usuario", r"usuario", r"nome\s*utente", r"nome\s*de\s*usu[áa]rio",
        r"gebruikersnaam", r"nazwa\s*u[żz]ytkownika", r"u[żz]ytkownik",
        r"anv[äa]ndarnamn", r"k[äa]ytt[äa]j[äa]tunnus", r"vartotojas", r"brugernavn",
        r"用户名", r"ユーザー名", r"사용자명", r"아이디",  # 用户名 / ユーザー名
        r"имя\s*пользователя", r"пользователь", r"логин",  # имя пользователя / пользователь / логин
        r"اسم\s*المستخدم",  # اسم المستخدم
        r"ชื่อผู้ใช้",          # ชื่อผู้ใช้ (th)
        r"t[eê]n\s*(?:đăng\s*nh[aậ]p|ng[uư][oờ]i\s*d[uù]ng)",          # vi
        r"felhaszn[aá]l[oó]n[eé]v",  # hu
    ],
    "developer.device_id": [
        r"device\s*(?:id|asset\s*tag|serial)?", r"device\s*asset\s*tag", r"asset\s*tag",
        r"\bIMEI\b", r"\bUDID\b", r"\bESN\b", r"\bMEID\b", r"\bSN\b\s*:?",
        r"serial\s*(?:no|number|#)?", r"seriennummer", r"ger[äa]te\s*-?\s*id",
        r"asset\s*-?\s*nr", r"asset\s*-?\s*nummer", r"ger[äa]tenummer",
        r"num[ée]ro\s*de\s*s[ée]rie", r"identifiant\s*(?:de\s*l['’]?)?appareil",
        r"n[uú]mero\s*de\s*serie", r"identificador\s*de\s*dispositivo",
        r"numero\s*di\s*serie", r"id\s*dispositivo", r"apparaat\s*id",
        r"设备号", r"设备标识", r"序列号",  # 设备号
        r"デバイス番号", r"シリアル番号",  # デバイス番号
        r"장치아이디", r"일련번호",            # 장치 아이디 (ko)
        r"номер\s*устройства",  # номер устройства
        r"رقم\s*الجهاز",             # رقم الجهاز (ar)
    ],
}

# Inline secondary cues for credential pairs: inside a login_credentials VALUE the
# user/pass tokens carry their own micro-cues. We keep the whole "User: x Pass: y"
# region as one login_credentials span (matches the gold which does the same).
_LOGIN_PAIR = re.compile(
    r"(?i)\b(?:user|usuario|benutzer|utilisateur|u|login|name)\b\s*[:=]?\s*\S+"
    r".{0,8}?\b(?:pass(?:word)?|pwd|passwort|mot\s*de\s*passe|p|token|secret)\b\s*[:=]?\s*\S+"
)

# Compile: (type, compiled cue regex). Longer/more-specific cues first within a type
# so the alternation prefers the most specific label.
def _compile(cues: list[str]) -> re.Pattern:
    # sort by descending raw length so e.g. "national insurance no" beats "id"
    ordered = sorted(cues, key=len, reverse=True)
    return re.compile(r"(?i)(?:" + r"|".join(ordered) + r")")

_CUE_RX = [(t, _compile(cs)) for t, cs in _CUES.items()]
_VAL_RX = re.compile(_VAL)

# types whose value may be all-letters (no digit required)
_LETTER_OK = {"credential.password", "online.username", "developer.login_credentials"}

# a username/handle: single token (handles may carry @ . _ -), NOT a capitalized prose
# word ("Holder", "Statement", "Bank"). Rejects multi-word values and Title-case words.
_PROSE_WORD = re.compile(r"^[A-ZÀ-Þ][a-zà-ÿ]+$")
def _username_ok(val: str, sep: str) -> bool:
    if " " in val:                       # usernames don't contain spaces
        return False
    if _PROSE_WORD.match(val):           # 'Holder', 'Statement', 'Bank', 'Sort'
        return False
    # require a real label delimiter (':' '=') OR an '@'-handle: a bare "User cannot"
    # (cue + space + word) is prose, not a labelled field.
    if not (":" in sep or "=" in sep or val.startswith("@")):
        return False
    return True
# login_credentials standalone value must look credential-ish (has digit/symbol or '='),
# not a bare prose word ('attempts', 'success', 'cannot').
def _login_ok(val: str) -> bool:
    if _PROSE_WORD.match(val) or (val.isalpha() and val.islower()):
        return False
    return True

# password-context English words that follow the cue 'password' in prose
# ("password authentication failed", "password expired") -> not a password value.
_PW_STOP = {"authentication", "expired", "required", "reset", "change", "changed",
            "incorrect", "invalid", "failed", "failure", "policy", "rotation",
            "manager", "protected", "field", "must", "should", "cannot", "and", "for",
            "the", "is", "was", "has", "not", "verification", "recovery", "strength"}
def _password_ok(val: str) -> bool:
    if " " in val:
        return False
    return val.lower() not in _PW_STOP

# A short stop-list of cue *contexts* that are traps even though a sub-cue matched.
# e.g. "ISBN", "DOI", "PMID", "PO#", "JIRA-" never become our types because their
# cue strings are simply not in _CUES — so no extra guard needed there. But a few
# generic English words ("user", "pass", "acc", "id", "SN") can appear in non-PII
# contexts; we gate them by requiring a plausible value right after.

# value characters that, if the value is ENTIRELY one of these shapes, indicate a
# version/path/non-PII tail we should reject even after a cue (rare; cue already gates).
_VERSIONISH = re.compile(r"^v?\d+(?:\.\d+){2,}$")  # 1.29.2, v2.0.0


def _trim(text: str, s: int, e: int) -> tuple[int, int]:
    while s < e and not _HAS_ALNUM.match(text[s]):
        s += 1
    while e > s and not _HAS_ALNUM.match(text[e - 1]):
        e -= 1
    return s, e


# country/jurisdiction & abbreviation connector tokens that legitimately sit INSIDE an
# id value ("DL: WA: SMITHJ123AB", "sort: 20-00-00 acc: 55779911", "No. 1 90 12 ...").
_ID_CONNECTORS = {"no", "no.", "nr", "nr.", "acc", "acc:", "acct", "sort", "bsb",
                  "transit", "routing", "checking", "savings", "de", "fr", "id",
                  "uid", "udid", "sn", "esn", "imei", "tva", "vat", "nip", "iva"}
_LOWER_WORD = re.compile(r"^[a-zà-öø-ÿ]+$")
def _id_like_chunk(c: str) -> bool:
    """A space-separated chunk that plausibly continues an arbitrary-ID value:
    contains a digit, is all-caps, or is a known connector/jurisdiction token. A plain
    lowercase word ('email', 'phone', 'oder', 'etwas', 'next') is prose -> ends value."""
    if not c:
        return False
    if _HAS_DIGIT.search(c):
        return True
    cc = c.rstrip(".:#-/")
    if not cc:
        return False
    if cc.lower() in _ID_CONNECTORS:
        return True
    if _LOWER_WORD.match(cc):              # pure lowercase word -> prose, stop
        return False
    if cc.isupper():                       # DNI, CPAM, BCBS, WA, NHS, SMITHJ ...
        return True
    return False                            # mixed-case word w/o digit -> stop


def _bound_value(text: str, vs: int, ve: int) -> int:
    """Stop a multi-token value at the first non-ID-like (prose) chunk, so a value on
    the same line as following prose ('C01X00T47 email a@b.com') is not over-captured.
    The first chunk after the cue is always kept (it IS the id head)."""
    seg = text[vs:ve]
    if " " not in seg:
        return ve
    parts = seg.split(" ")
    keep = 1
    for c in parts[1:]:
        if _id_like_chunk(c):
            keep += 1
        else:
            break
    if keep == len(parts):
        return ve
    new = vs + len(" ".join(parts[:keep]))
    return new


def context_cued_spans(text: str) -> list[dict]:
    """Detect cue -> value ID spans. Returns [{start,end,type,text}], non-overlapping,
    cue-gated (high precision)."""
    out = []
    claimed = [False] * len(text)

    # Within a type, find every cue occurrence and grab the following value.
    # Process types in a priority order so that when two cues overlap (e.g. "user"
    # for username vs "User:" inside a login pair) the more specific wins. We let
    # login_credentials pairs be detected first (they subsume user/pass micro-cues).
    candidates = []  # (start, end, type, specificity)
    sep_rx = re.compile(_SEP)
    for typ, rx in _CUE_RX:
        # credential secrets are single tokens with special chars -> use _CRED_VAL
        cred = typ in ("credential.password", "developer.login_credentials")
        for m in rx.finditer(text):
            cue_end = m.end()
            # capture value starting within _SEP chars after the cue
            sep = sep_rx.match(text, cue_end)
            val_start = sep.end() if sep else cue_end
            if cred:
                vm = _CRED_VAL.match(text, val_start)
                if not vm:
                    continue
                vs, ve = vm.start(1), vm.end(1)
            else:
                vm = _VAL_RX.match(text, val_start)
                if not vm:
                    continue
                vs, ve = _trim(text, vm.start(), vm.end())
                ve = _bound_value(text, vs, ve)
                vs, ve = _trim(text, vs, ve)
            if ve - vs < 3:
                continue
            val = text[vs:ve]
            # gating
            if typ not in _LETTER_OK and not _HAS_DIGIT.search(val):
                continue
            if _VERSIONISH.match(val.replace(" ", "")):
                continue
            if typ == "online.username" and not _username_ok(val, text[cue_end:val_start]):
                continue
            if typ == "developer.login_credentials" and not _login_ok(val):
                continue
            if typ == "credential.password" and not _password_ok(val):
                continue
            # specificity = length of matched cue (longer cue = more specific label)
            spec = m.end() - m.start()
            candidates.append((vs, ve, typ, spec, val))

    # Login-credential pairs: capture the whole user/pass region as one span.
    for m in _LOGIN_PAIR.finditer(text):
        s, e = _trim(text, m.start(), m.end())
        if e - s >= 3:
            candidates.append((s, e, "developer.login_credentials", 9999, text[s:e]))

    # Connection-string embedded password: scheme://user:PASSWORD@host
    for m in _CONN_PW.finditer(text):
        vs, ve = m.start(1), m.end(1)
        if ve - vs >= 3:
            candidates.append((vs, ve, "credential.password", 5000, text[vs:ve]))

    # Resolve overlaps: prefer higher specificity, then longer span.
    candidates.sort(key=lambda c: (-c[3], -(c[1] - c[0])))
    for vs, ve, typ, spec, val in candidates:
        if any(claimed[vs:ve]):
            continue
        for i in range(vs, ve):
            claimed[i] = True
        out.append({"start": vs, "end": ve, "type": typ, "text": text[vs:ve]})

    out.sort(key=lambda d: (d["start"], d["end"]))
    return out


# --------------------------------------------------------------------------- #
# Hybrid variant: AUTH -> CONTEXT (these IDs, authoritative) -> SNAP -> model.
# We reuse the shipped self-contained decode (hybrid_spans + _AUTH_TYPES) by loading
# it from the v8 model dir, so AUTH/SNAP logic is never re-implemented here.
_BASE_DECODE = None
def _load_base_decode():
    global _BASE_DECODE
    if _BASE_DECODE is None:
        import importlib.util
        path = "/lambdafs/simon/models/pii-detect-v8/pii_hybrid_decode.py"
        spec = importlib.util.spec_from_file_location("pii_hybrid_decode_v8", path)
        mod = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(mod)
        _BASE_DECODE = (mod.hybrid_spans, mod._AUTH_TYPES)
    return _BASE_DECODE


def hybrid_with_context(text: str, model_spans: list[dict]) -> list[dict]:
    """Run the existing hybrid decode, then layer the cue-gated CONTEXT tier on top.
    CONTEXT spans are authoritative for their (Group-A) types: they REPLACE any model
    span of the same type that they overlap, and own their boundaries. AUTH still wins
    over CONTEXT on overlap (shape-bearing formats take priority)."""
    hybrid_spans, _AUTH_TYPES = _load_base_decode()

    base = hybrid_spans(text, model_spans)
    cued = context_cued_spans(text)
    if not cued:
        return base

    cued_types = set(_CUES)
    # mark char ranges owned by AUTH spans (AUTH > CONTEXT)
    auth_claim = [False] * len(text)
    for sp in base:
        if sp["type"] in _AUTH_TYPES:
            for i in range(sp["start"], sp["end"]):
                auth_claim[i] = True

    kept_cued = []
    for c in cued:
        if any(auth_claim[c["start"]:c["end"]]):
            continue  # AUTH owns this region
        kept_cued.append(c)

    cued_ranges = [(c["start"], c["end"]) for c in kept_cued]

    def _overlaps_cued(sp):
        for s, e in cued_ranges:
            if min(e, sp["end"]) > max(s, sp["start"]):
                return True
        return False

    out = []
    for sp in base:
        # drop model/SNAP spans of a CONTEXT type that overlap a cued span
        if sp["type"] in cued_types and _overlaps_cued(sp):
            continue
        out.append(sp)
    out.extend(kept_cued)

    seen, uniq = set(), []
    for sp in sorted(out, key=lambda s: (s["start"], s["end"])):
        k = (sp["start"], sp["end"], sp["type"])
        if k in seen:
            continue
        if len(text[sp["start"]:sp["end"]].strip()) < 3:
            continue
        seen.add(k)
        uniq.append(sp)
    return uniq

CONTEXT_TYPES = set(_CUES)  # the 12 Group-A types this tier owns


# =========================================================================== #
# P2 — Group-B cue-gated capture (financial.amount / identity.date_of_birth /
# contact.phone / contact.postal_code). High-precision: BOTH a cue AND a
# value-shape must hold, so the precision-trap docs gain no new false positives.
# These ADD spans, authoritative for their type. Distinct from the Group-A ID
# tier above (those have no shape; these are shape + cue).
# --------------------------------------------------------------------------- #
# Shapes (reuse the shipped SNAP shapes so boundaries match the rest of the decode).
_GB_PHONE = re.compile(
    r"(?<!\d)(?:\+?\d{1,3}[ \-.]?)?(?:\(\d{2,4}\)[ \-.]?)?\d{2,4}[ \-.]?\d{3}[ \-.]?\d{3,4}(?!\d)")
_GB_DATE = re.compile(
    r"\b(?:\d{1,2}[\/.\-]\d{1,2}[\/.\-]\d{2,4}"
    r"|\d{4}[\/.\-]\d{1,2}[\/.\-]\d{1,2}"
    r"|(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\.?\s+\d{1,2},?\s+\d{4}"
    r"|\d{1,2}\.?\s+(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec|"
    r"janv|f[ée]vr|mars|avr|mai|juin|juil|ao[ûu]t|sept|oct|nov|d[ée]c|"
    r"ene|abr|ago|dic|gen|mag|giu|lug|set|ott)[a-zéûô]*\.?\s+\d{4}"
    r"|\d{1,2}\s*(?:de\s+)?(?:[A-Za-zÀ-ÿ]{3,12})\s*(?:de\s+)?\d{4})\b")
_GB_AMOUNT = re.compile(
    r"(?:[$€£¥₹₩]\s?\d[\d.,]*(?:\s?[KMB])?"
    r"|\b(?:USD|EUR|GBP|JPY|CHF|CAD|AUD|INR|KRW)\s?\d[\d.,]*(?:\s?[KMB])?\b"
    r"|\b\d[\d.,]*\s?(?:USD|EUR|GBP|JPY|CHF|INR|KRW|dollars|euros|pounds|yen)\b"
    r"|\b\d[\d.,]*\s?[$€£¥₹₩])")
_GB_POSTAL = re.compile(
    r"\b(?:\d{5}(?:-\d{4})?|[A-Z]{1,2}\d[A-Z\d]?\s?\d[A-Z]{2}|\d{4}\s?[A-Z]{2}|\d{4})\b")

# Cues (multilingual). Amount: also fires on a bare currency symbol adjacent to a
# number even WITHOUT a word cue (the symbol IS the cue). Phone reuses tel/mobile/...
_GB_CUES = {
    "financial.amount": re.compile(
        r"(?i)\b(?:amount|total|salary|balance|fee|price|sum|due|cost|charge|payment|"
        r"betrag|gehalt|saldo|geb[üu]hr|preis|summe|montant|salaire|solde|frais|prix|"
        r"importe|salario|saldo|precio|importo|stipendio|prezzo|valor|sal[áa]rio|"
        r"金额|金額|급여|잔액|사용료|сумма|оклад|баланс|المبلغ|الراتب|"
        r"kwota|wynagrodzenie|bel[oø]p|bel[ée]ag|summa|sum[ma]|m[ćc]e)\b"),
    "identity.date_of_birth": re.compile(
        r"(?i)(?:\bDOB\b|\bD\.?O\.?B\.?|date\s*of\s*birth|born(?:\s*on)?|"
        r"geburtsdatum|geb\.?\s*am|geboren|date\s*de\s*naissance|n[ée]\s*le|"
        r"fecha\s*de\s*nacimiento|nacid[oa]\s*el|data\s*di\s*nascita|nat[oa]\s*il|"
        r"data\s*de\s*nascimento|geboortedatum|data\s*urodzenia|"
        r"生年月日|生日|出生日期|생년월일|дата\s*рождения|تاريخ\s*الميلاد|"
        r"ng[àa]y\s*sinh|วันเกิด|f[öo]delsedatum|syntym[äa]aika)"),
    "contact.phone": re.compile(
        r"(?i)\b(?:tel|telephone|t[ée]l[ée]phone|phone|mobile|mob|cell|cellphone|"
        r"call|fax|whatsapp|handy|telefon|tel[ée]fono|telefono|telefoon|"
        r"電話|手机|手機|전화|휴대폰|телефон|موبايل|هاتف|"
        r"telefon|puhelin|tlf|s[đd]t|เบอร์โทร|m[óo]vil|celular|n[úu]mero)\b"),
    "contact.postal_code": re.compile(
        r"(?i)\b(?:zip|zip\s*code|postal\s*code|postcode|post\s*code|plz|postleitzahl|"
        r"code\s*postal|c[óo]digo\s*postal|cap|codice\s*postale|c[ée]p|postcode|"
        r"kod\s*pocztowy|邮编|邮政编码|郵便番号|우편번호|почтовый\s*индекс|"
        r"الرمز\s*البريدي|postnummer|postinumero|m[ãa]\s*b[uư]u)\b"),
}
# Amount also fires on bare currency symbol -> number (symbol is the cue).
_GB_BARE_AMOUNT = re.compile(
    r"(?:[$€£¥₹₩]\s?\d[\d.,]*(?:\s?[KMB])?|\b\d[\d.,]*\s?[$€£¥₹₩])")

# precision: a value must sit within this many chars AFTER the cue (cue->value gate).
_GB_WINDOW = 24
_GB_SHAPES = {
    "financial.amount": _GB_AMOUNT,
    "identity.date_of_birth": _GB_DATE,
    "contact.phone": _GB_PHONE,
    "contact.postal_code": _GB_POSTAL,
}


def group_b_cue_spans(text: str) -> list[dict]:
    """Cue+shape gated Group-B spans. A span is emitted only when a value of the
    right SHAPE appears within _GB_WINDOW chars after a type cue (or, for amounts,
    when a bare currency symbol abuts the number). High precision by construction:
    no cue -> no span, so the precision traps stay clean."""
    out = []
    claimed = [False] * len(text)
    cand = []  # (start, end, type, specificity)

    for typ, cue_rx in _GB_CUES.items():
        shape = _GB_SHAPES[typ]
        for cm in cue_rx.finditer(text):
            window = text[cm.end():cm.end() + _GB_WINDOW]
            vm = shape.search(window)
            if not vm:
                continue
            # value must START within the window (not be far downstream prose)
            vs = cm.end() + vm.start()
            ve = cm.end() + vm.end()
            # require only separators/space between cue and value
            gap = text[cm.end():vs]
            if not re.fullmatch(r"[\s:#=.\-/()　：№\"']{0,24}", gap):
                continue
            cand.append((vs, ve, typ, cm.end() - cm.start() + 100))

    # bare currency-symbol amounts (symbol is the cue)
    for vm in _GB_BARE_AMOUNT.finditer(text):
        cand.append((vm.start(), vm.end(), "financial.amount", 50))

    cand.sort(key=lambda c: (-c[3], -(c[1] - c[0])))
    for vs, ve, typ, _spec in cand:
        if vs < 0 or ve > len(text) or any(claimed[vs:ve]):
            continue
        if len(text[vs:ve].strip()) < 3:
            continue
        for i in range(vs, ve):
            claimed[i] = True
        out.append({"start": vs, "end": ve, "type": typ, "text": text[vs:ve]})
    out.sort(key=lambda d: (d["start"], d["end"]))
    return out