lighthouse/gate/export_expectations.py

"""Export the Python gate's verdicts so the TypeScript port can be diffed against them.

    python -m lighthouse.gate.export_expectations

Writes ``fixtures/gate_expectations.json``: for all 80 synthetic conversations, the
conversation-level verdict and every per-student-turn verdict, plus a bank of adversarial
one-line probes.

## Why this file exists

The gate now runs in two languages. Python owns the escalation card and every offline
number; TypeScript owns the live-chat path, because a gate that has to make an HTTP call to
a sleeping HF Space cannot honour "crisis resources render unconditionally". Two
implementations of one safety rule is a liability unless something forces them to agree.

This is that something, and it closes the loop in **both** directions:

* ``ml/tests/test_ts_conformance.py`` regenerates this file from the live Python gate and
  fails if the committed copy differs. So a Python pattern edit that is not re-exported
  breaks the Python suite.
* ``web/src/lib/gate/gate.conformance.test.ts`` reads this file, runs the TS gate over the
  same inputs, and diffs every field. So a Python edit that IS re-exported but not mirrored
  in TS breaks the web suite.

Neither test can be satisfied by editing one runtime alone, which is the property we want.

The probes are not decoration. Each one is a bug the gate actually had, or an adversarial
phrasing from the day 3 corpus. A regex that ports "almost right" tends to fail on exactly
these, not on the ordinary conversations.
"""

from __future__ import annotations

import json

from lighthouse import config
from lighthouse.data.synthetic import load
from lighthouse.gate.patterns import pattern_count
from lighthouse.gate.safety import evaluate_conversation, evaluate_turn

OUTPUT = config.FIXTURES_DIR / "gate_expectations.json"

#: Single-turn probes. Every one of these is either a bug the gate had during day 3 or an
#: adversarial phrasing that a naive port gets wrong. Keep the comments: they are why the
#: probe is here, and a future reader deleting a "redundant" probe would lose the reason.
PROBES: tuple[str, ...] = (
    # --- the five day 3 bugs, as regression probes -----------------------------------
    "we walked 5 kms for the charity thing",          # leetspeak vs digits
    "im gonna kill myself",                           # bare \w+ target swallowed "myself"
    "i'm going to end it all tonight",                # same, plus imminence promotion
    "he says he'll make me pay",                      # victim-voice direction
    "i wrote a note for my mum",                      # mandatory qualifier
    "he said he'd stab me after school",              # floor/ceiling contradiction
    # --- the day 5 bug: contraction vs expansion must floor identically ---------------
    "i've been cutting my arms",                      # floored T4 before the fix
    "i have been cutting my arms",                    # floored NOTHING before the fix
    "i had been cutting my arms again",
    "i have stopped cutting my arms",                 # must stay suppressed after the fix
    "i haven't been cutting my arms",                 # must stay suppressed after the fix
    # --- the day 7 bug: every future form of a reported threat must floor ----------
    "kai says he'll batter me",                       # floored NOTHING before the fix
    "kai said he'd batter me",
    "kai says he will batter me",
    "she says she'll kill me",
    "they said they'll beat me",
    # --- suppressor mechanics, containment not co-occurrence -------------------------
    "this coursework is killing me",
    "he told me to kill myself",
    "he told me to kill myself, and honestly i do want to die",
    "i would never kill myself",
    "i'm not going to hurt myself",
    "i stopped cutting last year",
    "it's suicide prevention week at school",
    "we used a butter knife in food tech",
    "i'm gonna smash you at fifa",
    "i still want to die of embarrassment",
    # --- severity grading, the rule that decides break-glass -------------------------
    "i've been cutting my arms",                      # STRONG  -> T4
    "we did a lesson on self harm",                   # suppressed
    "self harm",                                      # MODERATE -> T3, not T4
    "there was a knife",                              # WEAK    -> no floor at all
    "someone brought a knife to school",              # STRONG, one hit not three
    # --- modifiers never floor alone -------------------------------------------------
    "see you tomorrow",
    "after school",
    "i'll see him at lunch",
    # --- leetspeak evasion still caught ----------------------------------------------
    "i w4nt t0 d!e",
    "k1ll my5elf",
    # --- third party, and the empty case ---------------------------------------------
    "my friend said she wants to die and i'm scared",
    "",
    "   ",
)


def build() -> dict:
    conversations = []
    for convo in load():
        turns = convo.student_turns
        conversations.append(
            {
                "id": convo.id,
                "tier": convo.tier.value,
                "n_student_turns": len(turns),
                "conversation": evaluate_conversation(turns).to_dict(),
                "turns": [evaluate_turn(t, i).to_dict() for i, t in enumerate(turns)],
            }
        )

    return {
        "generated_by": "lighthouse.gate.export_expectations",
        "note": (
            "Generated, do not hand-edit. Regenerate with "
            "`python -m lighthouse.gate.export_expectations` after any change under "
            "ml/lighthouse/gate/, and mirror the change in web/src/lib/gate/."
        ),
        "pattern_count": pattern_count(),
        "mirrored_config": {
            "GATE_HIGH_SCORE": config.GATE_HIGH_SCORE,
            "GATE_GREY_SCORE": config.GATE_GREY_SCORE,
            "GATE_SEVERITY_WEIGHTS": config.GATE_SEVERITY_WEIGHTS,
            "GATE_FLOOR_MIN_WEIGHT": config.GATE_FLOOR_MIN_WEIGHT,
            "GATE_CEILING_WITHOUT_T4_EVIDENCE": config.GATE_CEILING_WITHOUT_T4_EVIDENCE,
            "MAX_CITED_QUOTES": config.MAX_CITED_QUOTES,
            "INTAKE_MODEL": config.INTAKE_MODEL,
            "CONCERN_THRESHOLD": config.CONCERN_THRESHOLD,
            "COUNSELLOR_WEEKLY_BUDGET": config.COUNSELLOR_WEEKLY_BUDGET,
            "RETENTION_DAYS_NON_ESCALATED": config.RETENTION_DAYS_NON_ESCALATED,
            "CLASSIFIER_TIMEOUT_SECONDS": config.CLASSIFIER_TIMEOUT_SECONDS,
        },
        "probes": [
            {"text": text, "verdict": evaluate_turn(text).to_dict()} for text in PROBES
        ],
        "conversations": conversations,
    }


def main() -> None:
    payload = build()
    OUTPUT.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")

    n_turns = sum(c["n_student_turns"] for c in payload["conversations"])
    print(f"wrote {OUTPUT.relative_to(config.REPO_ROOT)}")
    print(f"  {len(payload['conversations'])} conversations")
    print(f"  {n_turns} per-turn verdicts")
    print(f"  {len(payload['probes'])} probes")
    print(f"  {sum(payload['pattern_count'].values())} patterns")


if __name__ == "__main__":
    main()