modal_eval.py

"""Baseline fairness eval on real Qwen2.5-7B, run on Modal GPU.

Modal does ONLY the GPU generation; parsing + scoring happen locally with your
existing engine, so this also doubles as a clean test of whether the prompted 7B
emits parseable JSON (it reports the parse-failure rate explicitly).

    pip install modal && modal setup        # one-time
    modal run modal_eval.py                  # baseline on prompted Qwen2.5-7B

Run this BEFORE fine-tuning to get your "before" number. After you train the LoRA,
point MODEL_ID at the merged checkpoint and run it again for the "after". The delta
is your Field Notes post + the evidence the fine-tune made Gorm *fairer*.
"""

import json
from pathlib import Path

import modal

APP = modal.App("bridge-troll-eval")
MODEL_ID = "Qwen/Qwen2.5-7B-Instruct"

image = modal.Image.debian_slim().pip_install(
    "torch", "transformers>=4.45,<5", "accelerate", "sentencepiece"
)

SEEDS = Path(__file__).parent / "data" / "eval_seeds.jsonl"
BANDS = {"weak": (0, 2), "mid": (2, 4), "strong": (4, 5), "none": (0, 0)}


@APP.function(image=image, gpu="A10G", timeout=60 * 30)
def generate_all(system_prompt: str, texts: list[str]) -> list[str]:
    import torch
    from transformers import AutoModelForCausalLM, AutoTokenizer

    tok = AutoTokenizer.from_pretrained(MODEL_ID)
    model = AutoModelForCausalLM.from_pretrained(
        MODEL_ID, torch_dtype=torch.bfloat16, device_map="auto"
    )
    out = []
    for text in texts:
        msgs = [{"role": "system", "content": system_prompt},
                {"role": "user", "content": text}]
        inputs = tok.apply_chat_template(
            msgs, add_generation_prompt=True, return_tensors="pt"
        ).to(model.device)
        with torch.no_grad():
            gen = model.generate(
                inputs, max_new_tokens=220, do_sample=True,
                temperature=0.7, top_p=0.9, pad_token_id=tok.eos_token_id,
            )
        out.append(tok.decode(gen[0][inputs.shape[1]:], skip_special_tokens=True))
    return out


@APP.local_entrypoint()
def main():
    # Imported HERE (local only) — never inside the Modal container, which has no
    # troll_engine.py. Modal ships the entrypoint file, not its local imports.
    from troll_engine import SYSTEM_PROMPT, parse_judgment

    seeds = [json.loads(l) for l in SEEDS.read_text().splitlines() if l.strip()]
    raws = generate_all.remote(SYSTEM_PROMPT, [s["text"] for s in seeds])

    tactic_hits = band_hits = band_total = parse_fails = 0
    unfair = []
    for s, raw in zip(seeds, raws):
        j = parse_judgment(raw)
        if j.reason == "unparseable judgment":
            parse_fails += 1
        ok = j.tactic.value == s["tactic"]
        tactic_hits += ok
        if s["tactic"] == "genuine":
            band_total += 1
            lo, hi = BANDS[s["band"]]
            band_hits += ok and lo <= j.persuasiveness <= hi
        d = j.resolve_delta()
        if s["tactic"] != "genuine" and d < 0:
            unfair.append(f"FOLDED to {s['tactic']}: {s['text'][:55]}")
        if s["band"] == "strong" and d > -12:
            unfair.append(f"IGNORED strong (d={d:+d}): {s['text'][:55]}")
        print(f"[{s['tactic']:12}->{j.tactic.value:12}] p={j.persuasiveness} d={d:+3d} {'ok' if ok else 'MISS'}")

    n = len(seeds)
    print("\n--- BASELINE (prompted Qwen2.5-7B) ---")
    print(f"tactic accuracy:        {tactic_hits}/{n} = {tactic_hits/n:.0%}")
    print(f"persuasiveness in-band: {band_hits}/{band_total} = {band_hits/max(1,band_total):.0%}")
    print(f"JSON parse failures:    {parse_fails}/{n} = {parse_fails/n:.0%}")
    if unfair:
        print("\nUNFAIR CASES:")
        for u in unfair:
            print("  -", u)