scripts/eval_ai_baseline.py

"""SMOKE TEST of the AI detector on a small hand-written labelled set.

CAVEAT: this set is NOT a trustworthy accuracy benchmark.
  * the "blatant AI" rows are phrase-stuffed caricatures whose GPT-2 perplexity
    is genuinely human-range, so a well-calibrated model scores several low;
  * the "subtle AI" rows were hand-written to look human, so "human" is arguably
    the correct call on them.
The headline metric is held-out-source AUC on real data
(scripts/diag_overfit.py / recalibrate_v3.py: ~0.96). Use this harness only to
eyeball the per-sample calibrated p_ai and to confirm humans are not falsely
accused — not to read a recall percentage off the caricatures.
"""
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from plagdetect import aidetect
from plagdetect.textutils import sentences
from data.ai_eval_samples import SAMPLES


def band_says_ai(band):
    return band in ("likely_ai", "mixed_or_uncertain")  # caught (not cleared)


def decided_ai_final(score):
    return score >= 45  # _band threshold for "not likely_human"


rows = []
for name, label, text in SAMPLES:
    r = aidetect.detect_ai(text)
    dets = {d["name"]: d["score"] for d in r["detectors"]}
    strongest = max(dets.values()) if dets else 0.0
    meta_p = r.get("conformal", {}).get("p_ai") if r.get("conformal") else None
    # what the META alone (100*p_ai) would have scored, before max-pool override
    meta_score = 100.0 * meta_p if meta_p is not None else None
    rows.append(dict(name=name, label=label, score=r["score"], band=r["band"],
                     fusion=r.get("fusion"), meta_p=meta_p, meta_score=meta_score,
                     strongest=strongest, nsent=len(sentences(text))))

# ---------------------------------------------------------------- per sample
print("=" * 92)
print(f"{'sample':26s} {'lbl':3s} {'final':6s} {'band':20s} "
      f"{'meta_p':7s} {'fusion':10s} {'maxdet':6s}")
print("-" * 92)
for x in rows:
    mp = f"{x['meta_p']:.3f}" if x['meta_p'] is not None else "  -  "
    flag = ""
    if x['label'] == 1 and not decided_ai_final(x['score']):
        flag = "  <== MISSED AI (R1 fail)"
    if x['label'] == 0 and decided_ai_final(x['score']):
        flag = "  <== false alarm"
    print(f"{x['name']:26s} {x['label']:<3d} {x['score']:6.1f} {x['band']:20s} "
          f"{mp:7s} {str(x['fusion']):10s} {x['strongest']:6.1f}{flag}")

# ------------------------------------------------------- final-band metrics
ai = [x for x in rows if x['label'] == 1]
hu = [x for x in rows if x['label'] == 0]
ai_caught = sum(decided_ai_final(x['score']) for x in ai)
hu_clean = sum(not decided_ai_final(x['score']) for x in hu)
print("\n" + "=" * 92)
print("FINAL DECISION (with R1 max-pool override active — what the user sees)")
print(f"  AI recall      : {ai_caught}/{len(ai)} caught  "
      f"({100*ai_caught/len(ai):.0f}%)   <- R1 wants ~100%")
print(f"  Human specificity: {hu_clean}/{len(hu)} correctly cleared "
      f"({100*hu_clean/len(hu):.0f}%)")
print(f"  AI missed (false negatives): {len(ai)-ai_caught}")
print(f"  Human false alarms         : {len(hu)-hu_clean}")

# ------------------------------------------------- META-ONLY (overfit probe)
THR = 50.0  # meta_score >= 50 == p_ai >= 0.5
ai_meta = [x for x in ai if x['meta_score'] is not None]
hu_meta = [x for x in hu if x['meta_score'] is not None]
ai_meta_caught = sum(x['meta_score'] >= THR for x in ai_meta)
hu_meta_clean = sum(x['meta_score'] < THR for x in hu_meta)
print("\n" + "-" * 92)
print("META CLASSIFIER ALONE (ignore the override — this is the trained model)")
print(f"  ran on {len(ai_meta)}/{len(ai)} AI and {len(hu_meta)}/{len(hu)} human "
      f"(rest hit hand-fusion fallback)")
if ai_meta:
    print(f"  AI recall (meta only): {ai_meta_caught}/{len(ai_meta)} "
          f"({100*ai_meta_caught/len(ai_meta):.0f}%)")
    print(f"  mean meta p_ai on AI  : "
          f"{sum(x['meta_p'] for x in ai_meta)/len(ai_meta):.3f}")
if hu_meta:
    print(f"  mean meta p_ai on human: "
          f"{sum(x['meta_p'] for x in hu_meta)/len(hu_meta):.3f}")
saved = [x for x in ai if decided_ai_final(x['score'])
         and (x['meta_score'] is None or x['meta_score'] < THR)]
print(f"\n  meta path now runs on {len(ai_meta)+len(hu_meta)}/{len(rows)} samples "
      f"(was 9/29 with the all-7 gate).")
print("  NOTE: low recall here is the caricature caveat above, NOT overfit — "
      "real held-out-source AUC is ~0.96 (scripts/diag_overfit.py).")