scripts/score_daigt.py

"""Score DAIGT essays (real human + real multi-LLM AI text) through the 7-detector
ensemble to produce IN-DOMAIN calibration rows — the humanised-AI positives the
HC3-only meta never had (roadmap U1, the Ollama alternative: download real
LLM-written essays instead of generating them).

DAIGT: thedrcat/daigt-v2-train-dataset — columns text, label (0=human, 1=AI),
source (model). We sample a balanced, length-filtered subset so every essay
fires all seven detectors, then score.

Output: data/calibration/daigt_scored.jsonl  (schema: {y, x:[7], src})
Run:    python scripts/score_daigt.py [n_per_class]   (default 250)
"""
import glob
import json
import os
import random
import sys
import time

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, ROOT)
from plagdetect.aidetect import FEATURE_ORDER, detect_ai  # noqa: E402
from plagdetect.normalize import deobfuscate  # noqa: E402
from plagdetect.textutils import sentences  # noqa: E402

CSV = glob.glob(os.path.join(
    os.path.expanduser("~"), ".cache", "kagglehub", "datasets", "thedrcat",
    "daigt-v2-train-dataset", "versions", "*", "*.csv"))
OUT = os.path.join(ROOT, "data", "calibration", "daigt_scored.jsonl")
MIN_CHARS, MIN_SENTS = 700, 8


def load_balanced(n_per_class, seed=13):
    import pandas as pd
    df = pd.read_csv(sorted(CSV)[-1], usecols=["text", "label"])
    rng = random.Random(seed)
    pools = {0: [], 1: []}
    idx = list(range(len(df)))
    rng.shuffle(idx)
    for i in idx:
        lab = int(df.iloc[i]["label"])
        if lab not in (0, 1) or len(pools[lab]) >= n_per_class:
            continue
        t = str(df.iloc[i]["text"] or "").strip()
        if len(t) >= MIN_CHARS and len(sentences(t)) >= MIN_SENTS:
            pools[lab].append(t)
        if len(pools[0]) >= n_per_class and len(pools[1]) >= n_per_class:
            break
    return pools[0], pools[1]


def score_all(texts, label):
    rows, t0 = [], time.time()
    for i, t in enumerate(texts):
        t = deobfuscate(t)[0]
        try:
            r = detect_ai(t)
        except Exception as exc:
            print("  [skip]", exc); continue
        det = {d["name"]: d["score"] for d in r["detectors"]}
        if not all(k in det for k in FEATURE_ORDER):
            continue
        rows.append({"y": label, "x": [det[k] for k in FEATURE_ORDER],
                     "src": "daigt"})
        if (i + 1) % 25 == 0:
            rate = (i + 1) / (time.time() - t0)
            print(f"  label={label}: {i+1}/{len(texts)} "
                  f"({rate:.2f}/s, eta {(len(texts)-i-1)/max(rate,1e-9):.0f}s)",
                  flush=True)
    return rows


def main(n_per_class=250):
    if not CSV:
        print("DAIGT csv not found — download via kagglehub first."); return
    print("loading + length-filtering DAIGT...")
    human, ai = load_balanced(n_per_class)
    print(f"human={len(human)} ai={len(ai)} — scoring through 7 detectors")
    rows = score_all(human, 0) + score_all(ai, 1)
    with open(OUT, "w", encoding="utf-8") as f:
        for r in rows:
            f.write(json.dumps(r) + "\n")
    print(f"saved {len(rows)} DAIGT rows -> {OUT}")


if __name__ == "__main__":
    main(int(sys.argv[1]) if len(sys.argv) > 1 else 250)