"""Score DAIGT essays (real human + real multi-LLM AI text) through the 7-detector ensemble to produce IN-DOMAIN calibration rows — the humanised-AI positives the HC3-only meta never had (roadmap U1, the Ollama alternative: download real LLM-written essays instead of generating them). DAIGT: thedrcat/daigt-v2-train-dataset — columns text, label (0=human, 1=AI), source (model). We sample a balanced, length-filtered subset so every essay fires all seven detectors, then score. Output: data/calibration/daigt_scored.jsonl (schema: {y, x:[7], src}) Run: python scripts/score_daigt.py [n_per_class] (default 250) """ import glob import json import os import random import sys import time ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, ROOT) from plagdetect.aidetect import FEATURE_ORDER, detect_ai # noqa: E402 from plagdetect.normalize import deobfuscate # noqa: E402 from plagdetect.textutils import sentences # noqa: E402 CSV = glob.glob(os.path.join( os.path.expanduser("~"), ".cache", "kagglehub", "datasets", "thedrcat", "daigt-v2-train-dataset", "versions", "*", "*.csv")) OUT = os.path.join(ROOT, "data", "calibration", "daigt_scored.jsonl") MIN_CHARS, MIN_SENTS = 700, 8 def load_balanced(n_per_class, seed=13): import pandas as pd df = pd.read_csv(sorted(CSV)[-1], usecols=["text", "label"]) rng = random.Random(seed) pools = {0: [], 1: []} idx = list(range(len(df))) rng.shuffle(idx) for i in idx: lab = int(df.iloc[i]["label"]) if lab not in (0, 1) or len(pools[lab]) >= n_per_class: continue t = str(df.iloc[i]["text"] or "").strip() if len(t) >= MIN_CHARS and len(sentences(t)) >= MIN_SENTS: pools[lab].append(t) if len(pools[0]) >= n_per_class and len(pools[1]) >= n_per_class: break return pools[0], pools[1] def score_all(texts, label): rows, t0 = [], time.time() for i, t in enumerate(texts): t = deobfuscate(t)[0] try: r = detect_ai(t) except Exception as exc: print(" [skip]", exc); continue det = {d["name"]: d["score"] for d in r["detectors"]} if not all(k in det for k in FEATURE_ORDER): continue rows.append({"y": label, "x": [det[k] for k in FEATURE_ORDER], "src": "daigt"}) if (i + 1) % 25 == 0: rate = (i + 1) / (time.time() - t0) print(f" label={label}: {i+1}/{len(texts)} " f"({rate:.2f}/s, eta {(len(texts)-i-1)/max(rate,1e-9):.0f}s)", flush=True) return rows def main(n_per_class=250): if not CSV: print("DAIGT csv not found — download via kagglehub first."); return print("loading + length-filtering DAIGT...") human, ai = load_balanced(n_per_class) print(f"human={len(human)} ai={len(ai)} — scoring through 7 detectors") rows = score_all(human, 0) + score_all(ai, 1) with open(OUT, "w", encoding="utf-8") as f: for r in rows: f.write(json.dumps(r) + "\n") print(f"saved {len(rows)} DAIGT rows -> {OUT}") if __name__ == "__main__": main(int(sys.argv[1]) if len(sys.argv) > 1 else 250)