Spaces:
Sleeping
Sleeping
| """Run the current pipeline on each real draft and tabulate its predicted | |
| similarity index / AI score against the Turnitin ground truth. | |
| Usage: | |
| python scripts/eval_vs_turnitin.py [--depth standard|deep] [--only N] [--names a,b] | |
| Writes data/eval_vs_turnitin.json and prints a comparison table. | |
| """ | |
| import argparse, json, os, sys, time | |
| ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| sys.path.insert(0, ROOT) | |
| from plagdetect.webpipeline import analyze_document # noqa: E402 | |
| DSET = os.path.join(ROOT, "DATASET FOR training of turnitin") | |
| GT = os.path.join(ROOT, "data", "turnitin_groundtruth.json") | |
| OUT = os.path.join(ROOT, "data", "eval_vs_turnitin.json") | |
| def truth_ai(rec): | |
| a = (rec.get("ai") or {}).get("ai_pct") | |
| return a # int, '*' (=<20 suppressed), or None | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--depth", default="standard") | |
| ap.add_argument("--only", type=int, default=0) | |
| ap.add_argument("--names", default="") | |
| args = ap.parse_args() | |
| gt = json.load(open(GT, encoding="utf-8")) | |
| if args.names: | |
| want = set(args.names.split(",")) | |
| gt = [r for r in gt if any(w in (r["src_file"] or "") for w in want)] | |
| if args.only: | |
| gt = gt[:args.only] | |
| results = [] | |
| for rec in gt: | |
| draft = rec.get("draft") | |
| if not draft: | |
| continue | |
| path = os.path.join(DSET, draft) | |
| sim_t = (rec.get("similarity") or {}).get("overall") | |
| ai_t = truth_ai(rec) | |
| t0 = time.time() | |
| try: | |
| r = analyze_document(path, depth=args.depth) | |
| pred = {"sim_index": r["similarity_index"], | |
| "raw_verbatim": r.get("raw_verbatim_index"), | |
| "paraphrase": r.get("paraphrase_index"), | |
| "self_excluded": len(r.get("self_matches_excluded") or []), | |
| "ai_score": r["ai_score"], "verdict": r["verdict"]} | |
| err = None | |
| except Exception as e: | |
| pred = None | |
| err = str(e) | |
| dt = round(time.time() - t0, 1) | |
| row = {"draft": draft, "sim_truth": sim_t, "ai_truth": ai_t, | |
| "pred": pred, "err": err, "secs": dt} | |
| results.append(row) | |
| json.dump(results, open(OUT, "w", encoding="utf-8"), indent=2) | |
| p = pred or {} | |
| print(f"{draft[:30]:31s} simT={str(sim_t):>4s} simP={str(p.get('sim_index')):>5s}" | |
| f" aiT={str(ai_t):>3s} aiP={str(p.get('ai_score')):>5s}" | |
| f" {p.get('verdict','ERR')} {dt}s") | |
| if err: | |
| print(" ERROR:", err) | |
| print(f"\nwrote {OUT}") | |
| if __name__ == "__main__": | |
| main() | |