scripts/eval_similarity.py

"""Honest test of the sentence-similarity engine on REAL text neither the
synthetic Siamese encoder nor MiniLM was trained on.

Two evidence sources:
  A) PLOS real academic corpus (data/oa_corpus, 40 real 2024 papers). We build
     plagiarism-style pairs in the ACTUAL domain:
       positive (reuse, SHOULD match): a real source sentence + an attacked copy
         (synonymize / word-dropout — the very transforms the pipeline must catch;
         note this FAVOURS the Siamese, which trained on these exact transforms).
       negative (SHOULD NOT match): two unrelated real sentences.
  B) PAWS (real human paraphrase benchmark, adversarial: high word-overlap pairs
     that are NOT paraphrases) — the gold standard for semantic-vs-surface. The
     char-trigram Siamese is purely lexical, so PAWS exposes it directly.

Metric: ROC-AUC of the similarity score at separating positives from negatives,
for the Siamese encoder vs MiniLM. Higher = the engine actually understands
reuse rather than memorising the synthetic generator.
"""
import os
# Windows: onnxruntime and sklearn/scipy ship their own OpenMP runtimes; loading
# scipy first makes onnxruntime's DLL init fail. Allow the duplicate and warm up
# onnxruntime BEFORE sklearn is imported.
os.environ.setdefault("KMP_DUPLICATE_LIB_OK", "TRUE")
import json
import sys
import urllib.request

import numpy as np
import onnxruntime  # noqa: F401  (import first to win the OpenMP race)

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, ROOT)
from plagdetect import semantic                       # noqa: E402
semantic._load()                                       # warm up before sklearn
from plagdetect.siamese import SiameseEncoder         # noqa: E402
from plagdetect.textutils import sentences, synonymize  # noqa: E402

from sklearn.metrics import roc_auc_score             # noqa: E402

PLOS = os.path.join(ROOT, "data", "oa_corpus", "plos_complex_systems.jsonl")
PAWS_URL = ("https://raw.githubusercontent.com/google-research-datasets/"
            "paws/master/paws_wiki_labeled_final_dev_head.tsv")  # may 404; we fallback


def _word_dropout(text, rng, p=0.15):
    w = text.split()
    kept = [x for x in w if rng.rand() > p]
    return " ".join(kept if len(kept) > 3 else w)


def _embed_lookup(pairs, embed_fn):
    """Embed every unique sentence ONCE, return {sentence: vector}."""
    uniq = list(dict.fromkeys([s for ab in pairs for s in ab]))
    V = embed_fn(uniq)
    return {s: V[i] for i, s in enumerate(uniq)}


def _pair_scores(pos, neg, embed_fn):
    look = _embed_lookup(pos + neg, embed_fn)
    sc = [float(look[a] @ look[b]) for a, b in pos] + \
         [float(look[a] @ look[b]) for a, b in neg]
    return sc


def build_plos_pairs(n=200, seed=1):
    rng = np.random.RandomState(seed)
    rows = [json.loads(l) for l in open(PLOS, encoding="utf-8")]
    sents = []
    for r in rows:
        for fld in ("abstract", "fulltext"):
            for s in sentences(r.get(fld, "")):
                if 8 <= len(s.split()) <= 40:
                    sents.append(s)
    sents = list(dict.fromkeys(sents))           # dedupe, keep order
    rng.shuffle(sents)
    pos, neg = [], []
    for i in range(min(n, len(sents) - 1)):
        s = sents[i]
        atk = synonymize(s, rng, p=0.6) if rng.rand() < 0.5 else _word_dropout(s, rng)
        pos.append((s, atk))
        j = rng.randint(len(sents))
        while sents[j][:30] == s[:30]:
            j = rng.randint(len(sents))
        neg.append((s, sents[j]))
    return pos, neg


def try_load_paws(limit=400):
    """Best-effort fetch of a small real paraphrase benchmark. Returns
    (pos_pairs, neg_pairs) or (None, None) if unavailable."""
    candidates = [
        "https://raw.githubusercontent.com/IBM/quality-controlled-paraphrase-generation/main/data/mrpc/msr_paraphrase_test.txt",
        "https://raw.githubusercontent.com/wasiahmad/paraphrase_identification/master/dataset/msr-paraphrase-corpus/msr_paraphrase_test.txt",
    ]
    for url in candidates:
        try:
            req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
            raw = urllib.request.urlopen(req, timeout=15).read().decode("utf-8", "ignore")
        except Exception:
            continue
        pos, neg = [], []
        for ln in raw.splitlines()[1:]:               # MRPC: Quality \t id \t id \t s1 \t s2
            parts = ln.split("\t")
            if len(parts) < 5:
                continue
            lab, s1, s2 = parts[0].strip(), parts[3].strip(), parts[4].strip()
            (pos if lab == "1" else neg).append((s1, s2))
            if len(pos) + len(neg) >= limit:
                break
        if pos and neg:
            return pos[:limit // 2], neg[:limit // 2], url.split("/")[-1]
    return None, None, None


def auc_on(pos, neg, scores):
    y = [1] * len(pos) + [0] * len(neg)
    return (roc_auc_score(y, scores), float(np.mean(scores[:len(pos)])),
            float(np.mean(scores[len(pos):])))


def report(name, pos, neg, enc):
    print(f"\n=== {name}  (pos={len(pos)} neg={len(neg)}) ===")
    s_sc = _pair_scores(pos, neg, enc.embed_texts)
    m_raw = _pair_scores(pos, neg, semantic.embed)
    # map MiniLM cosine onto the siamese/lexical threshold scale, then ENSEMBLE
    # by max: verbatim reuse -> siamese high, paraphrase -> MiniLM high.
    m_scaled = semantic.to_lexical_scale(m_raw).tolist()
    ens = [max(s, m) for s, m in zip(s_sc, m_scaled)]
    a_s, p_s, n_s = auc_on(pos, neg, s_sc)
    a_m, p_m, n_m = auc_on(pos, neg, m_raw)
    a_e, p_e, n_e = auc_on(pos, neg, ens)
    print(f"  Siamese (synthetic char-trigram) : AUC={a_s:.3f}  "
          f"pos_sim={p_s:.3f} neg_sim={n_s:.3f}")
    print(f"  MiniLM  (pretrained, real)       : AUC={a_m:.3f}  "
          f"pos_sim={p_m:.3f} neg_sim={n_m:.3f}")
    print(f"  ENSEMBLE max(siamese, MiniLM)    : AUC={a_e:.3f}  "
          f"pos_sim={p_e:.3f} neg_sim={n_e:.3f}  <-- proposed")
    return a_s, a_m, a_e


def main():
    enc = SiameseEncoder.load(os.path.join(ROOT, "models", "siamese.npz"))
    if not semantic.available():
        print("MiniLM unavailable — cannot run."); return

    pos, neg = build_plos_pairs()
    s1, m1, e1 = report("A) PLOS real academic reuse (favours Siamese: its train transforms)",
                        pos, neg, enc)

    ppos, pneg, src = try_load_paws()
    if ppos:
        s2, m2, e2 = report(f"B) {src} real human paraphrase benchmark (semantic-vs-surface)",
                            ppos, pneg, enc)
    else:
        print("\n=== B) real paraphrase benchmark: download unavailable, skipped ===")
        s2 = m2 = e2 = None

    print("\n" + "=" * 70)
    print("VERDICT (AUC: Siamese | MiniLM | Ensemble-max)")
    print(f"  PLOS reuse domain : {s1:.3f} | {m1:.3f} | {e1:.3f}")
    if s2 is not None:
        print(f"  Real paraphrase   : {s2:.3f} | {m2:.3f} | {e2:.3f}")
    print("  Ship the ensemble if it is >= both on each row (no regression, adds")
    print("  MiniLM's real semantic channel without losing verbatim sensitivity).")


if __name__ == "__main__":
    main()