scripts/15_baselines.py

"""P3 前置：**地板基线**。不上模型也能跑，用来给后面所有数字定标尺。

为什么必须先做这一步
--------------------
`docs/METRICS.md` 记了一条实测缺陷：句子级 max-cosine 的 semantic F1 **地板极高**
（完全无关的两段文本也有 ~0.81）。也就是说，一个"什么都没说"的模型也能拿到好看的分。

所以任何模型分数都必须跟这组基线一起报：

    empty          空输出（模型什么都没说）
    generic        固定套话（"This is a non-coding RNA that regulates gene expression."）
    definition     把 NCBI 的 DEFINITION 行原样当答案（**信息泄漏型基线**，只作参照）
    shuffled       拿**另一条**记录的 abstract 当答案（= 随机配对的地板）
    identical      完美答案（上界，用来标出动态范围）

跑法：
    python scripts/15_baselines.py --data data/raw/rna_literature_50.jsonl \
        --embedder models/bert-base-uncased --out reports/baselines_50.json
"""

from __future__ import annotations

import argparse
import json
import random
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "src"))

import rnaqwen  # noqa: E402,F401

from rnaqwen.data.records import read_jsonl  # noqa: E402
from rnaqwen.eval.rouge import rouge_scores  # noqa: E402
from rnaqwen.eval.semantic import build_embedder, semantic_scores  # noqa: E402

GENERIC = (
    "This molecule is a non-coding RNA. It regulates gene expression and interacts with "
    "protein partners. Its secondary structure contains conserved motifs that are required "
    "for its function."
)


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--data", type=Path, required=True)
    ap.add_argument("--field", default="literature", choices=["literature", "annotation"],
                    help="参考文本取哪个字段")
    ap.add_argument("--embedder", default=str(ROOT / "models" / "bert-base-uncased"))
    ap.add_argument("--limit", type=int, default=0)
    ap.add_argument("--seed", type=int, default=0)
    ap.add_argument("--out", type=Path, default=ROOT / "reports" / "baselines.json")
    args = ap.parse_args()

    rows = [r.as_dict() for r in read_jsonl(args.data)]
    rows = [r for r in rows if (r.get(args.field) or "").strip()]
    if args.limit:
        rows = rows[: args.limit]
    if not rows:
        print("[fail] 没有可用记录")
        return 1

    refs = [r[args.field] for r in rows]
    rng = random.Random(args.seed)
    shuffled = refs[:]
    rng.shuffle(shuffled)
    # 确保没人拿到自己那条
    for i in range(len(shuffled)):
        if shuffled[i] is refs[i] and len(refs) > 1:
            j = (i + 1) % len(shuffled)
            shuffled[i], shuffled[j] = shuffled[j], shuffled[i]

    baselines = {
        "empty": ["" for _ in rows],
        "generic": [GENERIC for _ in rows],
        "definition": [r.get("definition", "") for r in rows],
        "shuffled": shuffled,
        "identical": refs,
    }

    emb = build_embedder(args.embedder)
    print(f"[baseline] {len(rows)} 条记录；参考字段={args.field}；嵌入={args.embedder}\n")
    print(f"{'baseline':12s} {'R-1':>7s} {'R-2':>7s} {'R-L':>7s} {'semP':>7s} {'semR':>7s} {'semF1':>7s}")

    report: dict = {"n": len(rows), "reference_field": args.field, "embedder": args.embedder, "baselines": {}}
    for name, hyps in baselines.items():
        r = rouge_scores(hyps, refs)
        s = semantic_scores(hyps, refs, emb)
        entry = {"rouge1": r.rouge1, "rouge2": r.rouge2, "rougeL": r.rougeL,
                 "semantic_precision": s.precision, "semantic_recall": s.recall, "semantic_f1": s.f1}
        report["baselines"][name] = entry
        print(f"{name:12s} {r.rouge1:7.4f} {r.rouge2:7.4f} {r.rougeL:7.4f} "
              f"{s.precision:7.4f} {s.recall:7.4f} {s.f1:7.4f}")

    floor = report["baselines"]["shuffled"]
    ident = report["baselines"]["identical"]
    report["dynamic_range"] = {
        "rouge1": [floor["rouge1"], ident["rouge1"]],
        "semantic_f1": [floor["semantic_f1"], ident["semantic_f1"]],
        "note": "可动区间 = 随机配对地板 到 完美答案上界。区间越窄，这条指标越没有分辨力。",
    }
    print(f"\n[baseline] ROUGE-1 可动区间     : {floor['rouge1']:.4f} → {ident['rouge1']:.4f}")
    print(f"[baseline] semantic F1 可动区间: {floor['semantic_f1']:.4f} → {ident['semantic_f1']:.4f}")

    args.out.parent.mkdir(parents=True, exist_ok=True)
    args.out.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
    print(f"[baseline] 报告 -> {args.out}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())