File size: 3,549 Bytes
dd18e84 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 | #!/usr/bin/env python3
"""评测双头模型的六种精度变体：二分类 / 7 类指标 + 与 fp32 预测一致率。
用法:
python scripts/eval_dual_variants.py --root .work/hf/v2_six \
--val data/processed_dual/val.jsonl --holdout data/processed_dual/holdout827.jsonl \
--out release/v2_variant_eval.json
"""
import argparse
import json
import sys
from pathlib import Path
import numpy as np
import torch
from transformers import AutoTokenizer
sys.path.insert(0, str(Path(__file__).parent))
from dual_head_model import load_dual_head_any # noqa: E402
from train_dual_head import evaluate, tokenize # noqa: E402
def run_variant(d, rows, split, dtype):
tok = AutoTokenizer.from_pretrained(d)
ids, am = tokenize(rows, tok, 384)
lengths = [len(x) for x in ids]
model = load_dual_head_any(d, device="cuda", dtype=dtype)
m, pb, pk = evaluate(model, rows, ids, am,
[r["label_bin"] for r in rows],
[r["label_kind"] for r in rows], lengths, 128, 384)
del model
torch.cuda.empty_cache()
return m, pb, pk
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--root", default=".work/hf/v2_six")
ap.add_argument("--val", default="data/processed_dual/val.jsonl")
ap.add_argument("--holdout", default="data/processed_dual/holdout827.jsonl")
ap.add_argument("--out", default="release/v2_variant_eval.json")
args = ap.parse_args()
root = Path(args.root)
splits = {
"val": [json.loads(l) for l in open(args.val, encoding="utf-8") if l.strip()],
"holdout827": [json.loads(l) for l in open(args.holdout, encoding="utf-8") if l.strip()],
}
variants = ["fp32", "fp16", "bf16", "fp8", "int8", "int4"]
results = {}
ref = {}
print(f"{'variant':<8}{'val bin':>9}{'val kind':>10}{'val agree':>11}"
f"{'hold bin':>10}{'hold kind':>11}{'hold agree':>12}{'size MB':>10}")
for v in variants:
d = root / v
dtype = torch.float32 if v == "fp32" else torch.bfloat16
results[v] = {"size_mb": round((d / "model.safetensors").stat().st_size / 2**20, 1)}
for split, rows in splits.items():
m, pb, pk = run_variant(d, rows, split, dtype)
entry = {k: m[k] for k in ("binary_acc", "ad_f1", "kind_acc", "kind_core_f1")}
if split == "val" and v == "fp32":
ref["val"] = (pb[:, 0] >= 0.5, pk.argmax(1))
if split == "holdout827" and v == "fp32":
ref["holdout827"] = (pb[:, 0] >= 0.5, pk.argmax(1))
if split in ref:
rp, rk = ref[split]
pred = pb[:, 0] >= 0.5
entry["agree_bin"] = round(float((pred == rp).mean()), 4)
kk = np.array([i for i, r in enumerate(rows) if r["label_kind"] >= 0])
entry["agree_kind"] = round(float((pk.argmax(1)[kk] == rk[kk]).mean()), 4)
results[v][split] = entry
rv, rh = results[v]["val"], results[v]["holdout827"]
print(f"{v:<8}{rv['binary_acc']*100:>8.2f}%{rv['kind_acc']*100:>9.2f}%"
f"{rv.get('agree_bin', 1)*100:>10.2f}%"
f"{rh['binary_acc']*100:>9.2f}%{rh['kind_acc']*100:>10.2f}%"
f"{rh.get('agree_bin', 1)*100:>11.2f}%{results[v]['size_mb']:>10.1f}")
out = Path(args.out)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
print("->", out.resolve())
if __name__ == "__main__":
main()
|