|
|
|
|
|
import argparse, json, re |
|
|
from typing import List, Dict, Any |
|
|
|
|
|
def normalize(s: str) -> str: |
|
|
s = s.replace("```", " ") |
|
|
s = s.strip().lower() |
|
|
|
|
|
s = re.sub(r"\s+", " ", s) |
|
|
return s |
|
|
|
|
|
def main(): |
|
|
ap = argparse.ArgumentParser() |
|
|
ap.add_argument("--pred_path", type=str, required=True, help="eval 的输出 JSON") |
|
|
ap.add_argument("--out_path", type=str, default="./valid_clean/valid.json", help="评分明细输出 JSON") |
|
|
args = ap.parse_args() |
|
|
|
|
|
with open(args.pred_path, "r", encoding="utf-8") as f: |
|
|
preds: List[Dict[str, Any]] = json.load(f) |
|
|
|
|
|
rows = [] |
|
|
hit, total = 0, 0 |
|
|
for item in preds: |
|
|
gt = item.get("ground_truth", "") |
|
|
pred = item.get("model_output", "") |
|
|
|
|
|
if gt is None or gt == "": |
|
|
rows.append({ |
|
|
"id": item.get("id"), |
|
|
"match": None, |
|
|
"reason": "missing_ground_truth", |
|
|
"ground_truth": gt, |
|
|
"model_output": pred |
|
|
}) |
|
|
continue |
|
|
|
|
|
total += 1 |
|
|
ngt = normalize(gt) |
|
|
npred = normalize(pred) |
|
|
|
|
|
match = (npred in ngt) |
|
|
if match: |
|
|
hit += 1 |
|
|
|
|
|
rows.append({ |
|
|
"id": item.get("id"), |
|
|
"match": bool(match), |
|
|
"ground_truth": gt, |
|
|
"model_output": pred |
|
|
}) |
|
|
|
|
|
summary = { |
|
|
"total_with_gt": total, |
|
|
"matched": hit, |
|
|
"accuracy": (hit / total) if total > 0 else None |
|
|
} |
|
|
|
|
|
out = {"summary": summary, "details": rows} |
|
|
with open(args.out_path, "w", encoding="utf-8") as f: |
|
|
json.dump(out, f, ensure_ascii=False, indent=2) |
|
|
|
|
|
print(f"[SUMMARY] matched {hit}/{total} = {summary['accuracy']:.4f}" if total else "[SUMMARY] no GT") |
|
|
|
|
|
if __name__ == "__main__": |
|
|
main() |
|
|
|
