File size: 2,863 Bytes
1168cd6 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 |
# training/parser_eval.py
# ------------------------------------------------------------
# Parser Evaluation (Stage 10A)
#
# This version ONLY evaluates:
# - Rule parser
# - Extended parser
#
# The LLM parser is intentionally disabled at this stage
# because alias maps and schema are not trained yet.
#
# This makes Stage 10A FAST and stable (< 3 seconds).
# ------------------------------------------------------------
import json
import os
from typing import Dict, Any
from engine.parser_rules import parse_text_rules
from engine.parser_ext import parse_text_extended
# Path to the gold tests
GOLD_PATH = "training/gold_tests.json"
def evaluate_single_test(test: Dict[str, Any]) -> Dict[str, Any]:
"""
Evaluate one gold test with rules + extended parsers.
"""
text = test.get("input", "")
expected = test.get("expected", {})
# Run deterministic parsers
rule_out = parse_text_rules(text).get("parsed_fields", {})
ext_out = parse_text_extended(text).get("parsed_fields", {})
# Merge rule + extended (extended overwrites rules)
merged = dict(rule_out)
for k, v in ext_out.items():
if v != "Unknown":
merged[k] = v
total = len(expected)
correct = 0
wrong = {}
for field, exp_val in expected.items():
got = merged.get(field, "Unknown")
if got.lower() == exp_val.lower():
correct += 0 if exp_val == "Unknown" else 1 # Unknown is neutral
else:
wrong[field] = {"expected": exp_val, "got": got}
return {
"correct": correct,
"total": total,
"accuracy": correct / total if total else 0,
"wrong": wrong,
"merged": merged,
}
def run_parser_eval(mode: str = "rules_extended") -> Dict[str, Any]:
"""
Evaluate ALL gold tests using rules + extended parsing only.
"""
if not os.path.exists(GOLD_PATH):
return {"error": f"Gold test file not found at {GOLD_PATH}"}
with open(GOLD_PATH, "r", encoding="utf-8") as f:
gold = json.load(f)
results = []
wrong_cases = []
total_correct = 0
total_fields = 0
for test in gold:
out = evaluate_single_test(test)
results.append(out)
total_correct += out["correct"]
total_fields += out["total"]
if out["wrong"]:
wrong_cases.append({
"name": test.get("name", "Unnamed"),
"wrong": out["wrong"],
"parsed": out["merged"],
"expected": test.get("expected", {})
})
summary = {
"mode": "rules+extended",
"tests": len(gold),
"total_correct": total_correct,
"total_fields": total_fields,
"overall_accuracy": total_correct / total_fields if total_fields else 0,
"wrong_cases": wrong_cases,
}
return summary
|