| """Score the brain against an eval set: did it pick the right items + quantities? | |
| Used to measure brain accuracy and as the dataset for prompt optimization (GEPA). | |
| """ | |
| import json | |
| def load_cases(path: str) -> list[dict]: | |
| with open(path) as f: | |
| return json.load(f)["cases"] | |
| def _by_desc(items: list[dict]) -> dict[str, float]: | |
| return {i["description"].strip().lower(): float(i.get("quantity", 1)) for i in items} | |
| def score_case(produced: list[dict], expected: list[dict]) -> dict: | |
| """Item-set F1 + quantity accuracy over matched items.""" | |
| p = _by_desc(produced) | |
| e = _by_desc(expected) | |
| matched = set(p) & set(e) | |
| precision = len(matched) / len(p) if p else 0.0 | |
| recall = len(matched) / len(e) if e else 0.0 | |
| f1 = (2 * precision * recall / (precision + recall)) if (precision + recall) else 0.0 | |
| if matched: | |
| correct_qty = sum(1 for k in matched if p[k] == e[k]) | |
| qty_accuracy = correct_qty / len(matched) | |
| else: | |
| qty_accuracy = 0.0 | |
| return { | |
| "item_f1": round(f1, 3), | |
| "qty_accuracy": round(qty_accuracy, 3), | |
| "precision": round(precision, 3), | |
| "recall": round(recall, 3), | |
| } | |