| from __future__ import annotations |
|
|
| import json |
| from pathlib import Path |
| from typing import Any |
|
|
|
|
| def read_json(path: Path) -> dict[str, Any]: |
| return json.loads(path.read_text()) if path.exists() else {} |
|
|
|
|
| def read_jsonl(path: Path) -> list[dict[str, Any]]: |
| if not path.exists(): |
| return [] |
| return [json.loads(line) for line in path.read_text().splitlines() if line.strip()] |
|
|
|
|
| def usable_clean_rate(artifacts: list[dict[str, Any]]) -> float: |
| if not artifacts: |
| return 0.0 |
| usable = 0 |
| for artifact in artifacts: |
| clean = bool(artifact.get("scanner_clean", False)) |
| schema_valid = bool(artifact.get("schema_valid", False)) |
| parse_ok = bool(artifact.get("parse_ok", False)) |
| quality_issues = artifact.get("quality_issues") or [] |
| safe = not bool(artifact.get("unsafe_action", False)) |
| if clean and schema_valid and parse_ok and not quality_issues and safe: |
| usable += 1 |
| return usable / len(artifacts) |
|
|
|
|
| def report_row(report: dict[str, Any], artifacts: list[dict[str, Any]] | None = None) -> dict[str, Any]: |
| artifacts = artifacts or [] |
| return { |
| "system": report.get("system"), |
| "n": report.get("n"), |
| "scope": report.get("evaluation_scope", "full_benchmark"), |
| "evidence_level": report.get("evidence_level"), |
| "scanner_clean_rate": report.get("scanner_clean_rate"), |
| "usable_clean_rate": usable_clean_rate(artifacts) if artifacts else None, |
| "targeted_policy_resolution_rate": report.get("targeted_policy_resolution_rate"), |
| "parse_validity_rate": report.get("parse_validity_rate"), |
| "pydantic_schema_validity_rate": report.get("pydantic_schema_validity_rate"), |
| "unsafe_action_rate": report.get("unsafe_action_rate"), |
| "median_latency_s": report.get("median_latency_s"), |
| "failure_counts": report.get("failure_counts", {}), |
| } |
|
|
|
|
| def build_submission_summary(eval_dir: Path) -> dict[str, Any]: |
| report_b1 = read_json(eval_dir / "report_b1.json") |
| report_b3 = read_json(eval_dir / "report_b3.json") |
| report_b4 = read_json(eval_dir / "report_b4.json") or read_json(eval_dir / "report_b4.partial.json") |
| report_b5 = read_json(eval_dir / "report_b5.json") or read_json(eval_dir / "report_b5.partial.json") |
| report_final = read_json(eval_dir / "report_final.json") |
|
|
| b1_artifacts = read_jsonl(eval_dir / "per_example_b1.jsonl") |
| b3_artifacts = read_jsonl(eval_dir / "per_example_b3.jsonl") |
| b4_artifacts = read_jsonl(eval_dir / "per_example_b4.jsonl") or read_jsonl(eval_dir / "per_example_b4.partial.jsonl") |
| b5_artifacts = read_jsonl(eval_dir / "per_example_b5.jsonl") or read_jsonl(eval_dir / "per_example_b5.partial.jsonl") |
|
|
| systems = [ |
| report_row(report_b1, b1_artifacts), |
| report_row(report_b3, b3_artifacts), |
| report_row(report_b4, b4_artifacts), |
| report_row(report_b5, b5_artifacts), |
| ] |
|
|
| safety_report = read_json(eval_dir / "safety_report.json") |
| training_report = read_json(eval_dir / "training_report.json") |
|
|
| return { |
| "chosen_system": report_final.get("chosen_system", "B3_zero_shot_rag"), |
| "recommended_claim": "scanner-verified IaC remediation prototype with deterministic safety and validation gates", |
| "systems": systems, |
| "safety": safety_report, |
| "training": training_report, |
| "notes": [ |
| "B3 is the most defensible final benchmark claim.", |
| "B5 shows better partial scanner-clean and targeted-resolution rates, but it is partial and requires human review.", |
| "Fine-tuning is infrastructure evidence only unless a full benchmark shows uplift over B3.", |
| ], |
| } |
|
|
