Spaces:
Running
Running
File size: 2,537 Bytes
d70361b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 | """
Record Delhi baseline metrics (Day 2 deliverable) for default settings.
Runs each primary method separately at sensitivity 0.5 (one process at a time
to avoid RAM spikes) and writes runs/delhi_baseline/metrics.json.
"""
from __future__ import annotations
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
OUT = ROOT / "runs" / "delhi_baseline"
MANIFEST = ROOT / "docs" / "delhi_eval" / "manifest.json"
METHODS = [
"Feature-Based",
"KPCA (Unsupervised)",
"Hybrid Approach",
"AI-Based Deep Learning",
"Hybrid AI",
]
def _run_method(method: str) -> list[dict]:
OUT.mkdir(parents=True, exist_ok=True)
report = OUT / f"report_{method.replace(' ', '_')}.json"
cmd = [
sys.executable,
str(ROOT / "scripts" / "compare_methods.py"),
"--manifest", str(MANIFEST),
"--methods", method,
"--sensitivities", "0.5",
"--out", str(OUT),
"--report-only",
]
print(f"\n=== Baseline: {method} ===")
subprocess.run(cmd, check=True, cwd=ROOT)
manifest_report = OUT / "manifest_report.json"
if manifest_report.is_file():
rows = json.loads(manifest_report.read_text(encoding="utf-8"))
report.write_text(json.dumps(rows, indent=2), encoding="utf-8")
return rows
return []
def main():
all_rows: list[dict] = []
for method in METHODS:
all_rows.extend(_run_method(method))
labeled = [r for r in all_rows if "f1" in r]
summary = {
"methods": METHODS,
"sensitivity": 0.5,
"n_pairs": len({r["pair_id"] for r in all_rows}),
"n_labeled_rows": len(labeled),
"per_method_mean_f1": {},
"per_method_mean_iou": {},
}
by_method: dict[str, list[dict]] = {}
for r in labeled:
by_method.setdefault(r["method"], []).append(r)
for m, rows in by_method.items():
summary["per_method_mean_f1"][m] = round(
sum(r["f1"] for r in rows) / len(rows), 4)
summary["per_method_mean_iou"][m] = round(
sum(r["iou"] for r in rows) / len(rows), 4)
(OUT / "metrics.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
(OUT / "manifest_report.json").write_text(json.dumps(all_rows, indent=2), encoding="utf-8")
print(f"\nWrote {OUT / 'metrics.json'}")
for m, f1 in summary["per_method_mean_f1"].items():
print(f" {m}: mean F1={f1} IoU={summary['per_method_mean_iou'][m]}")
if __name__ == "__main__":
main()
|