scripts/evaluate_damage_scoring.py

"""Evaluate the transparent vehicle damage scoring heuristic."""

from __future__ import annotations

import json
import sys
from pathlib import Path

PROJECT_ROOT = Path(__file__).resolve().parents[1]
sys.path.append(str(PROJECT_ROOT))

from app.damage_model import DAMAGE_WEIGHTS, analyze_manual_damage


REPORT_PATH = PROJECT_ROOT / "reports/damage_scoring_evaluation.md"
JSON_PATH = PROJECT_ROOT / "reports/damage_scoring_examples.json"


EVALUATION_CASES = [
    {
        "case_id": "no_damage",
        "base_price_eur": 15_000,
        "labels": [],
        "expected_interpretation": "No visible damage, no price adjustment.",
    },
    {
        "case_id": "minor_scratch",
        "base_price_eur": 15_000,
        "labels": ["scratch"],
        "expected_interpretation": "Minor visible damage.",
    },
    {
        "case_id": "scratch_and_dent",
        "base_price_eur": 15_000,
        "labels": ["scratch", "dent"],
        "expected_interpretation": "Small to moderate exterior damage.",
    },
    {
        "case_id": "lamp_and_glass_damage",
        "base_price_eur": 20_000,
        "labels": ["lamp broken", "glass shatter"],
        "expected_interpretation": "More serious visible damage with stronger adjustment.",
    },
    {
        "case_id": "multiple_severe_damages",
        "base_price_eur": 30_000,
        "labels": ["dent", "crack", "glass shatter", "lamp broken"],
        "expected_interpretation": "Severe damage capped by maximum damage score.",
    },
]


def evaluate_cases() -> list[dict]:
    """Run deterministic scoring examples."""
    rows = []
    for case in EVALUATION_CASES:
        analysis = analyze_manual_damage(
            labels=case["labels"],
            base_price_eur=case["base_price_eur"],
        )
        rows.append(
            {
                "case_id": case["case_id"],
                "base_price_eur": case["base_price_eur"],
                "labels": case["labels"],
                "damage_score": analysis.damage_score,
                "recommended_discount_eur": analysis.recommended_discount_eur,
                "adjusted_price_eur": analysis.adjusted_price_eur,
                "expected_interpretation": case["expected_interpretation"],
            }
        )
    return rows


def write_report(rows: list[dict]) -> None:
    """Write markdown and JSON evaluation artifacts."""
    REPORT_PATH.parent.mkdir(parents=True, exist_ok=True)
    JSON_PATH.write_text(json.dumps(rows, indent=2), encoding="utf-8")

    lines = [
        "# Damage Scoring Evaluation",
        "",
        "The current computer vision integration converts detected damage labels into a transparent damage score. The score is a heuristic price adjustment, not a certified repair-cost estimate.",
        "",
        "## Damage Weights",
        "",
        "| Damage label | Weight |",
        "|---|---:|",
    ]
    for label, weight in DAMAGE_WEIGHTS.items():
        if weight > 0:
            lines.append(f"| {label} | {weight:.2f} |")

    lines.extend(
        [
            "",
            "## Deterministic Evaluation Cases",
            "",
            "| Case | Labels | Base price | Damage score | Discount | Adjusted price | Interpretation |",
            "|---|---|---:|---:|---:|---:|---|",
        ]
    )
    for row in rows:
        labels = ", ".join(row["labels"]) if row["labels"] else "none"
        lines.append(
            f"| {row['case_id']} | {labels} | EUR {row['base_price_eur']:,.0f} | "
            f"{row['damage_score']:.3f} | EUR {row['recommended_discount_eur']:,.0f} | "
            f"EUR {row['adjusted_price_eur']:,.0f} | {row['expected_interpretation']} |"
        )

    lines.extend(
        [
            "",
            "## Interpretation",
            "",
            "The scoring logic behaves monotonically: more severe and more numerous damage labels produce a larger discount. The score is capped at 0.35 to avoid unrealistic automatic reductions. In the final app, a user-facing note must explain that this adjustment is based only on visible image evidence and should not replace a professional inspection.",
        ]
    )
    REPORT_PATH.write_text("\n".join(lines), encoding="utf-8")


def main() -> None:
    rows = evaluate_cases()
    write_report(rows)
    print(f"Wrote {REPORT_PATH}")
    print(f"Wrote {JSON_PATH}")


if __name__ == "__main__":
    main()