scripts/evaluate_cv_subset.py

"""Evaluate the optional local CV fallback on a small image subset.

This script is intentionally subset-based. It is meant for qualitative
computer-vision evidence without sending a large image dataset to OpenAI.
"""

from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path

PROJECT_ROOT = Path(__file__).resolve().parents[1]
sys.path.append(str(PROJECT_ROOT))

from app.damage_model import analyze_local_damage_image


DEFAULT_INPUT = PROJECT_ROOT / "data/samples/cv_subset"
REPORT_PATH = PROJECT_ROOT / "reports/cv_subset_evaluation.md"
JSON_PATH = PROJECT_ROOT / "reports/cv_subset_evaluation.json"


def collect_images(input_dir: Path, limit: int) -> list[Path]:
    """Collect a deterministic small subset of local image files."""
    extensions = {".jpg", ".jpeg", ".png", ".webp"}
    if not input_dir.exists():
        return []
    images = sorted(path for path in input_dir.iterdir() if path.suffix.lower() in extensions)
    return images[:limit]


def evaluate_subset(input_dir: Path, limit: int, base_price_eur: float) -> list[dict]:
    """Run the local fallback model on a small local image subset."""
    rows = []
    for image_path in collect_images(input_dir, limit):
        analysis = analyze_local_damage_image(
            image_path=image_path,
            base_price_eur=base_price_eur,
        )
        rows.append(
            {
                "image": image_path.name,
                "model": analysis.model_name,
                "labels": [item.label for item in analysis.detected_damages],
                "damage_score": analysis.damage_score,
                "discount_eur": analysis.recommended_discount_eur,
                "adjusted_price_eur": analysis.adjusted_price_eur,
                "notes": analysis.notes,
            }
        )
    return rows


def write_report(rows: list[dict], input_dir: Path, limit: int) -> None:
    """Write reproducible qualitative CV subset artifacts."""
    REPORT_PATH.parent.mkdir(parents=True, exist_ok=True)
    JSON_PATH.write_text(json.dumps(rows, indent=2), encoding="utf-8")
    try:
        display_input_dir = input_dir.resolve().relative_to(PROJECT_ROOT)
    except ValueError:
        display_input_dir = input_dir

    lines = [
        "# CV Subset Evaluation",
        "",
        "This evaluation uses only a small local subset of images and the locally trained CV baseline. It is designed to avoid sending large image datasets to OpenAI and to keep API usage controlled.",
        "",
        f"- Input folder: `{display_input_dir}`",
        f"- Maximum images: {limit}",
        f"- Evaluated images: {len(rows)}",
        "",
        "| Image | Model | Labels | Damage score | Discount | Adjusted price |",
        "|---|---|---|---:|---:|---:|",
    ]
    if not rows:
        lines.append("| none | not run | Add sample images to `data/samples/cv_subset/` | 0.000 | EUR 0 | EUR 0 |")
    for row in rows:
        labels = ", ".join(row["labels"]) if row["labels"] else "none"
        lines.append(
            f"| {row['image']} | {row['model']} | {labels} | "
            f"{row['damage_score']:.3f} | EUR {row['discount_eur']:,.0f} | "
            f"EUR {row['adjusted_price_eur']:,.0f} |"
        )

    lines.extend(
        [
            "",
            "## Interpretation",
            "",
            "The local classifier is an optional fallback and is not as expressive as the OpenAI Vision path used in the deployed app. The final application therefore reports visual evidence, confidence, image-quality warnings, and limitations instead of claiming certified repair-cost estimation.",
        ]
    )
    REPORT_PATH.write_text("\n".join(lines), encoding="utf-8")


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--input-dir", type=Path, default=DEFAULT_INPUT)
    parser.add_argument("--limit", type=int, default=10)
    parser.add_argument("--base-price-eur", type=float, default=15_000)
    return parser.parse_args()


def main() -> None:
    args = parse_args()
    rows = evaluate_subset(args.input_dir, args.limit, args.base_price_eur)
    write_report(rows, args.input_dir, args.limit)
    print(f"Wrote {REPORT_PATH}")
    print(f"Wrote {JSON_PATH}")


if __name__ == "__main__":
    main()