Occasion-Scanner / scripts /generate_example_cases.py
ganemmah's picture
Sync project documentation and evaluation artifacts
fef29bc verified
Raw
History Blame Contribute Delete
8.07 kB
"""Generate reproducible end-to-end example cases for the final report."""
from __future__ import annotations
import json
import re
import sys
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parents[1]
sys.path.append(str(PROJECT_ROOT))
from app.app import predict_listing
REPORT_PATH = PROJECT_ROOT / "reports/example_cases.md"
JSON_PATH = PROJECT_ROOT / "reports/example_cases.json"
CASES = [
{
"case_id": "clean_vehicle_manual_check",
"description": "Clean baseline case using verified vehicle data and manual no-damage fallback.",
"args": {
"make": "BMW",
"model": "3 Series",
"production_year": 2019,
"mileage_km": 64_000,
"power_hp": 184,
"fuel_category": "Gasoline",
"transmission": "Automatic",
"custom_make": "",
"body_type": "Sedan",
"seller_is_dealer": False,
"has_warranty": True,
"nr_prev_owners": 1,
"seller_notes": "Fresh service, non-smoking vehicle, winter tires included.",
"front_image": None,
"side_image": None,
"rear_image": None,
"interior_image": None,
"use_image_model": False,
"vision_mode": "OpenAI Vision",
"manual_damage_labels": [],
"prompt_strategy": "Sales-optimized but honest",
},
},
{
"case_id": "minor_damage_manual_check",
"description": "Minor visible-damage case using controlled scratch and dent labels.",
"args": {
"make": "Volkswagen",
"model": "Golf",
"production_year": 2018,
"mileage_km": 50_000,
"power_hp": 150,
"fuel_category": "Gasoline",
"transmission": "Manual",
"custom_make": "",
"body_type": "Hatchback",
"seller_is_dealer": False,
"has_warranty": False,
"nr_prev_owners": 1,
"seller_notes": "Fresh service, summer and winter tires included.",
"front_image": None,
"side_image": None,
"rear_image": None,
"interior_image": None,
"use_image_model": False,
"vision_mode": "OpenAI Vision",
"manual_damage_labels": ["scratch", "dent"],
"prompt_strategy": "Sales-optimized but honest",
},
},
{
"case_id": "local_cv_severe_damage",
"description": "Local CV baseline case using a held-out severe-damage sample image.",
"args": {
"make": "Volkswagen",
"model": "Golf",
"production_year": 2018,
"mileage_km": 50_000,
"power_hp": 150,
"fuel_category": "Gasoline",
"transmission": "Manual",
"custom_make": "",
"body_type": "Hatchback",
"seller_is_dealer": False,
"has_warranty": False,
"nr_prev_owners": 1,
"seller_notes": "Vehicle has visible exterior damage in the uploaded image.",
"front_image": str(PROJECT_ROOT / "data/samples/cv_subset/severe_damage_1.jpg"),
"side_image": None,
"rear_image": None,
"interior_image": None,
"use_image_model": True,
"vision_mode": "Local CV baseline",
"manual_damage_labels": [],
"prompt_strategy": "Neutral factual",
},
},
]
def strip_html(text: str) -> str:
"""Convert short app HTML snippets into readable report text."""
text = re.sub(r"<br\\s*/?>", "\n", text)
text = re.sub(r"<[^>]+>", " ", text)
text = text.replace("&nbsp;", " ")
text = re.sub(r"\\s+", " ", text)
return text.strip()
def format_chf_amount(value: float | int | None) -> str:
"""Format a value that is already expressed in CHF."""
if value is None:
return "n/a"
return f"CHF {float(value):,.0f}".replace(",", "'")
def run_case(case: dict) -> dict:
"""Run one app case and collect structured outputs."""
seller_cockpit, metrics, sensitivity, flow, vision, listing, explanation, details_json, prompt = predict_listing(**case["args"])
details = json.loads(details_json)
return {
"case_id": case["case_id"],
"description": case["description"],
"input": {
key: value
for key, value in case["args"].items()
if key not in {"front_image", "side_image", "rear_image", "interior_image"}
},
"images": {
"front": case["args"]["front_image"],
"side": case["args"]["side_image"],
"rear": case["args"]["rear_image"],
"interior": case["args"]["interior_image"],
},
"base_price_chf": details["base_price_chf"],
"discount_chf": details["discount_chf"],
"adjusted_price_chf": details["adjusted_price_chf"],
"damage_score": details["damage_score"],
"damage_model": details["damage_model"],
"detected_damages": details["detected_damages"],
"image_quality": details["image_quality"],
"quality_warnings": details["quality_warnings"],
"visual_evidence": details["visual_evidence"],
"seller_cockpit_text": strip_html(seller_cockpit),
"metrics_text": strip_html(metrics),
"sensitivity_text": strip_html(sensitivity),
"flow_text": strip_html(flow),
"vision_text": strip_html(vision),
"listing_text": listing,
"explanation": explanation,
"prompt_trace": prompt,
}
def write_report(rows: list[dict]) -> None:
"""Write markdown and JSON example case artifacts."""
REPORT_PATH.parent.mkdir(parents=True, exist_ok=True)
JSON_PATH.write_text(json.dumps(rows, indent=2), encoding="utf-8")
lines = [
"# Example Cases",
"",
"These cases were generated by running the actual application pipeline. They demonstrate how structured vehicle data, visual damage information, CHF pricing, and listing generation interact.",
"",
"| Case | Vision mode | Damage score | Base price | Discount | Recommended price | Detected damages |",
"|---|---|---:|---:|---:|---:|---|",
]
for row in rows:
labels = ", ".join(item["label"] for item in row["detected_damages"]) or "none"
lines.append(
f"| {row['case_id']} | {row['input']['vision_mode']} | {row['damage_score']:.3f} | "
f"{format_chf_amount(row['base_price_chf'])} | {format_chf_amount(row['discount_chf'])} | "
f"{format_chf_amount(row['adjusted_price_chf'])} | {labels} |"
)
for row in rows:
lines.extend(
[
"",
f"## {row['case_id']}",
"",
row["description"],
"",
f"- Vision mode: {row['input']['vision_mode']}",
f"- Damage model: `{row['damage_model']}`",
f"- Damage score: {row['damage_score']:.3f}",
f"- Base price: CHF {row['base_price_chf']:,.0f}".replace(",", "'"),
f"- Damage discount: CHF {row['discount_chf']:,.0f}".replace(",", "'"),
f"- Recommended price: CHF {row['adjusted_price_chf']:,.0f}".replace(",", "'"),
f"- Visual evidence: {row['visual_evidence'] or 'No visual evidence available.'}",
"",
"Listing output:",
"",
"```text",
row["listing_text"],
"```",
"",
"Explanation output:",
"",
"```text",
row["explanation"],
"```",
]
)
REPORT_PATH.write_text("\n".join(lines), encoding="utf-8")
def main() -> None:
rows = [run_case(case) for case in CASES]
write_report(rows)
print(f"Wrote {REPORT_PATH}")
print(f"Wrote {JSON_PATH}")
if __name__ == "__main__":
main()