examples/test_examples.py

#!/usr/bin/env python3
"""Decode the bundled examples and check them against the expected answers."""
import json
import subprocess
import sys
from pathlib import Path

HERE = Path(__file__).resolve().parent
ROOT = HERE.parent

def main() -> int:
    cases = json.loads((HERE / "examples.json").read_text(encoding="utf-8"))
    bad = 0
    for case in cases:
        out = subprocess.run(
            [sys.executable, str(ROOT / "inference.py"), "--model-dir", str(ROOT),
             "--image", str(HERE / case["image"]), "--precision", case.get("precision", "fp32")]
            + [arg for q in case["questions"] for arg in ("--question", q["question"])],
            capture_output=True, text=True, check=True,
        )
        got = json.loads(out.stdout)
        if got["record"] != case["expected_record"]:
            bad += 1
            print(f"RECORD MISMATCH {case['image']}: {got['record']} != {case['expected_record']}")
        for want, have in zip(case["questions"], got["answers"]):
            if want["answer"] != have["answer"]:
                bad += 1
                print(f"ANSWER MISMATCH {case['image']} {want['question']!r}: "
                      f"{have['answer']!r} != {want['answer']!r}")
        print(f"{case['image']}: record={got['record']} "
              f"answers={[a['answer'] for a in got['answers']]}")
    print("OK" if not bad else f"{bad} MISMATCHES")
    return 1 if bad else 0

if __name__ == "__main__":
    raise SystemExit(main())