scripts/verify_reference_codes.py

#!/usr/bin/env python3
"""Verify all 20 reference codes execute and produce valid geometry.

Usage:
    python scripts/verify_reference_codes.py
    python scripts/verify_reference_codes.py --task task_001_flat_plate
"""
import argparse
import json
import sys
import time
from pathlib import Path

TASKS_ROOT = Path(__file__).parent.parent / "server" / "tasks"


def verify_one(task_dir: Path) -> dict:
    task_id = task_dir.name
    ref_code = (task_dir / "reference_code.py").read_text()
    t0 = time.time()

    result = {
        "task_id": task_id,
        "code_executes": False,
        "shape_valid": False,
        "volume_gt_zero": False,
        "volume": None,
        "bbox": None,
        "face_count": None,
        "error": None,
    }

    try:
        import cadquery as cq
        import math

        local_ns = {"cq": cq, "cadquery": cq, "math": math}
        exec(ref_code, local_ns)

        if "result" not in local_ns:
            result["error"] = "No 'result' variable defined"
            return result

        result["code_executes"] = True
        obj = local_ns["result"]
        shape = obj.val() if hasattr(obj, "val") else obj

        result["shape_valid"] = shape.isValid()
        vol = shape.Volume()
        result["volume"] = round(vol, 2)
        result["volume_gt_zero"] = vol > 0

        bb = shape.BoundingBox()
        result["bbox"] = [round(bb.xlen, 2), round(bb.ylen, 2), round(bb.zlen, 2)]
        result["face_count"] = len(shape.Faces())

    except Exception as e:
        result["error"] = f"{type(e).__name__}: {e}"

    result["elapsed_s"] = round(time.time() - t0, 2)
    return result


def main():
    parser = argparse.ArgumentParser(description="Verify CadForge reference codes")
    parser.add_argument("--task", type=str, default=None, help="Specific task id to verify")
    args = parser.parse_args()

    if args.task:
        task_dirs = [TASKS_ROOT / args.task]
    else:
        task_dirs = sorted(TASKS_ROOT.glob("task_*"))

    print(f"Verifying {len(task_dirs)} reference codes...")
    print("-" * 80)

    results = []
    for task_dir in task_dirs:
        r = verify_one(task_dir)
        passed = r["code_executes"] and r["shape_valid"] and r["volume_gt_zero"]
        status = "PASS" if passed else "FAIL"
        print(
            f"  {r['task_id']:35s} {status:5s} "
            f"vol={r['volume'] or 'N/A':>10} "
            f"bbox={r['bbox'] or 'N/A'} "
            f"faces={r['face_count'] or 'N/A'} "
            f"({r['elapsed_s']:.1f}s)"
        )
        if r["error"]:
            print(f"    ERROR: {r['error']}")
        results.append(r)

    passed = sum(1 for r in results if r["code_executes"] and r["shape_valid"] and r["volume_gt_zero"])
    print("-" * 80)
    print(f"Result: {passed}/{len(results)} passed")

    out_path = TASKS_ROOT.parent / "verification_phase1.json"
    with open(out_path, "w") as f:
        json.dump(results, f, indent=2)
    print(f"Saved to {out_path}")

    return 0 if passed == len(results) else 1


if __name__ == "__main__":
    sys.exit(main())