#!/usr/bin/env python3 """Verify all 20 reference codes execute and produce valid geometry. Usage: python scripts/verify_reference_codes.py python scripts/verify_reference_codes.py --task task_001_flat_plate """ import argparse import json import sys import time from pathlib import Path TASKS_ROOT = Path(__file__).parent.parent / "server" / "tasks" def verify_one(task_dir: Path) -> dict: task_id = task_dir.name ref_code = (task_dir / "reference_code.py").read_text() t0 = time.time() result = { "task_id": task_id, "code_executes": False, "shape_valid": False, "volume_gt_zero": False, "volume": None, "bbox": None, "face_count": None, "error": None, } try: import cadquery as cq import math local_ns = {"cq": cq, "cadquery": cq, "math": math} exec(ref_code, local_ns) if "result" not in local_ns: result["error"] = "No 'result' variable defined" return result result["code_executes"] = True obj = local_ns["result"] shape = obj.val() if hasattr(obj, "val") else obj result["shape_valid"] = shape.isValid() vol = shape.Volume() result["volume"] = round(vol, 2) result["volume_gt_zero"] = vol > 0 bb = shape.BoundingBox() result["bbox"] = [round(bb.xlen, 2), round(bb.ylen, 2), round(bb.zlen, 2)] result["face_count"] = len(shape.Faces()) except Exception as e: result["error"] = f"{type(e).__name__}: {e}" result["elapsed_s"] = round(time.time() - t0, 2) return result def main(): parser = argparse.ArgumentParser(description="Verify CadForge reference codes") parser.add_argument("--task", type=str, default=None, help="Specific task id to verify") args = parser.parse_args() if args.task: task_dirs = [TASKS_ROOT / args.task] else: task_dirs = sorted(TASKS_ROOT.glob("task_*")) print(f"Verifying {len(task_dirs)} reference codes...") print("-" * 80) results = [] for task_dir in task_dirs: r = verify_one(task_dir) passed = r["code_executes"] and r["shape_valid"] and r["volume_gt_zero"] status = "PASS" if passed else "FAIL" print( f" {r['task_id']:35s} {status:5s} " f"vol={r['volume'] or 'N/A':>10} " f"bbox={r['bbox'] or 'N/A'} " f"faces={r['face_count'] or 'N/A'} " f"({r['elapsed_s']:.1f}s)" ) if r["error"]: print(f" ERROR: {r['error']}") results.append(r) passed = sum(1 for r in results if r["code_executes"] and r["shape_valid"] and r["volume_gt_zero"]) print("-" * 80) print(f"Result: {passed}/{len(results)} passed") out_path = TASKS_ROOT.parent / "verification_phase1.json" with open(out_path, "w") as f: json.dump(results, f, indent=2) print(f"Saved to {out_path}") return 0 if passed == len(results) else 1 if __name__ == "__main__": sys.exit(main())