from __future__ import annotations import asyncio import json import pytest from PIL import Image from autocad_bench.tasks.manifest import load_manifest from autocad_bench.evaluation.scoring import ( GoldCacheError, GoldCacheInvalidError, GoldCacheStore, StaticDWGEvaluator, build_gold_cache, ) from tests.scoring_helpers import valid_drawing def test_gold_cache_round_trip_and_tamper_detection(tmp_path) -> None: drawing = valid_drawing(b"AC1032-gold") store = GoldCacheStore(evaluator_version="static-test-v1", root=tmp_path) entry_dir = store.write("task-001", drawing) assert store.load("task-001") == drawing (entry_dir / "render.png").write_bytes(b"corrupt") with pytest.raises(GoldCacheInvalidError): store.load("task-001") def test_gold_cache_rejects_3d_gold_not_presented_on_layout1(tmp_path) -> None: drawing = valid_drawing(b"AC1032-3d-gold", drawing_type="3d") invalid_metadata = drawing.metadata.model_copy( update={ "rendered_layout": "Layout2", "paper_layouts": tuple( layout.model_copy(update={"name": "Layout2"}) for layout in drawing.metadata.paper_layouts ), } ) store = GoldCacheStore(evaluator_version="static-test-v1", root=tmp_path) with pytest.raises(GoldCacheError, match="must render paper-space Layout1"): store.write( "task-022", drawing.model_copy(update={"metadata": invalid_metadata}), ) def test_builds_and_verifies_all_five_development_entries(tmp_path) -> None: fixtures = {} dev_entries = [entry for entry in load_manifest() if entry.split == "dev"] for entry in dev_entries: gold_bytes = entry.resolve_gold_path().read_bytes() # Reference PNG use is test-only. Production caches must come from AutoCAD. render_png = entry.resolve_image_path().read_bytes() with Image.open(entry.resolve_image_path()) as image: width, height = image.size drawing = valid_drawing( gold_bytes, render_png=render_png, extents={ "min_x": 0, "min_y": 0, "max_x": width, "max_y": height, "units": "mm", }, drawing_type=entry.drawing_type, ) fixtures[drawing.metadata.source_sha256] = drawing evaluator = StaticDWGEvaluator(fixtures, evaluator_version="static-test-v1") store = GoldCacheStore(evaluator_version="static-test-v1", root=tmp_path) built = asyncio.run(build_gold_cache(evaluator, store, split="dev")) assert built == tuple(entry.task_id for entry in dev_entries) assert store.verify(list(built)) == built for task_id in built: metadata = json.loads( (tmp_path / "static-test-v1" / task_id / "metadata.json").read_text() ) assert metadata["task_id"] == task_id assert metadata["evaluator_version"] == "static-test-v1"