| from __future__ import annotations |
|
|
| import asyncio |
| import json |
|
|
| import pytest |
| from PIL import Image |
|
|
| from autocad_bench.tasks.manifest import load_manifest |
| from autocad_bench.evaluation.scoring import ( |
| GoldCacheError, |
| GoldCacheInvalidError, |
| GoldCacheStore, |
| StaticDWGEvaluator, |
| build_gold_cache, |
| ) |
| from tests.scoring_helpers import valid_drawing |
|
|
|
|
| def test_gold_cache_round_trip_and_tamper_detection(tmp_path) -> None: |
| drawing = valid_drawing(b"AC1032-gold") |
| store = GoldCacheStore(evaluator_version="static-test-v1", root=tmp_path) |
| entry_dir = store.write("task-001", drawing) |
|
|
| assert store.load("task-001") == drawing |
|
|
| (entry_dir / "render.png").write_bytes(b"corrupt") |
| with pytest.raises(GoldCacheInvalidError): |
| store.load("task-001") |
|
|
|
|
| def test_gold_cache_rejects_3d_gold_not_presented_on_layout1(tmp_path) -> None: |
| drawing = valid_drawing(b"AC1032-3d-gold", drawing_type="3d") |
| invalid_metadata = drawing.metadata.model_copy( |
| update={ |
| "rendered_layout": "Layout2", |
| "paper_layouts": tuple( |
| layout.model_copy(update={"name": "Layout2"}) |
| for layout in drawing.metadata.paper_layouts |
| ), |
| } |
| ) |
| store = GoldCacheStore(evaluator_version="static-test-v1", root=tmp_path) |
|
|
| with pytest.raises(GoldCacheError, match="must render paper-space Layout1"): |
| store.write( |
| "task-022", |
| drawing.model_copy(update={"metadata": invalid_metadata}), |
| ) |
|
|
|
|
| def test_builds_and_verifies_all_five_development_entries(tmp_path) -> None: |
| fixtures = {} |
| dev_entries = [entry for entry in load_manifest() if entry.split == "dev"] |
| for entry in dev_entries: |
| gold_bytes = entry.resolve_gold_path().read_bytes() |
| |
| render_png = entry.resolve_image_path().read_bytes() |
| with Image.open(entry.resolve_image_path()) as image: |
| width, height = image.size |
| drawing = valid_drawing( |
| gold_bytes, |
| render_png=render_png, |
| extents={ |
| "min_x": 0, |
| "min_y": 0, |
| "max_x": width, |
| "max_y": height, |
| "units": "mm", |
| }, |
| drawing_type=entry.drawing_type, |
| ) |
| fixtures[drawing.metadata.source_sha256] = drawing |
|
|
| evaluator = StaticDWGEvaluator(fixtures, evaluator_version="static-test-v1") |
| store = GoldCacheStore(evaluator_version="static-test-v1", root=tmp_path) |
|
|
| built = asyncio.run(build_gold_cache(evaluator, store, split="dev")) |
|
|
| assert built == tuple(entry.task_id for entry in dev_entries) |
| assert store.verify(list(built)) == built |
| for task_id in built: |
| metadata = json.loads( |
| (tmp_path / "static-test-v1" / task_id / "metadata.json").read_text() |
| ) |
| assert metadata["task_id"] == task_id |
| assert metadata["evaluator_version"] == "static-test-v1" |
|
|
