from __future__ import annotations import gzip import hashlib import json from pathlib import Path import numpy as np import pytest from PIL import Image from scripts.stages import evaluate_od06_od07_coco as evaluator ROOT = Path(__file__).resolve().parents[1] CONFIG_PATH = ROOT / "configs/evaluation/object_detection/OD06_OD07_coco2017_quality_eval.json" ANNOTATIONS = ROOT / "research/downloads/coco2017/annotations/instances_val2017.json" def test_deterministic_gzip_json(tmp_path: Path) -> None: value = [{"image_id": 1, "category_id": 2, "bbox": [1.0, 2.0, 3.0, 4.0], "score": 0.5}] first = tmp_path / "first.json.gz" second = tmp_path / "second.json.gz" evaluator.atomic_gzip_json(first, value) evaluator.atomic_gzip_json(second, value) assert first.read_bytes() == second.read_bytes() with gzip.open(first, "rt", encoding="utf-8") as handle: assert json.load(handle) == value def test_grayscale_is_forced_to_contiguous_rgb(tmp_path: Path) -> None: source = tmp_path / "gray.jpg" Image.fromarray(np.arange(35, dtype=np.uint8).reshape(5, 7), mode="L").save(source) array = evaluator.load_rgb_array(source, expected_width=7, expected_height=5) assert array.shape == (5, 7, 3) assert array.dtype == np.uint8 assert array.flags.c_contiguous assert np.array_equal(array[:, :, 0], array[:, :, 1]) assert np.array_equal(array[:, :, 1], array[:, :, 2]) @pytest.mark.skipif(not ANNOTATIONS.is_file(), reason="COCO annotations are an optional downloaded dataset") def test_exact_models_embed_paired_decoder_contract() -> None: config = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) annotations = json.loads((ROOT / config["dataset"]["annotation_path"]).read_text(encoding="utf-8")) coco_names = {row["name"] for row in annotations["categories"]} for model_id, model in config["models"].items(): metadata = { variant: evaluator.inspect_detector_metadata(ROOT / specification["path"]) for variant, specification in model["variants"].items() } assert metadata["fp32"]["detector_metadata_sha256"] == model["expected_detector_metadata_sha256"] assert metadata["public_int8"]["detector_metadata_sha256"] == model["expected_detector_metadata_sha256"] assert metadata["fp32"]["fixed_anchor_count"] == model["expected_anchor_count"] assert metadata["public_int8"]["fixed_anchor_count"] == model["expected_anchor_count"] assert metadata["fp32"]["decoding"]["num_boxes"] == model["expected_anchor_count"] assert metadata["public_int8"]["decoding"]["num_classes"] == 90 assert metadata["fp32"]["labels_sha256"] == metadata["public_int8"]["labels_sha256"] assert {label for label in metadata["fp32"]["labels"] if label != "???"} == coco_names assert metadata["fp32"]["placeholder_label_count"] == 10 @pytest.mark.skipif( not ANNOTATIONS.is_file(), reason="compact repository omits the COCO dataset and copied upstream evaluator sources", ) def test_config_pins_full_official_split_models_and_sources() -> None: config = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) assert config["dataset"]["expected_images"] == 5000 assert config["dataset"]["expected_categories"] == 80 assert config["protocol"]["max_detections"] == [1, 10, 100] assert config["protocol"]["latency_measurement"] is False assert config["protocol"]["acceptance_threshold"] is None for model in config["models"].values(): assert model["published_context"]["use"] == "CONTEXT_ONLY" assert model["published_context"]["exact_artifact_threshold"] is False for specification in model["variants"].values(): path = ROOT / specification["path"] assert path.stat().st_size == specification["expected_bytes"] assert hashlib.sha256(path.read_bytes()).hexdigest() == specification["expected_sha256"] for source in config["authoritative_sources"]: assert hashlib.sha256((ROOT / source["path"]).read_bytes()).hexdigest() == source["sha256"] def test_report_explicitly_rejects_exact_published_comparison() -> None: summary = { "models": { model_id: { "variants": { "fp32": {"metrics_percent": {"bbox_ap": 1.0, "bbox_ap50": 2.0, "bbox_ap75": 0.5}}, "public_int8": {"metrics_percent": {"bbox_ap": 0.9, "bbox_ap50": 1.9, "bbox_ap75": 0.4}}, }, "pair_comparison": {"bbox_ap_retention_percent": 90.0}, "published_context": { "value": value, "relationship": "Related postprocessed variant, not byte-identical", }, } for model_id, value in (("OD06", 25.69), ("OD07", 33.97)) } } report = evaluator.build_report(summary) assert "byte-identical exact artifact 값이 아니라" in report assert "합격 기준으로 사용하지 않았다" in report