tests/test_od06_od07_coco_evaluator.py

from __future__ import annotations

import gzip
import hashlib
import json
from pathlib import Path

import numpy as np
import pytest
from PIL import Image

from scripts.stages import evaluate_od06_od07_coco as evaluator


ROOT = Path(__file__).resolve().parents[1]
CONFIG_PATH = ROOT / "configs/evaluation/object_detection/OD06_OD07_coco2017_quality_eval.json"
ANNOTATIONS = ROOT / "research/downloads/coco2017/annotations/instances_val2017.json"


def test_deterministic_gzip_json(tmp_path: Path) -> None:
    value = [{"image_id": 1, "category_id": 2, "bbox": [1.0, 2.0, 3.0, 4.0], "score": 0.5}]
    first = tmp_path / "first.json.gz"
    second = tmp_path / "second.json.gz"
    evaluator.atomic_gzip_json(first, value)
    evaluator.atomic_gzip_json(second, value)
    assert first.read_bytes() == second.read_bytes()
    with gzip.open(first, "rt", encoding="utf-8") as handle:
        assert json.load(handle) == value


def test_grayscale_is_forced_to_contiguous_rgb(tmp_path: Path) -> None:
    source = tmp_path / "gray.jpg"
    Image.fromarray(np.arange(35, dtype=np.uint8).reshape(5, 7), mode="L").save(source)
    array = evaluator.load_rgb_array(source, expected_width=7, expected_height=5)
    assert array.shape == (5, 7, 3)
    assert array.dtype == np.uint8
    assert array.flags.c_contiguous
    assert np.array_equal(array[:, :, 0], array[:, :, 1])
    assert np.array_equal(array[:, :, 1], array[:, :, 2])


@pytest.mark.skipif(not ANNOTATIONS.is_file(), reason="COCO annotations are an optional downloaded dataset")
def test_exact_models_embed_paired_decoder_contract() -> None:
    config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
    annotations = json.loads((ROOT / config["dataset"]["annotation_path"]).read_text(encoding="utf-8"))
    coco_names = {row["name"] for row in annotations["categories"]}
    for model_id, model in config["models"].items():
        metadata = {
            variant: evaluator.inspect_detector_metadata(ROOT / specification["path"])
            for variant, specification in model["variants"].items()
        }
        assert metadata["fp32"]["detector_metadata_sha256"] == model["expected_detector_metadata_sha256"]
        assert metadata["public_int8"]["detector_metadata_sha256"] == model["expected_detector_metadata_sha256"]
        assert metadata["fp32"]["fixed_anchor_count"] == model["expected_anchor_count"]
        assert metadata["public_int8"]["fixed_anchor_count"] == model["expected_anchor_count"]
        assert metadata["fp32"]["decoding"]["num_boxes"] == model["expected_anchor_count"]
        assert metadata["public_int8"]["decoding"]["num_classes"] == 90
        assert metadata["fp32"]["labels_sha256"] == metadata["public_int8"]["labels_sha256"]
        assert {label for label in metadata["fp32"]["labels"] if label != "???"} == coco_names
        assert metadata["fp32"]["placeholder_label_count"] == 10


@pytest.mark.skipif(
    not ANNOTATIONS.is_file(),
    reason="compact repository omits the COCO dataset and copied upstream evaluator sources",
)
def test_config_pins_full_official_split_models_and_sources() -> None:
    config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
    assert config["dataset"]["expected_images"] == 5000
    assert config["dataset"]["expected_categories"] == 80
    assert config["protocol"]["max_detections"] == [1, 10, 100]
    assert config["protocol"]["latency_measurement"] is False
    assert config["protocol"]["acceptance_threshold"] is None
    for model in config["models"].values():
        assert model["published_context"]["use"] == "CONTEXT_ONLY"
        assert model["published_context"]["exact_artifact_threshold"] is False
        for specification in model["variants"].values():
            path = ROOT / specification["path"]
            assert path.stat().st_size == specification["expected_bytes"]
            assert hashlib.sha256(path.read_bytes()).hexdigest() == specification["expected_sha256"]
    for source in config["authoritative_sources"]:
        assert hashlib.sha256((ROOT / source["path"]).read_bytes()).hexdigest() == source["sha256"]


def test_report_explicitly_rejects_exact_published_comparison() -> None:
    summary = {
        "models": {
            model_id: {
                "variants": {
                    "fp32": {"metrics_percent": {"bbox_ap": 1.0, "bbox_ap50": 2.0, "bbox_ap75": 0.5}},
                    "public_int8": {"metrics_percent": {"bbox_ap": 0.9, "bbox_ap50": 1.9, "bbox_ap75": 0.4}},
                },
                "pair_comparison": {"bbox_ap_retention_percent": 90.0},
                "published_context": {
                    "value": value,
                    "relationship": "Related postprocessed variant, not byte-identical",
                },
            }
            for model_id, value in (("OD06", 25.69), ("OD07", 33.97))
        }
    }
    report = evaluator.build_report(summary)
    assert "byte-identical exact artifact 값이 아니라" in report
    assert "합격 기준으로 사용하지 않았다" in report