File size: 5,039 Bytes
ed3aeeb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 | from __future__ import annotations
import gzip
import hashlib
import json
from pathlib import Path
import numpy as np
import pytest
from PIL import Image
from scripts.stages import evaluate_od06_od07_coco as evaluator
ROOT = Path(__file__).resolve().parents[1]
CONFIG_PATH = ROOT / "configs/evaluation/object_detection/OD06_OD07_coco2017_quality_eval.json"
ANNOTATIONS = ROOT / "research/downloads/coco2017/annotations/instances_val2017.json"
def test_deterministic_gzip_json(tmp_path: Path) -> None:
value = [{"image_id": 1, "category_id": 2, "bbox": [1.0, 2.0, 3.0, 4.0], "score": 0.5}]
first = tmp_path / "first.json.gz"
second = tmp_path / "second.json.gz"
evaluator.atomic_gzip_json(first, value)
evaluator.atomic_gzip_json(second, value)
assert first.read_bytes() == second.read_bytes()
with gzip.open(first, "rt", encoding="utf-8") as handle:
assert json.load(handle) == value
def test_grayscale_is_forced_to_contiguous_rgb(tmp_path: Path) -> None:
source = tmp_path / "gray.jpg"
Image.fromarray(np.arange(35, dtype=np.uint8).reshape(5, 7), mode="L").save(source)
array = evaluator.load_rgb_array(source, expected_width=7, expected_height=5)
assert array.shape == (5, 7, 3)
assert array.dtype == np.uint8
assert array.flags.c_contiguous
assert np.array_equal(array[:, :, 0], array[:, :, 1])
assert np.array_equal(array[:, :, 1], array[:, :, 2])
@pytest.mark.skipif(not ANNOTATIONS.is_file(), reason="COCO annotations are an optional downloaded dataset")
def test_exact_models_embed_paired_decoder_contract() -> None:
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
annotations = json.loads((ROOT / config["dataset"]["annotation_path"]).read_text(encoding="utf-8"))
coco_names = {row["name"] for row in annotations["categories"]}
for model_id, model in config["models"].items():
metadata = {
variant: evaluator.inspect_detector_metadata(ROOT / specification["path"])
for variant, specification in model["variants"].items()
}
assert metadata["fp32"]["detector_metadata_sha256"] == model["expected_detector_metadata_sha256"]
assert metadata["public_int8"]["detector_metadata_sha256"] == model["expected_detector_metadata_sha256"]
assert metadata["fp32"]["fixed_anchor_count"] == model["expected_anchor_count"]
assert metadata["public_int8"]["fixed_anchor_count"] == model["expected_anchor_count"]
assert metadata["fp32"]["decoding"]["num_boxes"] == model["expected_anchor_count"]
assert metadata["public_int8"]["decoding"]["num_classes"] == 90
assert metadata["fp32"]["labels_sha256"] == metadata["public_int8"]["labels_sha256"]
assert {label for label in metadata["fp32"]["labels"] if label != "???"} == coco_names
assert metadata["fp32"]["placeholder_label_count"] == 10
@pytest.mark.skipif(
not ANNOTATIONS.is_file(),
reason="compact repository omits the COCO dataset and copied upstream evaluator sources",
)
def test_config_pins_full_official_split_models_and_sources() -> None:
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
assert config["dataset"]["expected_images"] == 5000
assert config["dataset"]["expected_categories"] == 80
assert config["protocol"]["max_detections"] == [1, 10, 100]
assert config["protocol"]["latency_measurement"] is False
assert config["protocol"]["acceptance_threshold"] is None
for model in config["models"].values():
assert model["published_context"]["use"] == "CONTEXT_ONLY"
assert model["published_context"]["exact_artifact_threshold"] is False
for specification in model["variants"].values():
path = ROOT / specification["path"]
assert path.stat().st_size == specification["expected_bytes"]
assert hashlib.sha256(path.read_bytes()).hexdigest() == specification["expected_sha256"]
for source in config["authoritative_sources"]:
assert hashlib.sha256((ROOT / source["path"]).read_bytes()).hexdigest() == source["sha256"]
def test_report_explicitly_rejects_exact_published_comparison() -> None:
summary = {
"models": {
model_id: {
"variants": {
"fp32": {"metrics_percent": {"bbox_ap": 1.0, "bbox_ap50": 2.0, "bbox_ap75": 0.5}},
"public_int8": {"metrics_percent": {"bbox_ap": 0.9, "bbox_ap50": 1.9, "bbox_ap75": 0.4}},
},
"pair_comparison": {"bbox_ap_retention_percent": 90.0},
"published_context": {
"value": value,
"relationship": "Related postprocessed variant, not byte-identical",
},
}
for model_id, value in (("OD06", 25.69), ("OD07", 33.97))
}
}
report = evaluator.build_report(summary)
assert "byte-identical exact artifact 값이 아니라" in report
assert "합격 기준으로 사용하지 않았다" in report
|