File size: 4,943 Bytes
ed3aeeb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 | from __future__ import annotations
import importlib.util
import json
import tempfile
import unittest
from pathlib import Path
import numpy as np
REPO_ROOT = Path(__file__).resolve().parents[1]
def load_module(name: str, relative_path: str):
spec = importlib.util.spec_from_file_location(name, REPO_ROOT / relative_path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
evaluate = load_module("evaluate_voc2012_segmentation", "scripts/stages/evaluate_voc2012_segmentation.py")
validate = load_module("validate_voc2012_segmentation_quality", "scripts/stages/validate_voc2012_segmentation_quality.py")
class Voc2012QualityTests(unittest.TestCase):
def test_official_top_left_padding_and_ignore_label(self) -> None:
image = np.arange(2 * 3 * 3, dtype=np.uint8).reshape(2, 3, 3)
label = np.asarray([[0, 1, 2], [3, 4, 5]], dtype=np.uint8)
padded_image, padded_label = evaluate.pad_common_input(image, label, 5, 6, 128, 255)
self.assertEqual(padded_image.shape, (1, 5, 6, 3))
self.assertEqual(padded_label.shape, (5, 6))
np.testing.assert_array_equal(padded_image[0, :2, :3], image)
np.testing.assert_array_equal(padded_label[:2, :3], label)
self.assertTrue(np.all(padded_image[0, 2:, :] == 128))
self.assertTrue(np.all(padded_image[0, :, 3:] == 128))
self.assertTrue(np.all(padded_label[2:, :] == 255))
self.assertTrue(np.all(padded_label[:, 3:] == 255))
def test_confusion_excludes_ignore_255(self) -> None:
label = np.asarray([[0, 1, 255], [1, 2, 255]], dtype=np.uint8)
prediction = np.asarray([[[0, 2, 1], [1, 2, 0]]], dtype=np.int64)
histogram, valid_pixels = evaluate.confusion_matrix(label, prediction, 3, 255)
self.assertEqual(valid_pixels, 4)
expected = np.asarray([[1, 0, 0], [0, 1, 1], [0, 0, 1]], dtype=np.int64)
np.testing.assert_array_equal(histogram, expected)
def test_metric_matches_independent_recompute(self) -> None:
histogram = np.asarray([[4, 1, 0], [1, 3, 0], [0, 1, 2]], dtype=np.int64)
first = evaluate.metrics_from_confusion(histogram)
second = validate.recompute_metrics(histogram)
self.assertAlmostEqual(first["miou"], second["miou"], places=15)
self.assertAlmostEqual(first["pixel_accuracy"], second["pixel_accuracy"], places=15)
self.assertEqual(first["valid_pixels"], second["valid_pixels"])
self.assertEqual(first["valid_class_count"], second["valid_class_count"])
def test_checkpoint_rejects_changed_run_fingerprint(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
path = Path(temporary) / "checkpoint.jsonl"
path.write_text(json.dumps({"run_fingerprint": "old", "image_id": "a"}) + "\n")
with self.assertRaisesRegex(ValueError, "fingerprint mismatch"):
evaluate.load_checkpoint(path, "new")
def test_common_report_contract_is_literal_in_evaluator(self) -> None:
source = (REPO_ROOT / "scripts/stages/evaluate_voc2012_segmentation.py").read_text()
for token in [
'"acceptance_status": "MEASURED_NO_ACCEPTANCE_THRESHOLD"',
'"metric_name": "mean_iou"',
'"threshold": None',
'"fp32"',
'"public_int8"',
'"sample_count"',
]:
self.assertIn(token, source)
def test_metadata_finalizer_preserves_metrics_and_adds_formats(self) -> None:
finalizer = load_module(
"finalize_voc2012_quality_summary",
"scripts/stages/finalize_voc2012_quality_summary.py",
)
with tempfile.TemporaryDirectory() as temporary:
path = Path(temporary) / "quality_summary.json"
path.write_text(
json.dumps(
{
"model_id": "SGXX",
"quality": {
"fp32": {"miou": 0.75, "sample_count": 1449},
"public_int8": {"miou": 0.74, "sample_count": 1449},
},
}
)
)
original_argv = __import__("sys").argv
try:
__import__("sys").argv = ["finalize", "--summary", str(path)]
self.assertEqual(finalizer.main(), 0)
finally:
__import__("sys").argv = original_argv
value = json.loads(path.read_text())
self.assertEqual(value["quality"]["fp32"]["format"], "tensorflow_graphdef")
self.assertEqual(value["quality"]["public_int8"]["format"], "tflite")
self.assertTrue(value["pair_comparison"]["same_semantic_input"])
self.assertEqual(value["quality"]["fp32"]["miou"], 0.75)
if __name__ == "__main__":
unittest.main()
|