| import json |
| import sys |
| import tempfile |
| import unittest |
| from pathlib import Path |
|
|
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) |
|
|
| from evaluation.tools import paired_bootstrap |
|
|
|
|
| def _write(path, payload): |
| path.write_text(json.dumps(payload), encoding="utf-8") |
|
|
|
|
| class PairedBootstrapTests(unittest.TestCase): |
| def _toy_gt(self): |
| return { |
| "images": [{"id": 1}, {"id": 2}, {"id": 3}], |
| "categories": [ |
| {"id": 1, "name": "Button"}, |
| {"id": 2, "name": "Sphere"}, |
| ], |
| "annotations": [ |
| {"id": 1, "image_id": 1, "category_id": 1, "bbox": [0, 0, 10, 10]}, |
| {"id": 2, "image_id": 2, "category_id": 2, "bbox": [0, 0, 10, 10]}, |
| ], |
| } |
|
|
| def test_greedy_matching_uses_score_order_and_complete_images(self): |
| gt = self._toy_gt() |
| preds = [ |
| {"image_id": 1, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 0.9}, |
| {"image_id": 1, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 0.2}, |
| {"image_id": 2, "category_id": 2, "bbox": [0, 0, 10, 10], "score": 0.8}, |
| ] |
|
|
| prepared_gt = paired_bootstrap.prepare_ground_truth(gt) |
| evaluated = paired_bootstrap.evaluate_predictions( |
| prepared_gt, |
| preds, |
| dimension="interactable", |
| iou_threshold=0.75, |
| score_threshold=0.0, |
| ) |
|
|
| self.assertEqual(evaluated.counts["tp"], 2) |
| self.assertEqual(evaluated.counts["fp"], 1) |
| self.assertEqual(evaluated.counts["fn"], 0) |
| self.assertEqual(len(evaluated.per_image), 3) |
| self.assertEqual(evaluated.per_image[3].support, 0) |
|
|
| def test_auto_threshold_maximizes_micro_f1(self): |
| gt = self._toy_gt() |
| preds = [ |
| {"image_id": 1, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 0.4}, |
| {"image_id": 2, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 0.9}, |
| {"image_id": 2, "category_id": 2, "bbox": [0, 0, 10, 10], "score": 0.8}, |
| {"image_id": 3, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 0.4}, |
| {"image_id": 3, "category_id": 1, "bbox": [1, 1, 10, 10], "score": 0.4}, |
| {"image_id": 3, "category_id": 2, "bbox": [2, 2, 10, 10], "score": 0.4}, |
| {"image_id": 3, "category_id": 2, "bbox": [3, 3, 10, 10], "score": 0.4}, |
| ] |
|
|
| prepared_gt = paired_bootstrap.prepare_ground_truth(gt) |
| threshold, evaluated = paired_bootstrap.select_best_threshold( |
| prepared_gt, |
| preds, |
| dimension="interactable", |
| iou_threshold=0.75, |
| ) |
|
|
| self.assertEqual(threshold, 0.8) |
| self.assertAlmostEqual(evaluated.metrics["f1"], 0.5) |
|
|
| def test_fast_auto_threshold_matches_bruteforce_selection(self): |
| gt = self._toy_gt() |
| preds = [ |
| {"image_id": 1, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 0.91}, |
| {"image_id": 1, "category_id": 1, "bbox": [1, 1, 10, 10], "score": 0.42}, |
| {"image_id": 2, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 0.88}, |
| {"image_id": 2, "category_id": 2, "bbox": [0, 0, 10, 10], "score": 0.73}, |
| {"image_id": 3, "category_id": 2, "bbox": [0, 0, 10, 10], "score": 0.11}, |
| {"image_id": 99, "category_id": 2, "bbox": [0, 0, 10, 10], "score": 0.72}, |
| ] |
| prepared_gt = paired_bootstrap.prepare_ground_truth(gt) |
|
|
| fast_threshold = paired_bootstrap.select_best_threshold_fast( |
| prepared_gt, |
| preds, |
| dimension="interactable", |
| iou_threshold=0.75, |
| ) |
| brute_threshold, brute_result = self._select_best_threshold_bruteforce(prepared_gt, preds) |
|
|
| self.assertEqual(fast_threshold, brute_threshold) |
| self.assertEqual(fast_threshold, 0.72) |
| fast_result = paired_bootstrap.evaluate_predictions( |
| prepared_gt, |
| preds, |
| dimension="interactable", |
| iou_threshold=0.75, |
| score_threshold=fast_threshold, |
| ) |
| self.assertEqual(fast_result.metrics, brute_result.metrics) |
|
|
| def _select_best_threshold_bruteforce(self, prepared_gt, preds): |
| best_threshold = None |
| best_result = None |
| for threshold in sorted({float(pred["score"]) for pred in preds}, reverse=True): |
| result = paired_bootstrap.evaluate_predictions( |
| prepared_gt, |
| preds, |
| dimension="interactable", |
| iou_threshold=0.75, |
| score_threshold=threshold, |
| ) |
| if best_result is None or ( |
| result.metrics["f1"], |
| result.metrics["precision"], |
| result.metrics["recall"], |
| -threshold, |
| ) > ( |
| best_result.metrics["f1"], |
| best_result.metrics["precision"], |
| best_result.metrics["recall"], |
| -best_threshold, |
| ): |
| best_threshold = threshold |
| best_result = result |
| return best_threshold, best_result |
|
|
| def test_semantic_cache_match_maps_numeric_categories(self): |
| with tempfile.TemporaryDirectory() as tmpdir: |
| tmp = Path(tmpdir) |
| cache = tmp / "embedding.json" |
| _write(cache, {"Button": [1.0, 0.0], "Control": [0.9, 0.1]}) |
| matcher = paired_bootstrap.SemanticMatcher(cache) |
|
|
| gt = self._toy_gt() |
| preds = [ |
| {"image_id": 1, "category_id": "control", "bbox": [0, 0, 10, 10], "score": 1.0} |
| ] |
| prepared_gt = paired_bootstrap.prepare_ground_truth(gt) |
| evaluated = paired_bootstrap.evaluate_predictions( |
| prepared_gt, |
| preds, |
| dimension="semantics", |
| iou_threshold=0.75, |
| score_threshold=0.0, |
| semantic_matcher=matcher, |
| ) |
|
|
| self.assertEqual(evaluated.counts["tp"], 1) |
|
|
| def test_semantic_cache_preserves_historical_camelcase_keys(self): |
| with tempfile.TemporaryDirectory() as tmpdir: |
| cache = Path(tmpdir) / "embedding.json" |
| _write(cache, {"QuitButton": [1.0, 0.0], "ConfirmButton": [0.9, 0.1]}) |
| matcher = paired_bootstrap.SemanticMatcher(cache) |
|
|
| self.assertTrue(matcher.matches("quit_button", "confirm_button")) |
|
|
| def test_semantic_match_defaults_to_historical_raw_dot_product(self): |
| with tempfile.TemporaryDirectory() as tmpdir: |
| cache = Path(tmpdir) / "embedding.json" |
| _write(cache, {"A": [1.0, 0.0], "B": [0.84, 0.20]}) |
|
|
| historical = paired_bootstrap.SemanticMatcher(cache) |
| cosine = paired_bootstrap.SemanticMatcher(cache, similarity_mode="cosine") |
|
|
| self.assertFalse(historical.matches("a", "b")) |
| self.assertTrue(cosine.matches("a", "b")) |
|
|
| def test_semantic_cache_miss_is_hard_error(self): |
| with tempfile.TemporaryDirectory() as tmpdir: |
| cache = Path(tmpdir) / "embedding.json" |
| _write(cache, {"Button": [1.0, 0.0]}) |
| matcher = paired_bootstrap.SemanticMatcher(cache) |
|
|
| with self.assertRaisesRegex(KeyError, "Missing frozen embedding cache entries"): |
| matcher.matches("button", "unknown") |
|
|
| def test_cli_writes_json_report(self): |
| with tempfile.TemporaryDirectory() as tmpdir: |
| tmp = Path(tmpdir) |
| gt_path = tmp / "gt.json" |
| a_path = tmp / "a.json" |
| b_path = tmp / "b.json" |
| out_path = tmp / "report.json" |
| _write(gt_path, self._toy_gt()) |
| _write(a_path, [{"image_id": 1, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 1.0}]) |
| _write( |
| b_path, |
| [ |
| {"image_id": 1, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 1.0}, |
| {"image_id": 2, "category_id": 2, "bbox": [0, 0, 10, 10], "score": 0.9}, |
| ], |
| ) |
|
|
| paired_bootstrap.main( |
| [ |
| "--gt", |
| str(gt_path), |
| "--method-a", |
| str(a_path), |
| "--method-b", |
| str(b_path), |
| "--dimension", |
| "interactable", |
| "--auto-threshold", |
| "--replicates", |
| "100", |
| "--seed", |
| "7", |
| "--output", |
| str(out_path), |
| ] |
| ) |
|
|
| report = json.loads(out_path.read_text(encoding="utf-8")) |
| self.assertEqual(report["protocol"]["dimension"], "interactable") |
| self.assertEqual(report["protocol"]["bootstrap_unit"], "image_id") |
| self.assertGreater(report["methods"]["method_b"]["point"]["micro"]["f1"], report["methods"]["method_a"]["point"]["micro"]["f1"]) |
| self.assertIn("mean_per_all_images", report["methods"]["method_a"]["point"]) |
| self.assertIn("mean_per_positive_support_images", report["methods"]["method_a"]["point"]) |
| self.assertIn("mean_per_all_images_delta", report["bootstrap"]) |
| self.assertIn("mean_per_positive_support_images_delta", report["bootstrap"]) |
| self.assertEqual(report["bootstrap"]["delta_direction"], "method_b_minus_method_a") |
|
|
|
|
| if __name__ == "__main__": |
| unittest.main() |
|
|
