import json import sys import tempfile import unittest from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from evaluation.tools import paired_bootstrap def _write(path, payload): path.write_text(json.dumps(payload), encoding="utf-8") class PairedBootstrapTests(unittest.TestCase): def _toy_gt(self): return { "images": [{"id": 1}, {"id": 2}, {"id": 3}], "categories": [ {"id": 1, "name": "Button"}, {"id": 2, "name": "Sphere"}, ], "annotations": [ {"id": 1, "image_id": 1, "category_id": 1, "bbox": [0, 0, 10, 10]}, {"id": 2, "image_id": 2, "category_id": 2, "bbox": [0, 0, 10, 10]}, ], } def test_greedy_matching_uses_score_order_and_complete_images(self): gt = self._toy_gt() preds = [ {"image_id": 1, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 0.9}, {"image_id": 1, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 0.2}, {"image_id": 2, "category_id": 2, "bbox": [0, 0, 10, 10], "score": 0.8}, ] prepared_gt = paired_bootstrap.prepare_ground_truth(gt) evaluated = paired_bootstrap.evaluate_predictions( prepared_gt, preds, dimension="interactable", iou_threshold=0.75, score_threshold=0.0, ) self.assertEqual(evaluated.counts["tp"], 2) self.assertEqual(evaluated.counts["fp"], 1) self.assertEqual(evaluated.counts["fn"], 0) self.assertEqual(len(evaluated.per_image), 3) self.assertEqual(evaluated.per_image[3].support, 0) def test_auto_threshold_maximizes_micro_f1(self): gt = self._toy_gt() preds = [ {"image_id": 1, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 0.4}, {"image_id": 2, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 0.9}, {"image_id": 2, "category_id": 2, "bbox": [0, 0, 10, 10], "score": 0.8}, {"image_id": 3, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 0.4}, {"image_id": 3, "category_id": 1, "bbox": [1, 1, 10, 10], "score": 0.4}, {"image_id": 3, "category_id": 2, "bbox": [2, 2, 10, 10], "score": 0.4}, {"image_id": 3, "category_id": 2, "bbox": [3, 3, 10, 10], "score": 0.4}, ] prepared_gt = paired_bootstrap.prepare_ground_truth(gt) threshold, evaluated = paired_bootstrap.select_best_threshold( prepared_gt, preds, dimension="interactable", iou_threshold=0.75, ) self.assertEqual(threshold, 0.8) self.assertAlmostEqual(evaluated.metrics["f1"], 0.5) def test_fast_auto_threshold_matches_bruteforce_selection(self): gt = self._toy_gt() preds = [ {"image_id": 1, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 0.91}, {"image_id": 1, "category_id": 1, "bbox": [1, 1, 10, 10], "score": 0.42}, {"image_id": 2, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 0.88}, {"image_id": 2, "category_id": 2, "bbox": [0, 0, 10, 10], "score": 0.73}, {"image_id": 3, "category_id": 2, "bbox": [0, 0, 10, 10], "score": 0.11}, {"image_id": 99, "category_id": 2, "bbox": [0, 0, 10, 10], "score": 0.72}, ] prepared_gt = paired_bootstrap.prepare_ground_truth(gt) fast_threshold = paired_bootstrap.select_best_threshold_fast( prepared_gt, preds, dimension="interactable", iou_threshold=0.75, ) brute_threshold, brute_result = self._select_best_threshold_bruteforce(prepared_gt, preds) self.assertEqual(fast_threshold, brute_threshold) self.assertEqual(fast_threshold, 0.72) fast_result = paired_bootstrap.evaluate_predictions( prepared_gt, preds, dimension="interactable", iou_threshold=0.75, score_threshold=fast_threshold, ) self.assertEqual(fast_result.metrics, brute_result.metrics) def _select_best_threshold_bruteforce(self, prepared_gt, preds): best_threshold = None best_result = None for threshold in sorted({float(pred["score"]) for pred in preds}, reverse=True): result = paired_bootstrap.evaluate_predictions( prepared_gt, preds, dimension="interactable", iou_threshold=0.75, score_threshold=threshold, ) if best_result is None or ( result.metrics["f1"], result.metrics["precision"], result.metrics["recall"], -threshold, ) > ( best_result.metrics["f1"], best_result.metrics["precision"], best_result.metrics["recall"], -best_threshold, ): best_threshold = threshold best_result = result return best_threshold, best_result def test_semantic_cache_match_maps_numeric_categories(self): with tempfile.TemporaryDirectory() as tmpdir: tmp = Path(tmpdir) cache = tmp / "embedding.json" _write(cache, {"Button": [1.0, 0.0], "Control": [0.9, 0.1]}) matcher = paired_bootstrap.SemanticMatcher(cache) gt = self._toy_gt() preds = [ {"image_id": 1, "category_id": "control", "bbox": [0, 0, 10, 10], "score": 1.0} ] prepared_gt = paired_bootstrap.prepare_ground_truth(gt) evaluated = paired_bootstrap.evaluate_predictions( prepared_gt, preds, dimension="semantics", iou_threshold=0.75, score_threshold=0.0, semantic_matcher=matcher, ) self.assertEqual(evaluated.counts["tp"], 1) def test_semantic_cache_preserves_historical_camelcase_keys(self): with tempfile.TemporaryDirectory() as tmpdir: cache = Path(tmpdir) / "embedding.json" _write(cache, {"QuitButton": [1.0, 0.0], "ConfirmButton": [0.9, 0.1]}) matcher = paired_bootstrap.SemanticMatcher(cache) self.assertTrue(matcher.matches("quit_button", "confirm_button")) def test_semantic_match_defaults_to_historical_raw_dot_product(self): with tempfile.TemporaryDirectory() as tmpdir: cache = Path(tmpdir) / "embedding.json" _write(cache, {"A": [1.0, 0.0], "B": [0.84, 0.20]}) historical = paired_bootstrap.SemanticMatcher(cache) cosine = paired_bootstrap.SemanticMatcher(cache, similarity_mode="cosine") self.assertFalse(historical.matches("a", "b")) self.assertTrue(cosine.matches("a", "b")) def test_semantic_cache_miss_is_hard_error(self): with tempfile.TemporaryDirectory() as tmpdir: cache = Path(tmpdir) / "embedding.json" _write(cache, {"Button": [1.0, 0.0]}) matcher = paired_bootstrap.SemanticMatcher(cache) with self.assertRaisesRegex(KeyError, "Missing frozen embedding cache entries"): matcher.matches("button", "unknown") def test_cli_writes_json_report(self): with tempfile.TemporaryDirectory() as tmpdir: tmp = Path(tmpdir) gt_path = tmp / "gt.json" a_path = tmp / "a.json" b_path = tmp / "b.json" out_path = tmp / "report.json" _write(gt_path, self._toy_gt()) _write(a_path, [{"image_id": 1, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 1.0}]) _write( b_path, [ {"image_id": 1, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 1.0}, {"image_id": 2, "category_id": 2, "bbox": [0, 0, 10, 10], "score": 0.9}, ], ) paired_bootstrap.main( [ "--gt", str(gt_path), "--method-a", str(a_path), "--method-b", str(b_path), "--dimension", "interactable", "--auto-threshold", "--replicates", "100", "--seed", "7", "--output", str(out_path), ] ) report = json.loads(out_path.read_text(encoding="utf-8")) self.assertEqual(report["protocol"]["dimension"], "interactable") self.assertEqual(report["protocol"]["bootstrap_unit"], "image_id") self.assertGreater(report["methods"]["method_b"]["point"]["micro"]["f1"], report["methods"]["method_a"]["point"]["micro"]["f1"]) self.assertIn("mean_per_all_images", report["methods"]["method_a"]["point"]) self.assertIn("mean_per_positive_support_images", report["methods"]["method_a"]["point"]) self.assertIn("mean_per_all_images_delta", report["bootstrap"]) self.assertIn("mean_per_positive_support_images_delta", report["bootstrap"]) self.assertEqual(report["bootstrap"]["delta_direction"], "method_b_minus_method_a") if __name__ == "__main__": unittest.main()