| from __future__ import annotations |
|
|
| import importlib.util |
| from pathlib import Path |
| import unittest |
|
|
|
|
| ROOT = Path(__file__).resolve().parents[1] |
| SPEC = importlib.util.spec_from_file_location("live_analysis", ROOT / "scripts/analyze_live_agent.py") |
| assert SPEC and SPEC.loader |
| analysis = importlib.util.module_from_spec(SPEC) |
| SPEC.loader.exec_module(analysis) |
|
|
|
|
| class LiveAgentAnalysisTests(unittest.TestCase): |
| def test_exact_mcnemar_uses_discordant_pairs(self) -> None: |
| n10, n01, value = analysis.exact_mcnemar([1, 1, 1, 0], [0, 0, 1, 1]) |
| self.assertEqual((n10, n01), (2, 1)); self.assertEqual(value, 1.0) |
|
|
| def test_holm_is_monotone_after_sorting(self) -> None: |
| raw = [0.04, 0.001, 0.02, 0.5]; adjusted = analysis.holm(raw) |
| ordered = sorted(range(len(raw)), key=raw.__getitem__) |
| values = [adjusted[index] for index in ordered] |
| self.assertEqual(values, sorted(values)); self.assertTrue(all(0 <= value <= 1 for value in values)) |
|
|
| def test_exact_sign_flip_detects_consistent_direction(self) -> None: |
| self.assertAlmostEqual(analysis.exact_sign_flip([1.0] * 10), 2 / 1024) |
|
|
| def test_failure_analysis_preserves_terminal_partition(self) -> None: |
| rows = [ |
| { |
| "harness_id": "H000", "finished_reason": "finish_tool", "failure_stage": "resolved", |
| "patch_applied": True, "modified_files": ["a.go"], "protocol_violations": [], |
| "search_localization_metrics": {"all_gold_in_top_10": True}, |
| "read_localization_metrics": {"all_gold_in_top_10": True}, |
| "resolved_at_1": True, "model_calls": 2, "tool_calls": 3, "test_runs": 1, |
| "model_switch_count": 1, "model_switch_seconds": 2.0, "elapsed_seconds": 5.0, |
| "model_elapsed_seconds": 2.0, "tool_counts": {"finish": 1}, |
| } |
| ] |
| value = analysis.failure_analysis(rows) |
| self.assertEqual(value["failure_stages"], {"resolved": 1}) |
| self.assertEqual(value["resolutions_given_read_complete"], 1) |
|
|
|
|
| if __name__ == "__main__": unittest.main() |
|
|
