File size: 2,102 Bytes
d61821a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | from __future__ import annotations
import importlib.util
from pathlib import Path
import unittest
ROOT = Path(__file__).resolve().parents[1]
SPEC = importlib.util.spec_from_file_location("live_analysis", ROOT / "scripts/analyze_live_agent.py")
assert SPEC and SPEC.loader
analysis = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(analysis)
class LiveAgentAnalysisTests(unittest.TestCase):
def test_exact_mcnemar_uses_discordant_pairs(self) -> None:
n10, n01, value = analysis.exact_mcnemar([1, 1, 1, 0], [0, 0, 1, 1])
self.assertEqual((n10, n01), (2, 1)); self.assertEqual(value, 1.0)
def test_holm_is_monotone_after_sorting(self) -> None:
raw = [0.04, 0.001, 0.02, 0.5]; adjusted = analysis.holm(raw)
ordered = sorted(range(len(raw)), key=raw.__getitem__)
values = [adjusted[index] for index in ordered]
self.assertEqual(values, sorted(values)); self.assertTrue(all(0 <= value <= 1 for value in values))
def test_exact_sign_flip_detects_consistent_direction(self) -> None:
self.assertAlmostEqual(analysis.exact_sign_flip([1.0] * 10), 2 / 1024)
def test_failure_analysis_preserves_terminal_partition(self) -> None:
rows = [
{
"harness_id": "H000", "finished_reason": "finish_tool", "failure_stage": "resolved",
"patch_applied": True, "modified_files": ["a.go"], "protocol_violations": [],
"search_localization_metrics": {"all_gold_in_top_10": True},
"read_localization_metrics": {"all_gold_in_top_10": True},
"resolved_at_1": True, "model_calls": 2, "tool_calls": 3, "test_runs": 1,
"model_switch_count": 1, "model_switch_seconds": 2.0, "elapsed_seconds": 5.0,
"model_elapsed_seconds": 2.0, "tool_counts": {"finish": 1},
}
]
value = analysis.failure_analysis(rows)
self.assertEqual(value["failure_stages"], {"resolved": 1})
self.assertEqual(value["resolutions_given_read_complete"], 1)
if __name__ == "__main__": unittest.main()
|