tests/test_confirmatory_analysis.py

from __future__ import annotations

import importlib.util
from pathlib import Path
import unittest


SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "analyze_confirmatory.py"
SPEC = importlib.util.spec_from_file_location("confirmatory_analysis", SCRIPT)
assert SPEC is not None and SPEC.loader is not None
analysis = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(analysis)


class ConfirmatoryAnalysisTests(unittest.TestCase):
    def test_exact_sign_flip(self) -> None:
        self.assertEqual(analysis.exact_sign_flip([0.0, 0.0]), 1.0)
        self.assertEqual(analysis.exact_sign_flip([1.0, 1.0, 1.0]), 0.25)

    def test_holm_is_monotone_in_sorted_order(self) -> None:
        rows = [
            {"p_exact_two_sided": 0.03},
            {"p_exact_two_sided": 0.001},
            {"p_exact_two_sided": 0.04},
        ]
        analysis.holm(rows)
        ordered = sorted(rows, key=lambda row: row["p_exact_two_sided"])
        self.assertEqual([row["p_holm"] for row in ordered], [0.003, 0.06, 0.06])

    def test_iterative_usage_is_summed(self) -> None:
        usage = {
            "first": {
                "prompt_tokens": 10,
                "completion_tokens": 3,
                "total_tokens": 13,
                "completion_tokens_details": {"reasoning_tokens": 2},
            },
            "second": {
                "prompt_tokens": 20,
                "completion_tokens": 5,
                "total_tokens": 25,
                "completion_tokens_details": {"reasoning_tokens": 4},
            },
        }
        self.assertEqual(
            analysis.usage_totals(usage),
            {"prompt_tokens": 30, "completion_tokens": 8, "total_tokens": 38, "reasoning_tokens": 6},
        )