File size: 1,742 Bytes
d61821a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 | from __future__ import annotations
import importlib.util
from pathlib import Path
import unittest
SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "analyze_confirmatory.py"
SPEC = importlib.util.spec_from_file_location("confirmatory_analysis", SCRIPT)
assert SPEC is not None and SPEC.loader is not None
analysis = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(analysis)
class ConfirmatoryAnalysisTests(unittest.TestCase):
def test_exact_sign_flip(self) -> None:
self.assertEqual(analysis.exact_sign_flip([0.0, 0.0]), 1.0)
self.assertEqual(analysis.exact_sign_flip([1.0, 1.0, 1.0]), 0.25)
def test_holm_is_monotone_in_sorted_order(self) -> None:
rows = [
{"p_exact_two_sided": 0.03},
{"p_exact_two_sided": 0.001},
{"p_exact_two_sided": 0.04},
]
analysis.holm(rows)
ordered = sorted(rows, key=lambda row: row["p_exact_two_sided"])
self.assertEqual([row["p_holm"] for row in ordered], [0.003, 0.06, 0.06])
def test_iterative_usage_is_summed(self) -> None:
usage = {
"first": {
"prompt_tokens": 10,
"completion_tokens": 3,
"total_tokens": 13,
"completion_tokens_details": {"reasoning_tokens": 2},
},
"second": {
"prompt_tokens": 20,
"completion_tokens": 5,
"total_tokens": 25,
"completion_tokens_details": {"reasoning_tokens": 4},
},
}
self.assertEqual(
analysis.usage_totals(usage),
{"prompt_tokens": 30, "completion_tokens": 8, "total_tokens": 38, "reasoning_tokens": 6},
)
|