| """U10-5: assert_meets_golden's tolerance must be explicit and tightenable.""" | |
| import pytest | |
| from align.eval.harness import assert_meets_golden | |
| GOLDEN = {"gsm8k": 0.258} | |
| def _rep(acc, n=500): | |
| return {"per_set": {"gsm8k": {"n": n, "pass@1": acc}}} | |
| def test_replay_gate_can_be_tight(): | |
| # Default tolerance is the target's own Wilson CI (noisy re-training): a | |
| # 3pp drop passes. A deterministic replay opts into a tight drop bound. | |
| assert_meets_golden(_rep(0.225), GOLDEN) # documented, loose default | |
| with pytest.raises(RuntimeError, match="max_drop"): | |
| assert_meets_golden(_rep(0.225), GOLDEN, max_drop=0.01) | |
| assert_meets_golden(_rep(0.25), GOLDEN, max_drop=0.01) # within 1pp | |
| assert_meets_golden(_rep(0.30), GOLDEN, max_drop=0.01) # above is fine | |
| def test_docstring_states_what_is_actually_checked(): | |
| doc = assert_meets_golden.__doc__ | |
| assert "LOWER BOUND of the TARGET" in doc.replace("\n", " ").replace(" ", " ") | |