File size: 4,290 Bytes
201dc77 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 | """Config + launch banner. Run: uv run pytest tests/test_config_registry.py -v -m "not gpu"
The banner contract: NO GPU run starts without printing every resolved value
it depends on. A run that cannot answer its question does not start.
"""
from __future__ import annotations
import math
import os
import pytest
from align import registry
from align.config import default_full_ft_config, load_config
def _complete_resolved() -> dict:
"""A synthetic fully-resolved value set covering every mandatory key.
Built by iterating registry.MANDATORY_BANNER_KEYS so the test tracks the
list as data instead of hardcoding a parallel copy."""
samples = {
"attention_backend": "sdpa_flash",
"loss_type": "dpo_sigmoid",
"beta": 0.1,
"lr": 5e-7,
"scheduler_horizon_optimizer_steps": 1500,
"dataset_hashes": {"train": "abc123", "eval_gsm8k": "def456"},
"ceiling_results": {"extraction_roundtrip": "PASS"},
"rss_budget": {"total_gb": 108.0, "optimizer_state_gb": 54.0},
"pinned_staging_gb": 1.8,
"probe_step_time_s": float("nan"), # explicit NaN, never absent
"mde": {"n": 1319, "mde_pp": 2.8},
}
missing = [k for k in registry.MANDATORY_BANNER_KEYS if k not in samples]
assert not missing, f"test sample set out of date with registry: {missing}"
return samples
def test_banner_contains_every_mandatory_key():
"""Rendered banner for a complete value set contains ALL mandatory keys,
each with its resolved value on the same line."""
resolved = _complete_resolved()
banner = registry.render_banner(resolved)
for key in registry.MANDATORY_BANNER_KEYS:
assert key in banner, f"banner missing key {key}"
assert "sdpa_flash" in banner
assert "nan" in banner.lower() # explicit NaN is printed, not dropped
def test_banner_missing_key_raises():
"""A value set missing ANY mandatory key -- or carrying None for one --
RAISES rather than printing a partial banner. Silent omission is the
root shape of every predecessor silent failure: a missing value replaced
by a plausible one."""
for key in registry.MANDATORY_BANNER_KEYS:
resolved = _complete_resolved()
del resolved[key]
with pytest.raises(RuntimeError, match=key):
registry.render_banner(resolved)
resolved = _complete_resolved()
resolved["beta"] = None
with pytest.raises(RuntimeError, match="beta"):
registry.render_banner(resolved)
def test_cli_dotted_overrides(tmp_workdir):
"""`scripts/x.py configs/x.toml --optimizer.lr=1e-5`: dotted overrides
reach the dataclass field with correct type; unknown keys raise under
struct mode, never silently create new fields."""
toml_path = os.path.join(tmp_workdir, "t.toml")
with open(toml_path, "w") as f:
f.write("[optimizer]\nlr = 1.0e-6\n")
cfg = load_config(toml_path, ["optimizer.lr=1e-5"])
assert isinstance(cfg.optimizer.lr, float)
assert cfg.optimizer.lr == pytest.approx(1e-5)
with pytest.raises(Exception, match="optimzer|struct|key"):
load_config(toml_path, ["optimzer.lr=1e-5"]) # typo'd section
with pytest.raises(Exception, match="not_a_field|struct|key"):
load_config(toml_path, ["optimizer.not_a_field=3"])
def test_undefined_statistic_is_nan_never_zero(tmp_workdir):
"""Tracker contract: an undefined statistic is logged as NaN, NEVER 0 and
never omitted. Incident: an EV clamp turned an undefined explained
variance into -1.8e9 while the critic looked healthy; three analyses were
later retracted for trusting plausible stand-in numbers."""
from align.utils.tracking import Tracker
t = Tracker(run_dir=tmp_workdir, stage="test", mandatory_keys=("loss", "margin"))
t.log(step=0, metrics={"loss": 1.0, "margin": float("nan")})
with pytest.raises(RuntimeError, match="margin"):
t.log(step=1, metrics={"loss": 1.0}) # omission raises
with pytest.raises(RuntimeError, match="None"):
t.log(step=2, metrics={"loss": 1.0, "margin": None}) # None raises
import json
lines = [json.loads(ln) for ln in open(os.path.join(tmp_workdir, "history.jsonl"))]
assert len(lines) == 1
assert math.isnan(lines[0]["margin"])
|