align / code /tests /test_config_registry.py

Add files using upload-large-folder tool

201dc77 verified 15 days ago

4.29 kB

"""Config + launch banner. Run: uv run pytest tests/test_config_registry.py -v -m "not gpu"
The banner contract: NO GPU run starts without printing every resolved value
it depends on. A run that cannot answer its question does not start.
"""

from __future__ import annotations

import math
import os

import pytest

from align import registry
from align.config import default_full_ft_config, load_config


def _complete_resolved() -> dict:
    """A synthetic fully-resolved value set covering every mandatory key.
    Built by iterating registry.MANDATORY_BANNER_KEYS so the test tracks the
    list as data instead of hardcoding a parallel copy."""
    samples = {
        "attention_backend": "sdpa_flash",
        "loss_type": "dpo_sigmoid",
        "beta": 0.1,
        "lr": 5e-7,
        "scheduler_horizon_optimizer_steps": 1500,
        "dataset_hashes": {"train": "abc123", "eval_gsm8k": "def456"},
        "ceiling_results": {"extraction_roundtrip": "PASS"},
        "rss_budget": {"total_gb": 108.0, "optimizer_state_gb": 54.0},
        "pinned_staging_gb": 1.8,
        "probe_step_time_s": float("nan"),  # explicit NaN, never absent
        "mde": {"n": 1319, "mde_pp": 2.8},
    }
    missing = [k for k in registry.MANDATORY_BANNER_KEYS if k not in samples]
    assert not missing, f"test sample set out of date with registry: {missing}"
    return samples


def test_banner_contains_every_mandatory_key():
    """Rendered banner for a complete value set contains ALL mandatory keys,
    each with its resolved value on the same line."""
    resolved = _complete_resolved()
    banner = registry.render_banner(resolved)
    for key in registry.MANDATORY_BANNER_KEYS:
        assert key in banner, f"banner missing key {key}"
    assert "sdpa_flash" in banner
    assert "nan" in banner.lower()  # explicit NaN is printed, not dropped


def test_banner_missing_key_raises():
    """A value set missing ANY mandatory key -- or carrying None for one --
    RAISES rather than printing a partial banner. Silent omission is the
    root shape of every predecessor silent failure: a missing value replaced
    by a plausible one."""
    for key in registry.MANDATORY_BANNER_KEYS:
        resolved = _complete_resolved()
        del resolved[key]
        with pytest.raises(RuntimeError, match=key):
            registry.render_banner(resolved)

    resolved = _complete_resolved()
    resolved["beta"] = None
    with pytest.raises(RuntimeError, match="beta"):
        registry.render_banner(resolved)


def test_cli_dotted_overrides(tmp_workdir):
    """`scripts/x.py configs/x.toml --optimizer.lr=1e-5`: dotted overrides
    reach the dataclass field with correct type; unknown keys raise under
    struct mode, never silently create new fields."""
    toml_path = os.path.join(tmp_workdir, "t.toml")
    with open(toml_path, "w") as f:
        f.write("[optimizer]\nlr = 1.0e-6\n")

    cfg = load_config(toml_path, ["optimizer.lr=1e-5"])
    assert isinstance(cfg.optimizer.lr, float)
    assert cfg.optimizer.lr == pytest.approx(1e-5)

    with pytest.raises(Exception, match="optimzer|struct|key"):
        load_config(toml_path, ["optimzer.lr=1e-5"])  # typo'd section

    with pytest.raises(Exception, match="not_a_field|struct|key"):
        load_config(toml_path, ["optimizer.not_a_field=3"])


def test_undefined_statistic_is_nan_never_zero(tmp_workdir):
    """Tracker contract: an undefined statistic is logged as NaN, NEVER 0 and
    never omitted. Incident: an EV clamp turned an undefined explained
    variance into -1.8e9 while the critic looked healthy; three analyses were
    later retracted for trusting plausible stand-in numbers."""
    from align.utils.tracking import Tracker

    t = Tracker(run_dir=tmp_workdir, stage="test", mandatory_keys=("loss", "margin"))
    t.log(step=0, metrics={"loss": 1.0, "margin": float("nan")})

    with pytest.raises(RuntimeError, match="margin"):
        t.log(step=1, metrics={"loss": 1.0})  # omission raises

    with pytest.raises(RuntimeError, match="None"):
        t.log(step=2, metrics={"loss": 1.0, "margin": None})  # None raises

    import json

    lines = [json.loads(ln) for ln in open(os.path.join(tmp_workdir, "history.jsonl"))]
    assert len(lines) == 1
    assert math.isnan(lines[0]["margin"])

	"""Config + launch banner. Run: uv run pytest tests/test_config_registry.py -v -m "not gpu"

	The banner contract: NO GPU run starts without printing every resolved value
	it depends on. A run that cannot answer its question does not start.
	"""

	from __future__ import annotations

	import math
	import os

	import pytest

	from align import registry
	from align.config import default_full_ft_config, load_config


	def _complete_resolved() -> dict:
	"""A synthetic fully-resolved value set covering every mandatory key.
	Built by iterating registry.MANDATORY_BANNER_KEYS so the test tracks the
	list as data instead of hardcoding a parallel copy."""
	samples = {
	"attention_backend": "sdpa_flash",
	"loss_type": "dpo_sigmoid",
	"beta": 0.1,
	"lr": 5e-7,
	"scheduler_horizon_optimizer_steps": 1500,
	"dataset_hashes": {"train": "abc123", "eval_gsm8k": "def456"},
	"ceiling_results": {"extraction_roundtrip": "PASS"},
	"rss_budget": {"total_gb": 108.0, "optimizer_state_gb": 54.0},
	"pinned_staging_gb": 1.8,
	"probe_step_time_s": float("nan"), # explicit NaN, never absent
	"mde": {"n": 1319, "mde_pp": 2.8},
	}
	missing = [k for k in registry.MANDATORY_BANNER_KEYS if k not in samples]
	assert not missing, f"test sample set out of date with registry: {missing}"
	return samples


	def test_banner_contains_every_mandatory_key():
	"""Rendered banner for a complete value set contains ALL mandatory keys,
	each with its resolved value on the same line."""
	resolved = _complete_resolved()
	banner = registry.render_banner(resolved)
	for key in registry.MANDATORY_BANNER_KEYS:
	assert key in banner, f"banner missing key {key}"
	assert "sdpa_flash" in banner
	assert "nan" in banner.lower() # explicit NaN is printed, not dropped


	def test_banner_missing_key_raises():
	"""A value set missing ANY mandatory key -- or carrying None for one --
	RAISES rather than printing a partial banner. Silent omission is the
	root shape of every predecessor silent failure: a missing value replaced
	by a plausible one."""
	for key in registry.MANDATORY_BANNER_KEYS:
	resolved = _complete_resolved()
	del resolved[key]
	with pytest.raises(RuntimeError, match=key):
	registry.render_banner(resolved)

	resolved = _complete_resolved()
	resolved["beta"] = None
	with pytest.raises(RuntimeError, match="beta"):
	registry.render_banner(resolved)


	def test_cli_dotted_overrides(tmp_workdir):
	"""`scripts/x.py configs/x.toml --optimizer.lr=1e-5`: dotted overrides
	reach the dataclass field with correct type; unknown keys raise under
	struct mode, never silently create new fields."""
	toml_path = os.path.join(tmp_workdir, "t.toml")
	with open(toml_path, "w") as f:
	f.write("[optimizer]\nlr = 1.0e-6\n")

	cfg = load_config(toml_path, ["optimizer.lr=1e-5"])
	assert isinstance(cfg.optimizer.lr, float)
	assert cfg.optimizer.lr == pytest.approx(1e-5)

	with pytest.raises(Exception, match="optimzer\|struct\|key"):
	load_config(toml_path, ["optimzer.lr=1e-5"]) # typo'd section

	with pytest.raises(Exception, match="not_a_field\|struct\|key"):
	load_config(toml_path, ["optimizer.not_a_field=3"])


	def test_undefined_statistic_is_nan_never_zero(tmp_workdir):
	"""Tracker contract: an undefined statistic is logged as NaN, NEVER 0 and
	never omitted. Incident: an EV clamp turned an undefined explained
	variance into -1.8e9 while the critic looked healthy; three analyses were
	later retracted for trusting plausible stand-in numbers."""
	from align.utils.tracking import Tracker

	t = Tracker(run_dir=tmp_workdir, stage="test", mandatory_keys=("loss", "margin"))
	t.log(step=0, metrics={"loss": 1.0, "margin": float("nan")})

	with pytest.raises(RuntimeError, match="margin"):
	t.log(step=1, metrics={"loss": 1.0}) # omission raises

	with pytest.raises(RuntimeError, match="None"):
	t.log(step=2, metrics={"loss": 1.0, "margin": None}) # None raises

	import json

	lines = [json.loads(ln) for ln in open(os.path.join(tmp_workdir, "history.jsonl"))]
	assert len(lines) == 1
	assert math.isnan(lines[0]["margin"])