File size: 3,916 Bytes
bcc8d35 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 | """Thin eval entry: single / paired / golden.
python scripts/eval.py configs/eval.toml --mode=single --model.path=...
ORDER OF OPERATIONS IS LOAD-BEARING
-----------------------------------
1. `apply_resource_guards` runs ONLY when we are not inside a multiprocessing
child. The verifier's worker pool uses 'spawn', which RE-IMPORTS this
module in every worker; a resource guard or an exclusive flock at module
scope therefore executes in each child and kills it on startup. Every
future then dies with BrokenProcessPool -- which is precisely the
documented cause of the "all answers incorrect" masquerade the verifier
raises on. The guard is the fix; `_in_multiprocessing_child()` is what
makes it safe.
2. Banner next: resolved config, dataset hashes, MDE, model hash, backend.
A run that cannot answer its own question does not begin.
3. Then dispatch.
"""
from __future__ import annotations
import argparse
import os
import sys
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if REPO_ROOT not in sys.path:
sys.path.insert(0, REPO_ROOT)
from align.utils.guard import _in_multiprocessing_child # noqa: E402
# See docstring point 1. This condition is not an optimization.
if not _in_multiprocessing_child():
from align.utils.guard import apply_resource_guards
apply_resource_guards(cpu_threads=8, heavy=False, label="eval")
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(description="nmo evaluation entry")
p.add_argument("config", nargs="?", default="configs/eval.toml")
p.add_argument("--mode", choices=("single", "paired", "golden"), default="single")
p.add_argument(
"--out", default=None, help="report path; defaults to outputs/eval/ under the repo"
)
p.add_argument(
"overrides", nargs="*", default=[], help="dotted overrides, e.g. eval.limit=8"
)
return p
def default_out_dir() -> str:
"""Reports land under /apps -- asserted, never assumed."""
from align.utils.guard import assert_paths_off_root
out = os.path.join(REPO_ROOT, "outputs", "eval")
os.makedirs(out, exist_ok=True)
assert_paths_off_root([out])
return out
def main(argv=None) -> int:
args = build_parser().parse_args(argv)
from align.config import load_toml
# Schemaless on purpose (eval.toml has no dataclass): plain nested dict,
# dotted overrides TOML-literal parsed. Same shape OmegaConf.to_object gave.
cfg = load_toml(args.config, list(args.overrides))
out_dir = args.out or default_out_dir()
if args.mode == "golden":
from align.eval.golden import run_golden_regression
banner: dict = {}
path = os.path.join(out_dir, "checkpoint-200-golden.json")
# `run_golden_regression` writes the report ITSELF, before gating, so
# a failing gate still leaves the evidence. Writing again here would
# be dead code on the path that matters: the gate raises, and anything
# after this call never runs.
try:
run_golden_regression(banner=banner, out_path=path)
finally:
# U01-9: name the eval source that produced the report (also when the gate raised), so
# `record_golden_from_report` cannot re-certify it after align/eval/ changes.
if os.path.isfile(path):
from align.train.stage_gates import stamp_golden_report
stamp_golden_report(path)
print(f"golden gate PASSED; report at {path}")
return 0
raise SystemExit(
f"mode {args.mode!r} needs a generation backend, which arrives "
"with section-04 (nmo model) / section-09 (engine). The harness "
"itself is complete and tested: run_eval takes a generate_fn. "
"Use --mode=golden, which uses plain HF generation by design."
)
if __name__ == "__main__":
raise SystemExit(main())
|