"""WIN2-R2 / S92 FIX 1: a chain that stops outside a window's own HOLD restores production (CPU only). S92's counterexamples (scratch/S92/work/test_s92_counterexamples.py at 48bc7a7) are kept here inverted: each now shows the fixed behaviour. Every host probe is a fake: production on an unreachable loopback port, ssh a script in tmp_path. """ from __future__ import annotations import json import os import re import subprocess import sys import time from collections.abc import Callable, Mapping from pathlib import Path import pytest from offload.launch.windows import chain, hold_admission from offload.prefill_maxlen.evidence import Json, record REPO = Path(__file__).resolve().parents[4] WINDOWS = REPO / 'offload/launch/windows' OWNER = 'chain-20261027T220000Z-0123abcd' @pytest.fixture(autouse=True) def isolate(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: marker = tmp_path / 'CHAINED' monkeypatch.setattr(chain, 'MARKER', marker) for k in ('WINDOW_CHAIN', 'WINDOW_CHAIN_LAST'): monkeypatch.delenv(k, raising=False) return marker def _chain_left_production_down(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv('WINDOW_CHAIN', '1') chain.finish_chained(tmp_path / 'W-prev', {'sparkhead': 'QUIET', 'sparkworker-wired': 'QUIET'}, placeholder_owner=OWNER) # ---------------------------------------------------------------- end decision def test_end_action_table(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: assert chain.end_action(stopped=True, hold_written=False) == 'none' # no chain, nothing touched assert chain.end_action(stopped=True, hold_written=True) == 'restore' assert not chain.restore_required(stopped=True, hold_written=False) _chain_left_production_down(tmp_path, monkeypatch) assert chain.end_action(stopped=False, hold_written=True) == 'chained' assert chain.end_action(stopped=True, hold_written=True) == 'restore' assert chain.end_action(stopped=True, hold_written=False) == 'abort' # S92: production down, nothing restored assert chain.restore_required(stopped=True, hold_written=False) monkeypatch.setenv('WINDOW_CHAIN_LAST', '1') assert chain.end_action(stopped=False, hold_written=True) == 'restore' def test_torn_marker_still_requires_restore(isolate: Path) -> None: isolate.write_text('{"schema": "dsv41-chain-v1"') assert chain.marker_present() and chain.end_action(stopped=True, hold_written=False) == 'abort' # ---------------------------------------------------------------- chained start with production down def test_chained_start_admits_hold_with_production_down(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """S92: runners opened the admission HOLD in 'require-serving' mode, so a chained window failed before its HOLD.""" monkeypatch.setenv('HOME', str(tmp_path)) monkeypatch.setenv('SCOPED_REMOTE_DEADLINE', '9999999999') (tmp_path / 'dsv41-deploy').mkdir() def down(path: str) -> str: raise subprocess.CalledProcessError(7, ['curl', path]) monkeypatch.setattr(hold_admission, 'fetch', down) monkeypatch.setattr(time, 'sleep', lambda s: None) # hold_admission's drain poll with pytest.raises(subprocess.CalledProcessError): hold_admission.admit('r2-tag', '0' * 32, 'WIN2-R2 test', 30.0, allow_unreachable=chain.allow_unreachable('production')) assert not (tmp_path / 'dsv41-deploy/HOLD').exists() hold_admission.admit('r2-tag', '1' * 32, 'WIN2-R2 test', 30.0, allow_unreachable=chain.allow_unreachable('chained')) assert (tmp_path / 'dsv41-deploy/HOLD').exists() with pytest.raises(ValueError, match='unknown chain preflight mode'): chain.allow_unreachable('serving') # ---------------------------------------------------------------- abort helper class Fakes: def __init__(self, *, hold: bool = False, recover_error: Exception | None = None, watch: str = 'PASS', serving: bool = False) -> None: self.hold, self.recover_error, self.watch_status, self.serving = hold, recover_error, watch, serving self.recovered = self.watched = self.ended = 0 self.lines: list[str] = [] def hold_probe(self) -> bool: return self.hold def recover(self, out: Path, log: Callable[[str], None]) -> Mapping[str, Json]: self.recovered += 1 if self.recover_error is not None: raise self.recover_error return {'start': 'PASS', 'check': 'PASS', 'close': 'PASS'} def watch_fn(self, out: Path, log: Callable[[str], None]) -> Mapping[str, Json]: self.watched += 1 return {'status': self.watch_status, 'polls': 30, 'failures': 0, 'recoveries': 0} def end(self) -> Mapping[str, Json]: self.ended += 1 return chain.end_chain(run=lambda host, command: (0, 'PLACEHOLDER DOWN\n')) def run(self, out: Path) -> dict[str, Json]: return chain.abort(out, hold=self.hold_probe, recover=self.recover, watch=self.watch_fn, end=self.end, log=self.lines.append, serving=lambda: self.serving) def _prev_ok_accepts(summary: Path) -> bool: """The exact summary test of chain_prev_ok.sh / prev-ok.sh.""" s = json.loads(summary.read_text()) chained = s.get('chain') == 'CHAINED' restored = (str(s.get('restore', {}).get('status', '')).startswith('RESTORED') and str(s.get('post_close_watch', {}).get('status', '')) == 'PASS') return chained or restored def test_pre_hold_failure_in_chain_restores_and_clears(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """S92: a chained window failing before its HOLD said 'NOT NEEDED' and the next preflight said 'chained' again.""" _chain_left_production_down(tmp_path, monkeypatch) stopped, hold_written = True, False assert chain.end_action(stopped, hold_written) == 'abort' fakes = Fakes() result = fakes.run(tmp_path / 'W-now' / 'chain-abort') assert fakes.recovered == 1 and fakes.watched == 1 and fakes.ended == 1 assert str(result['status']).startswith('RESTORED') and chain.read_marker() is None assert 'placeholders down' in str(result['status']) and record(result['end_chain'])['placeholder_owner'] == OWNER assert record(result['restore'])['status'] == chain.ABORT_RESTORED assert _prev_ok_accepts(tmp_path / 'W-now' / 'chain-abort' / 'summary.json') with pytest.raises(ValueError, match='not serving and no chain marker'): chain.preflight(production_serving=False, hold_exists=False) # never 'chained' again assert chain.preflight(production_serving=True, hold_exists=False) == 'production' def test_abort_with_hold_alerts_and_touches_nothing(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: _chain_left_production_down(tmp_path, monkeypatch) fakes = Fakes(hold=True) result = fakes.run(tmp_path / 'abort') assert str(result['status']).startswith('ALERT') and fakes.recovered == fakes.watched == 0 assert chain.read_marker() is not None and not _prev_ok_accepts(tmp_path / 'abort' / 'summary.json') @pytest.mark.parametrize('fakes', [Fakes(recover_error=ValueError('W8 check: greedy is not 6/6')), Fakes(watch='FAILED: x')]) def test_abort_keeps_marker_unless_restored_and_watch_pass(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fakes: Fakes) -> None: _chain_left_production_down(tmp_path, monkeypatch) result = fakes.run(tmp_path / 'abort') assert not str(result['status']).startswith('RESTORED: chain ended') assert chain.read_marker() is not None and not _prev_ok_accepts(tmp_path / 'abort' / 'summary.json') assert fakes.ended == 0 # production unproved: the chain placeholders stay up def test_abort_without_marker_does_nothing(tmp_path: Path) -> None: fakes = Fakes() result = fakes.run(tmp_path / 'abort') assert str(result['status']).startswith('NOT NEEDED') and fakes.recovered == 0 def test_abort_cli_is_coordinator_only_and_exit_codes(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv('WINDOW_COORDINATOR', raising=False) assert chain.main(['abort', '--out', str(tmp_path / 'a')]) == 2 monkeypatch.setenv('WINDOW_COORDINATOR', chain.COORDINATOR) assert chain.main(['abort', '--out', str(tmp_path / 'b')]) == 0 # no marker: NOT NEEDED _chain_left_production_down(tmp_path, monkeypatch) fakes = Fakes(hold=True) real = chain.abort monkeypatch.setattr(chain, 'abort', lambda out: real(out, hold=fakes.hold_probe, recover=fakes.recover, watch=fakes.watch_fn, end=fakes.end, log=fakes.lines.append, serving=lambda: False)) assert chain.main(['abort', '--out', str(tmp_path / 'c')]) == 4 fakes.hold, fakes.watch_status = False, 'FAILED: recurrence' assert chain.main(['abort', '--out', str(tmp_path / 'd')]) == 3 fakes.watch_status = 'PASS' assert chain.main(['abort', '--out', str(tmp_path / 'e')]) == 4 and fakes.recovered == 1 # S96 FIX 2: once assert chain.main(['abort', '--again', '--out', str(tmp_path / 'f')]) == 0 and chain.read_marker() is None assert fakes.recovered == 2 and not chain.abort_attempt().exists() # ---------------------------------------------------------------- chain_prev_ok.sh end to end (fakes only) def _fake_campaign(tmp_path: Path, stale_rc: int) -> Path: camp = tmp_path / 'campaign' (camp / 'launch').mkdir(parents=True) script = camp / 'launch/no-stale-containers.sh' script.write_text(f'#!/bin/bash\necho stale-check rc={stale_rc}\nexit {stale_rc}\n') script.chmod(0o755) return camp def _gate_env(tmp_path: Path, marker: Path, *, stale_rc: int, hold: bool = False) -> dict[str, str]: bin_dir = tmp_path / 'bin' bin_dir.mkdir(exist_ok=True) ssh = bin_dir / 'ssh' # HOLD probe: exit 1 = a HOLD exists. Placeholder probe (absolute tmux): this chain's placeholder is up. ssh.write_text(f'#!/bin/bash\necho "fake ssh $*" >> {tmp_path}/ssh.log\n' f'case "$*" in *"{chain.PLACEHOLDER_TMUX}"*) echo "PLACEHOLDER UP owner={OWNER} pid=4242"; exit 0 ;; esac\n' f'exit {1 if hold else 0}\n') ssh.chmod(0o755) abort = bin_dir / 'abort.sh' abort.write_text(f'#!/bin/sh\necho ABORT-CALLED >> {tmp_path}/abort.log\necho fake abort ran\nexit 0\n') abort.chmod(0o755) env = {k: v for k, v in os.environ.items() if k not in ('WINDOW_CHAIN_LAST', 'TMUX', 'TMUX_PANE')} env.update({'PATH': f'{bin_dir}:{env["PATH"]}', 'WINDOW_CHAIN': '1', 'DSV41_CHAIN_MARKER': str(marker), 'WINDOW_COORDINATOR': chain.COORDINATOR, 'DSV41_CHAIN_PRODUCTION_URL': 'http://127.0.0.1:9/v1/models', 'DSV41_CHAIN_PY': sys.executable, 'DSV41_CHAIN_CAMPAIGN': str(_fake_campaign(tmp_path, stale_rc)), 'DSV41_CHAIN_ABORT': str(abort)}) return env def _prev(tmp_path: Path, summary: dict[str, Json]) -> Path: prev = tmp_path / 'W-prev' prev.mkdir(exist_ok=True) (prev / 'summary.json').write_text(json.dumps(summary)) prev_file = tmp_path / 'prev.txt' prev_file.write_text(str(prev) + '\n') return prev_file def _gate(env: dict[str, str], prev_file: Path | str) -> subprocess.CompletedProcess[str]: return subprocess.run(['bash', str(WINDOWS / 'chain_prev_ok.sh'), 'win2r2-no-such-tmux', str(prev_file)], capture_output=True, text=True, env=env, timeout=120, cwd=REPO) CHAINED_SUMMARY: dict[str, Json] = {'chain': 'CHAINED', 'restore': {'status': 'CHAINED: production left down'}} def test_gate_refusal_in_chained_state_exits_3_and_takes_abort_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, isolate: Path) -> None: """S92: the stale-container refusal in the chained state exited 2, ran no W8 and left production down.""" _chain_left_production_down(tmp_path, monkeypatch) r = _gate(_gate_env(tmp_path, isolate, stale_rc=2), _prev(tmp_path, CHAINED_SUMMARY)) assert 'chain gate: chained' in r.stdout, r.stdout + r.stderr assert r.returncode == 3 and 'CHAIN STOPPED: PRODUCTION DOWN' in r.stdout assert (tmp_path / 'abort.log').read_text().splitlines() == ['ABORT-CALLED'] @pytest.mark.parametrize('case', ['bad-summary', 'never-started', 'hold']) def test_every_refusal_with_marker_exits_3(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, isolate: Path, case: str) -> None: _chain_left_production_down(tmp_path, monkeypatch) env = _gate_env(tmp_path, isolate, stale_rc=0, hold=case == 'hold') summary: dict[str, Json] = {'restore': {'status': 'FAILED'}} if case == 'bad-summary' else CHAINED_SUMMARY prev_file: Path | str = _prev(tmp_path, summary) if case == 'never-started': prev_file = tmp_path / 'empty.txt' prev_file.write_text('') r = _gate(env, prev_file) assert r.returncode == 3 and 'CHAIN STOPPED: PRODUCTION DOWN' in r.stdout, r.stdout + r.stderr assert (tmp_path / 'abort.log').exists() def test_refusal_without_marker_exits_2_and_never_aborts(tmp_path: Path, isolate: Path) -> None: r = _gate(_gate_env(tmp_path, isolate, stale_rc=0), '-') # production down, no marker: STOP as prev-ok assert r.returncode == 2 and 'CHAIN STOPPED' not in r.stdout and not (tmp_path / 'abort.log').exists() def test_clean_chained_gate_admits(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, isolate: Path) -> None: _chain_left_production_down(tmp_path, monkeypatch) r = _gate(_gate_env(tmp_path, isolate, stale_rc=0), _prev(tmp_path, CHAINED_SUMMARY)) assert r.returncode == 0 and 'chain gate: chained' in r.stdout and not (tmp_path / 'abort.log').exists() def test_chain_abort_sh_without_marker_is_a_no_op(tmp_path: Path, isolate: Path) -> None: env = _gate_env(tmp_path, isolate, stale_rc=0) env['WINDOW_COORDINATOR'] = chain.COORDINATOR r = subprocess.run(['bash', str(WINDOWS / 'chain_abort.sh'), str(tmp_path / 'abort-out')], capture_output=True, text=True, env=env, timeout=120, cwd=REPO) assert r.returncode == 0 and 'NOT NEEDED' in r.stdout, r.stdout + r.stderr assert json.loads((tmp_path / 'abort-out/summary.json').read_text())['marker_path'] == str(isolate) env.pop('WINDOW_COORDINATOR') r = subprocess.run(['bash', str(WINDOWS / 'chain_abort.sh'), str(tmp_path / 'abort-out2')], capture_output=True, text=True, env=env, timeout=120, cwd=REPO) assert r.returncode == 2 and 'coordinator-only' in r.stderr # ---------------------------------------------------------------- S92 positive checks kept as regression tests def test_corrupt_or_partial_marker_fails_closed(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, isolate: Path) -> None: monkeypatch.setenv('WINDOW_CHAIN', '1') for text in ('', '{"schema": "dsv41-chain-v1"', 'null'): isolate.write_text(text) with pytest.raises(ValueError): chain.preflight(production_serving=False, hold_exists=False) isolate.unlink() (tmp_path / 'CHAINED.new').write_text('{}') # crash between temp write and rename with pytest.raises(ValueError, match='no chain marker'): chain.preflight(production_serving=False, hold_exists=False) chain.clear() assert not (tmp_path / 'CHAINED.new').exists() @pytest.mark.parametrize('serving,hold', [(True, False), (False, True), (True, True)]) def test_marker_never_admits_live_production_or_hold(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, serving: bool, hold: bool) -> None: _chain_left_production_down(tmp_path, monkeypatch) for last in ('0', '1'): monkeypatch.setenv('WINDOW_CHAIN_LAST', last) with pytest.raises(ValueError, match='STOP'): chain.preflight(production_serving=serving, hold_exists=hold) def test_failed_hold_probe_counts_as_hold(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: bin_dir = tmp_path / 'bin' bin_dir.mkdir() (bin_dir / 'ssh').write_text('#!/bin/sh\nexit 255\n') (bin_dir / 'ssh').chmod(0o755) monkeypatch.setenv('PATH', f'{bin_dir}:{os.environ["PATH"]}') _chain_left_production_down(tmp_path, monkeypatch) assert chain.hold_exists() is True with pytest.raises(ValueError, match='HOLD'): chain.gate(lambda: False) def test_recovery_tags_are_unique_and_valid() -> None: a = chain._tag(Path('/c/PC1-threshold2048-W-20261027T153817Z/chain-abort'), 'abort') b = chain._tag(Path('/c/NBW2-W-20261027T210000Z/chain-abort'), 'abort') long = chain._tag(Path('/c/' + 'X' * 120 + '/chain-abort'), 'watch') assert a != b and a.endswith('-chain-abort-abort') and len(long) <= 64 assert all(re.fullmatch('[A-Za-z0-9_][A-Za-z0-9_-]{0,63}', t) for t in (a, b, long))