"""Selftest fixtures: each yields a context where the matching check FAILs. Every fixture creates an obviously-temporary broken state under REPO, yields, then fully restores in a `finally` so the repo is not left dirty. """ from __future__ import annotations import contextlib import json import os from audit.core import REPO, sh FIXTURES: dict = {} @contextlib.contextmanager def _f_t1_1(): """Track a symlink (mode 120000) so T1.1 FAILs.""" link = REPO / "_audit_selftest_symlink" if link.exists() or link.is_symlink(): link.unlink() os.symlink("README.md", link) sh(["git", "add", "-f", "_audit_selftest_symlink"]) try: yield finally: sh(["git", "rm", "--cached", "-q", "_audit_selftest_symlink"]) if link.exists() or link.is_symlink(): link.unlink() @contextlib.contextmanager def _f_t1_2(): """Track a >512KB file with an extension not covered by any LFS glob.""" big = REPO / "_audit_selftest_big.dat" big.write_bytes(b"A" * (768 * 1024)) sh(["git", "add", "-f", "_audit_selftest_big.dat"]) try: yield finally: sh(["git", "rm", "--cached", "-q", "_audit_selftest_big.dat"]) if big.exists(): big.unlink() @contextlib.contextmanager def _f_t1_3(): """Track a file containing a key-shaped string so T1.3 FAILs.""" secret = REPO / "_audit_selftest_secret.txt" secret.write_text("token = hf_" + "a1B2c3D4e5F6g7H8i9J0kLmNoP\n", encoding="utf-8") sh(["git", "add", "-f", "_audit_selftest_secret.txt"]) try: yield finally: sh(["git", "rm", "--cached", "-q", "_audit_selftest_secret.txt"]) if secret.exists(): secret.unlink() @contextlib.contextmanager def _f_t1_4(): """Remove the slash-less `rag/corpus` line from .gitignore so T1.4 FAILs.""" gi = REPO / ".gitignore" original = gi.read_text(encoding="utf-8") patched = "\n".join( ln for ln in original.split("\n") if ln != "rag/corpus" ) gi.write_text(patched, encoding="utf-8") try: yield finally: gi.write_text(original, encoding="utf-8") @contextlib.contextmanager def _f_t1_5(): """Track a path containing .DS_Store so T1.5 FAILs.""" d = REPO / "_audit_selftest_dir" d.mkdir(exist_ok=True) junk = d / ".DS_Store" junk.write_bytes(b"\x00junk\x00") sh(["git", "add", "-f", "_audit_selftest_dir/.DS_Store"]) try: yield finally: sh(["git", "rm", "--cached", "-q", "_audit_selftest_dir/.DS_Store"]) if junk.exists(): junk.unlink() if d.exists(): d.rmdir() @contextlib.contextmanager def _f_t2_1(): """Track backend/_audit_st_syntax.py with a SyntaxError so T2.1 FAILs.""" rel = "backend/_audit_st_syntax.py" f = REPO / rel f.write_text("def (:\n pass\n", encoding="utf-8") sh(["git", "add", "-f", rel]) try: yield finally: sh(["git", "rm", "--cached", "-q", rel]) if f.exists(): f.unlink() @contextlib.contextmanager def _f_t2_2(): """Track backend/_audit_st_import.py that raises ImportError at import time.""" rel = "backend/_audit_st_import.py" f = REPO / rel f.write_text('raise ImportError("audit selftest")\n', encoding="utf-8") sh(["git", "add", "-f", rel]) try: yield finally: sh(["git", "rm", "--cached", "-q", rel]) if f.exists(): f.unlink() @contextlib.contextmanager def _f_t2_3(): """Track backend/_audit_st_dead.py with a CODE ref to a deleted module. The line is a real import statement (not a comment/docstring) so T2.3 must classify it as a code_hit and return FAIL, not WARN. """ rel = "backend/_audit_st_dead.py" f = REPO / rel f.write_text("from backend.orchestrator import handle_turn\n", encoding="utf-8") sh(["git", "add", "-f", rel]) try: yield finally: sh(["git", "rm", "--cached", "-q", rel]) if f.exists(): f.unlink() @contextlib.contextmanager def _f_t2_4(): """Track frontend/_audit_st.css with an orphan */ so T2.4 FAILs. `/* a */ b */` — the first `*/` legitimately closes the comment; the trailing ` */` is then an orphan terminator outside any comment, which is exactly the comment-terminator footgun T2.4's state machine flags. """ rel = "frontend/_audit_st.css" f = REPO / rel f.write_text("/* a */ b */\n", encoding="utf-8") sh(["git", "add", "-f", rel]) try: yield finally: sh(["git", "rm", "--cached", "-q", rel]) if f.exists(): f.unlink() @contextlib.contextmanager def _f_t2_5(): """Track backend/_audit_st_path.py with a hardcoded 40-data path so T2.5 FAILs.""" rel = "backend/_audit_st_path.py" f = REPO / rel f.write_text( 'x = settings.CORPUS_DIR.parent.parent / "40-data" / "y.json"\n', encoding="utf-8", ) sh(["git", "add", "-f", rel]) try: yield finally: sh(["git", "rm", "--cached", "-q", rel]) if f.exists(): f.unlink() def _ruff_available() -> bool: return (REPO / ".venv" / "bin" / "ruff").exists() def _tsc_available() -> bool: return (REPO / "frontend" / "node_modules" / ".bin" / "tsc").exists() @contextlib.contextmanager def _f_t2_6(): """Force a lint/type failure so T2.6 FAILs. Prefer a tsc type error (a .ts under frontend/src/ that fails strict type checking). If tsc is unavailable, fall back to a ruff-only fixture (a flagrant unused-import + bare-except .py under audit/). If NEITHER ruff nor tsc is available this single fixture cannot legitimately force a FAIL — raise a clear RuntimeError; the hardened core.selftest treats that raise as the check failing on the broken fixture, which is the acceptable outcome. """ have_ruff = _ruff_available() have_tsc = _tsc_available() if not have_ruff and not have_tsc: raise RuntimeError("T2.6 selftest needs ruff or tsc") created = [] try: if have_tsc: rel = "frontend/src/_audit_st_bad.ts" f = REPO / rel f.write_text('const x: number = "str";\n', encoding="utf-8") sh(["git", "add", "-f", rel]) created.append(rel) else: rel = "audit/_audit_st_lint.py" f = REPO / rel f.write_text("import os\ntry:\n pass\nexcept:\n pass\n", encoding="utf-8") sh(["git", "add", "-f", rel]) created.append(rel) yield finally: for rel in created: sh(["git", "rm", "--cached", "-q", rel]) fp = REPO / rel if fp.exists(): fp.unlink() @contextlib.contextmanager def _f_t3_1(): """Track a temp test that fails so `pytest -q` exits non-zero -> T3.1 FAIL. The file lives under tests/ (testpaths=tests in pytest.ini) AND is named `test_*.py` so pytest's default `python_files` glob actually collects it — a bare `_audit_st_fail.py` is silently skipped from collection, leaving the suite green and T3.1 a false PASS. Restored via `git rm --cached` + unlink so the suite is green again and the repo is byte-identical after. """ rel = "tests/test__audit_st_fail.py" f = REPO / rel f.write_text( "def test_audit_selftest_intentional_fail():\n assert False\n", encoding="utf-8", ) sh(["git", "add", "-f", rel]) try: yield finally: sh(["git", "rm", "--cached", "-q", rel]) if f.exists(): f.unlink() @contextlib.contextmanager def _f_t3_2(): """Track a temp .tsx with a hard syntax error so `next build` fails. The unbalanced/invalid TSX makes Next's compilation step error out fast, so T3.2 FAILs without needing a green full build. Restored so the repo is byte-identical after (this fixture's selftest is inherently a ~minute real build attempt — that is expected). """ rel = "frontend/src/app/_audit_st_bad.tsx" f = REPO / rel f.write_text( "export default function(){ return
}\nconst x: = ;\n", encoding="utf-8", ) sh(["git", "add", "-f", rel]) try: yield finally: sh(["git", "rm", "--cached", "-q", rel]) if f.exists(): f.unlink() @contextlib.contextmanager def _f_t3_3(): """Make `import backend.main` raise so T3.3 FAILs deterministically. T3.3 PASSes only if `import backend.main` succeeds AND :8000 is healthy; it SKIPs when no backend is up. To force a deterministic FAIL regardless of whether a local backend is running, the only sound lever is to make the `import backend.main` subprocess raise. We do that WITHOUT corrupting real source: create a temp tracked module that raises on import, and append a single import line to the END of backend/main.py. The original backend/main.py bytes are captured verbatim BEFORE any mutation and written back EXACTLY in `finally` — even if the check raises mid-way — so backend/main.py is byte-for-byte identical afterwards. A corrupted main.py would be a disaster, so the restore is unconditional and uses the captured raw bytes (not a re-render). """ helper_rel = "backend/_audit_st_importbreak.py" helper = REPO / helper_rel main_py = REPO / "backend" / "main.py" original_bytes = main_py.read_bytes() # capture EXACT bytes first helper.write_text('raise SyntaxError("audit selftest import break")\n', encoding="utf-8") sh(["git", "add", "-f", helper_rel]) try: main_py.write_bytes( original_bytes + b"\nimport backend._audit_st_importbreak # AUDIT-ST\n" ) yield finally: # Restore backend/main.py byte-for-byte, unconditionally. main_py.write_bytes(original_bytes) sh(["git", "rm", "--cached", "-q", helper_rel]) if helper.exists(): helper.unlink() # --------------------------------------------------------------------------- # Tier 4 fixtures. # # T4 checks hit a LIVE local backend / frontend / playwright runner, so a # file-on-disk fixture (the T1–T3 pattern) cannot force a deterministic FAIL. # Instead each fixture monkeypatches the *module seams* of # `audit.tier4_functional` (`_backend_up`, `_get`, `_post`, the module's # `urllib.request.urlopen`, `sh`) so the check exercises its real logic against # a controlled response and returns FAIL. Every patch is captured BEFORE the # yield and restored UNCONDITIONALLY in `finally` — even if the check raises — # so no module attr (and no file) is left mutated. No repo files are touched. # --------------------------------------------------------------------------- @contextlib.contextmanager def _f_t4_1(): """Backend 'up' but /api/health reports a non-ok status -> T4.1 FAIL.""" import audit.tier4_functional as m orig_up, orig_get = m._backend_up, m._get m._backend_up = lambda: True m._get = lambda p, timeout=20: (200, '{"status":"bad"}') try: yield finally: m._backend_up, m._get = orig_up, orig_get @contextlib.contextmanager def _f_t4_2(): """Coverage counts far outside the sane window -> T4.2 FAIL.""" import audit.tier4_functional as m orig_up, orig_get = m._backend_up, m._get m._backend_up = lambda: True m._get = lambda p, timeout=20: ( 200, '{"total_policies":1,"total_insurers":1,"total_chunks":1}', ) try: yield finally: m._backend_up, m._get = orig_up, orig_get @contextlib.contextmanager def _f_t4_3(): """Chat returns an empty reply_text -> T4.3 FAIL (empty reply).""" import audit.tier4_functional as m orig_up, orig_post = m._backend_up, m._post m._backend_up = lambda: True m._post = lambda p, payload, timeout=90: ( 200, '{"reply_text":"","brain_used":"x"}', ) try: yield finally: m._backend_up, m._post = orig_up, orig_post @contextlib.contextmanager def _f_t4_4(): """Make the junk-PDF upload POST 'succeed' (fake 200) -> T4.4 FAIL. T4.4 calls `urllib.request.urlopen` directly (not via `_post`). A real junk upload raises HTTPError(4xx) -> PASS. To force the FAIL branch we patch the `urllib.request.urlopen` *as referenced inside tier4_functional* so the POST returns a fake 200 response instead of raising — i.e. the app 'accepted junk', which T4.4 must report as FAIL. """ import audit.tier4_functional as m orig_up = m._backend_up orig_urlopen = m.urllib.request.urlopen class _FakeResp: status = 200 def read(self): return b"{}" def __enter__(self): return self def __exit__(self, *a): return False def _fake_urlopen(req, *a, **k): return _FakeResp() m._backend_up = lambda: True m.urllib.request.urlopen = _fake_urlopen try: yield finally: m._backend_up = orig_up m.urllib.request.urlopen = orig_urlopen @contextlib.contextmanager def _f_t4_5(): """Profile/session endpoints return HTTP 500 -> T4.5 FAIL.""" import audit.tier4_functional as m orig_up, orig_get, orig_post = m._backend_up, m._get, m._post m._backend_up = lambda: True m._get = lambda p, timeout=20: (500, "err") m._post = lambda p, payload, timeout=90: (500, "err") try: yield finally: m._backend_up, m._get, m._post = orig_up, orig_get, orig_post @contextlib.contextmanager def _f_t4_e2e(): """Frontend 'reachable' + e2e run reports a failed journey -> T4.E2E FAIL. PW_RUN exists on this machine, so `os.path.exists(PW_RUN)` is already True. We patch the module's `urllib.request.urlopen` so the FE :3000 reachability probe 'passes' (fake 200) regardless of whether a dev server is up, and patch the module's `sh` to return a fake CompletedProcess whose stdout is `RJSON {"loads":false}` — a failing journey, which T4.E2E must report as FAIL. No real browser/runner is spawned. """ import subprocess import audit.tier4_functional as m orig_urlopen = m.urllib.request.urlopen orig_sh = m.sh class _FakeResp: status = 200 def read(self): return b"

x

" def __enter__(self): return self def __exit__(self, *a): return False m.urllib.request.urlopen = lambda req, *a, **k: _FakeResp() m.sh = lambda cmd, timeout=120: subprocess.CompletedProcess( cmd, 0, stdout='RJSON {"loads":false}\n', stderr="" ) try: yield finally: m.urllib.request.urlopen = orig_urlopen m.sh = orig_sh # --------------------------------------------------------------------------- # Tier 5 fixtures. # # T5.1/T5.2/T5.4 are file/index-on-disk fixtures (the T1–T3 pattern): create an # obviously-temporary broken tracked state, yield, then fully restore so the # repo is byte-identical afterwards. T5.2 captures the Dockerfile's EXACT bytes # BEFORE any mutation and rewrites them UNCONDITIONALLY first in `finally` # (the T3.3/main.py critical-integrity pattern) — a corrupted Dockerfile would # break the deploy. T5.3 hits the HF Space API, so it monkeypatches the module # seam (`tier5_deploy.urllib.request.urlopen`) to return a controlled # non-RUNNING runtime; the patch is restored UNCONDITIONALLY in `finally`. # T5.3/T5.4's checks carry selftest_expect=Status.WARN, so their fixtures must # make the check RETURN Status.WARN (not raise). # --------------------------------------------------------------------------- @contextlib.contextmanager def _f_t5_1(): """Track _audit_st.parquet as a PLAIN blob (NOT an LFS pointer). `.parquet` is in `.gitattributes` as `filter=lfs`, so a non-pointer `.parquet` blob is exactly the "LFS-globbed but stored raw" state HF's pre-push hook rejects -> T5.1 FAIL. We add it with the lfs clean/smudge filters neutralised so git stores the raw bytes instead of a pointer, then assert it is NOT in `git lfs ls-files`. Restored via `git rm --cached` + unlink so the repo is byte-identical after. """ rel = "_audit_st.parquet" f = REPO / rel f.write_bytes(b"PAR1_audit_selftest_not_a_pointer_PAR1\n") # Empty ALL lfs filters (clean/smudge/process) and drop the `required` # guard so `git add` stores the raw bytes instead of running the LFS # clean filter (which, if only partially neutralised, aborts the add and # silently leaves nothing staged — a false-PASS for T5.1). sh(["git", "-c", "filter.lfs.clean=", "-c", "filter.lfs.smudge=", "-c", "filter.lfs.process=", "-c", "filter.lfs.required=false", "add", "-f", rel]) # Sanity: it must be tracked AND NOT an LFS pointer for the fixture valid. tracked = sh(["git", "ls-files"]).stdout.split() tracked_lfs = sh(["git", "lfs", "ls-files", "-n"]).stdout.split() if rel not in tracked or rel in tracked_lfs: sh(["git", "rm", "--cached", "-qf", rel]) if f.exists(): f.unlink() raise RuntimeError( "T5.1 fixture: _audit_st.parquet not staged as a plain blob " f"(tracked={rel in tracked} lfs={rel in tracked_lfs})") try: yield finally: # staged content differs from HEAD (file is new) -> needs -f. sh(["git", "rm", "--cached", "-qf", rel]) if f.exists(): f.unlink() @contextlib.contextmanager def _f_t5_2(): """Append a `COPY ./x` line to Dockerfile so T5.2 FAILs. The Dockerfile's EXACT original bytes are captured BEFORE any mutation and written back UNCONDITIONALLY as the FIRST statement in `finally` (even if the check raises mid-way) — a corrupted Dockerfile is a deploy disaster, so the restore is the verbatim captured bytes, not a re-render. """ df = REPO / "Dockerfile" original_bytes = df.read_bytes() # capture EXACT bytes first try: df.write_bytes( original_bytes + b"\nCOPY _audit_st_nonexistent_dir ./x\n" ) yield finally: # Restore Dockerfile byte-for-byte, unconditionally, FIRST. df.write_bytes(original_bytes) @contextlib.contextmanager def _f_t5_3(): """Patch tier5_deploy.urllib.request.urlopen so SPACE_API runtime.stage is BUILDING (not RUNNING) -> T5.3 returns Status.WARN. The patched urlopen is captured BEFORE the yield and restored UNCONDITIONALLY in `finally`. No repo files are touched. """ import io import audit.tier5_deploy as m orig_urlopen = m.urllib.request.urlopen payload = json.dumps({"runtime": {"stage": "BUILDING", "sha": "deadbeefcafe0000"}}) def _fake_urlopen(url, *a, **k): # Only intercept the HF Space API call; anything else is unexpected # here (the BUILDING branch returns before any LIVE smoke fetch). return io.BytesIO(payload.encode()) m.urllib.request.urlopen = _fake_urlopen try: yield finally: m.urllib.request.urlopen = orig_urlopen @contextlib.contextmanager def _f_t5_4(): """Track 70-docs/_audit_st_stale.md containing both `Status | Live` and `orchestrator.py` so T5.4's stale-present-state-doc tripwire -> WARN. Restored via `git rm --cached` + unlink so the repo is byte-identical after. """ rel = "70-docs/_audit_st_stale.md" f = REPO / rel f.parent.mkdir(parents=True, exist_ok=True) f.write_text( "| Status | Live |\n\nReferences backend/orchestrator.py here.\n", encoding="utf-8", ) sh(["git", "add", "-f", rel]) try: yield finally: sh(["git", "rm", "--cached", "-q", rel]) if f.exists(): f.unlink() FIXTURES.update({ "T1.1": _f_t1_1, "T1.2": _f_t1_2, "T1.3": _f_t1_3, "T1.4": _f_t1_4, "T1.5": _f_t1_5, "T2.1": _f_t2_1, "T2.2": _f_t2_2, "T2.3": _f_t2_3, "T2.4": _f_t2_4, "T2.5": _f_t2_5, "T2.6": _f_t2_6, "T3.1": _f_t3_1, "T3.2": _f_t3_2, "T3.3": _f_t3_3, "T4.1": _f_t4_1, "T4.2": _f_t4_2, "T4.3": _f_t4_3, "T4.4": _f_t4_4, "T4.5": _f_t4_5, "T4.E2E": _f_t4_e2e, "T5.1": _f_t5_1, "T5.2": _f_t5_2, "T5.3": _f_t5_3, "T5.4": _f_t5_4, })