"""Quality-gate honesty checks. Catches the classes of bug that this repo has been bitten by and that no existing gate prevents: 1. **Bypassed quality gates.** Patterns like `|| echo`, `|| true`, `--no-verify`, `--force` inside scripts that the gates run, or `if : ; then ... else exit 0 ; fi` rationalisations. If the gate swallows a non-zero exit, it's theatre, not a gate. 2. **Fake-working code.** `Math.random()` returning numbers presented as real metrics, `setTimeout(() => fakeData)` simulating a backend, comments like `// TODO`, `// FIXME`, `// HACK`, `// XXX`, `// stub`, `// placeholder`, `// mock` left in shipped TS/TSX/JS code. 3. **Shims and polyfills.** Custom `.d.ts` files under apps/*/src/ that declare modules to paper over upstream packages. The right fix is a pinned/forked/replaced dependency, not a local declaration shim. 4. **Disabled lockfile invariants.** `pnpm.overrides` removed without a paired security advisory note in the commit, or `audit:dev` reverted to swallow exit codes. If any check fails, prints the exact file:line and the rule, and exits non-zero. CI and local gates run this; both must agree. Why this exists: previously `audit:dev` was wired with `pnpm audit --dev || echo '... not blocking'`. The `||` swallowed the non-zero exit code so 20 real CVEs (1 low, 2 moderate, 17 high) sailed through pre-push without ever blocking a commit. That bypass was the symptom; this script is the antibody — it scans every package.json and shell script for the same pattern so a future copy-paste can't reintroduce it. """ from __future__ import annotations import json import re import subprocess import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[2] def tracked_files() -> list[Path]: res = subprocess.run( ["git", "ls-files"], cwd=ROOT, text=True, capture_output=True, check=True ) return [Path(line) for line in res.stdout.splitlines() if line] # --------------------------------------------------------------------------- # Check 1: no quality-gate bypass patterns inside scripts run by gates # --------------------------------------------------------------------------- # Anchor: only flag inside `scripts` blocks of package.json files. Other # fields (descriptions, etc.) are free-form and may legitimately mention # `|| true` in documentation. GATE_BYPASS_PATTERNS = [ # `... || echo '...'` — silences a failed command with a printed message (re.compile(r"\|\|\s*echo\b"), "uses '|| echo' to swallow a non-zero exit"), # `... || true` — silences any failure (re.compile(r"\|\|\s*true\b"), "uses '|| true' to swallow a non-zero exit"), # `... || :` — same as `|| true` but trickier to grep (re.compile(r"\|\|\s*:\s*(?:&&|;|$)"), "uses '|| :' to swallow a non-zero exit"), # `--no-verify` — bypasses git hooks (re.compile(r"--no-verify\b"), "uses '--no-verify' to bypass git hooks"), # `--force` on git push — destroys remote history ( re.compile(r"git\s+push.*--force\b"), "uses 'git push --force' (history destruction)", ), # `set +e` — disables fail-fast in middle of a script (re.compile(r"^\s*set\s+\+e\b"), "uses 'set +e' (disables fail-fast)"), ] # Allowlist: this script is the rule, so it can NAME the patterns it bans. ALLOWLISTED_PATHS = { Path("scripts/quality/check_honesty.py"), Path("scripts/quality/CHECK_HONESTY.md"), } # Files whose `scripts.*` entries are quality gates and must not bypass. QUALITY_GATE_PACKAGE_JSONS = [Path("package.json")] # Quality-gate script names that absolutely must not bypass. GATE_SCRIPT_NAMES = { "audit:prod", "audit:dev", "audit:deps", "build", "lint", "typecheck", "deps:check", "security:check", "quality:pre-commit", "quality:pre-push", "honesty:check", } def check_quality_gate_bypass() -> list[str]: violations: list[str] = [] # 1a. Inspect package.json scripts for bypass patterns. for rel in QUALITY_GATE_PACKAGE_JSONS: path = ROOT / rel if not path.exists(): continue try: data = json.loads(path.read_text(encoding="utf-8")) except json.JSONDecodeError as e: violations.append(f"{rel}: invalid JSON ({e})") continue scripts = data.get("scripts", {}) for name, cmd in scripts.items(): if name not in GATE_SCRIPT_NAMES: continue for pattern, why in GATE_BYPASS_PATTERNS: if pattern.search(cmd): violations.append( f"{rel}:scripts.{name}: {why} -> {cmd!r}" ) # 1b. Inspect shell scripts under .githooks/ — they must not bypass. for rel in tracked_files(): if not rel.parts: continue if rel.parts[0] not in {".githooks", "scripts"}: continue if rel in ALLOWLISTED_PATHS: continue if rel.suffix not in {"", ".sh"}: continue path = ROOT / rel if not path.exists(): continue try: text = path.read_text(encoding="utf-8") except UnicodeDecodeError: continue for line_no, line in enumerate(text.splitlines(), start=1): for pattern, why in GATE_BYPASS_PATTERNS: if pattern.search(line): violations.append(f"{rel}:{line_no}: {why} -> {line.strip()!r}") return violations # --------------------------------------------------------------------------- # Check 2: no fake-working code in shipped TS/TSX/JS # --------------------------------------------------------------------------- FAKE_CODE_DIRS = ("apps/web/src", "apps/workspace/src", "apps/api/src", "apps/mobile/app", "apps/mobile/lib") FAKE_CODE_EXTS = {".ts", ".tsx", ".js", ".jsx"} FAKE_CODE_PATTERNS = [ # Comment markers that signal half-done work in shipped code. # Must be line-leading whitespace + `//` to avoid matching `http://placeholder` # inside string literals. (re.compile(r"(?m)^\s*//\s*(TODO|FIXME|HACK|XXX|STUB|PLACEHOLDER|MOCK)\b", re.I), "left a TODO/FIXME/HACK/XXX/STUB/PLACEHOLDER/MOCK comment"), # Math.random or crypto.randomUUID returning user-visible "metrics" # is a fake-data tell. We ban Math.random outright in shipped code; # legitimate uses (cryptographic salts, IDs) should use crypto APIs. (re.compile(r"\bMath\.random\s*\("), "uses Math.random() in shipped code (fake-data risk; use crypto.randomUUID for IDs)"), # setTimeout that returns mock data (heuristic: a setTimeout literal # body containing `mock`/`fake`/`stub` strings). (re.compile(r"setTimeout\s*\([^)]*\b(mock|fake|stub|placeholder)\b", re.I), "setTimeout simulating a fake async backend"), ] # Allowlist: legitimate uses (test files, dev-only utilities). FAKE_CODE_PATH_ALLOWLIST_PATTERNS = [ re.compile(r"\.test\.(ts|tsx|js|jsx)$"), re.compile(r"\.spec\.(ts|tsx|js|jsx)$"), re.compile(r"__tests__/"), re.compile(r"__mocks__/"), ] def check_fake_code() -> list[str]: violations: list[str] = [] for rel in tracked_files(): if rel.suffix not in FAKE_CODE_EXTS: continue if not any(str(rel).startswith(d) for d in FAKE_CODE_DIRS): continue if any(p.search(str(rel)) for p in FAKE_CODE_PATH_ALLOWLIST_PATTERNS): continue path = ROOT / rel if not path.exists(): continue try: text = path.read_text(encoding="utf-8") except UnicodeDecodeError: continue for line_no, line in enumerate(text.splitlines(), start=1): for pattern, why in FAKE_CODE_PATTERNS: if pattern.search(line): violations.append(f"{rel}:{line_no}: {why}") return violations # --------------------------------------------------------------------------- # Check 3: no .d.ts shims under apps/*/src/ # --------------------------------------------------------------------------- # A shim is a .d.ts file that declares an external module — i.e. it's # papering over a published npm package whose own types are broken. # Project-internal ambient declarations (e.g. CSS modules, *.svg) are # legitimate and don't trigger this. SHIM_DECLARE_RE = re.compile(r'declare\s+module\s+["\']([^"\']+)["\']') SHIM_INTERNAL_PREFIXES = ("*.", "@/", ".", "./", "../") def check_shim_dts() -> list[str]: violations: list[str] = [] for rel in tracked_files(): if rel.suffix != ".ts" or not str(rel).endswith(".d.ts"): continue # Only flag shims under app source directories, not tooling. if not any(str(rel).startswith(d) for d in FAKE_CODE_DIRS): continue path = ROOT / rel if not path.exists(): continue try: text = path.read_text(encoding="utf-8") except UnicodeDecodeError: continue for match in SHIM_DECLARE_RE.finditer(text): module_name = match.group(1) # Skip ambient-asset declarations (those start with * or look # like local file globs) — they aren't shims for npm packages. if module_name.startswith(SHIM_INTERNAL_PREFIXES): continue # Anything else is a declaration for an external package = # a shim for broken upstream types. That's banned in production. line_no = text[: match.start()].count("\n") + 1 violations.append( f"{rel}:{line_no}: declares module {module_name!r} (shim for upstream — " "pin/fork/remove the dep instead)" ) return violations # --------------------------------------------------------------------------- # Check 4: pnpm.overrides invariant — known security pins must remain # --------------------------------------------------------------------------- # When we patched the 20 transitive CVEs (commit 5ebc8e8), we added # pnpm.overrides for 6 packages. If those overrides ever go missing # without `pnpm audit` proving the upstream chain has caught up, the # vulns silently return. This guards the invariant. REQUIRED_OVERRIDES = { "@xmldom/xmldom", "diff", "minimatch", "node-forge", "tar", "yaml", } def check_pnpm_overrides() -> list[str]: violations: list[str] = [] pkg = ROOT / "package.json" if not pkg.exists(): return violations data = json.loads(pkg.read_text(encoding="utf-8")) overrides = data.get("pnpm", {}).get("overrides", {}) missing = REQUIRED_OVERRIDES - set(overrides.keys()) if missing: violations.append( "package.json:pnpm.overrides: missing required security pins for " f"{sorted(missing)} (added in commit 5ebc8e8 to patch 20 transitive CVEs; " "remove only if `pnpm audit --dev` proves the upstream chain has caught up)." ) return violations # --------------------------------------------------------------------------- # Driver # --------------------------------------------------------------------------- def main() -> int: sections: list[tuple[str, list[str]]] = [ ("Quality-gate bypass", check_quality_gate_bypass()), ("Fake-working code", check_fake_code()), ("Shim .d.ts under apps/*/src/", check_shim_dts()), ("Required pnpm.overrides (security pins)", check_pnpm_overrides()), ] total = sum(len(v) for _, v in sections) if total == 0: print("honesty:check OK — no gate bypasses, fake code, shims, or missing security overrides.") return 0 sys.stderr.write("honesty:check FAILED:\n") for name, violations in sections: if not violations: continue sys.stderr.write(f"\n[{name}]\n") for v in violations: sys.stderr.write(f" - {v}\n") sys.stderr.write( "\nFix the root cause. Do NOT add an allowlist entry to silence a check; " "the rule exists because the alternative was a real production incident.\n" ) return 1 if __name__ == "__main__": sys.exit(main())