scripts/quality/check_honesty.py

"""Quality-gate honesty checks.

Catches the classes of bug that this repo has been bitten by and that no
existing gate prevents:

  1. **Bypassed quality gates.** Patterns like `|| echo`, `|| true`,
     `--no-verify`, `--force` inside scripts that the gates run, or
     `if : ; then ... else exit 0 ; fi` rationalisations. If the gate
     swallows a non-zero exit, it's theatre, not a gate.

  2. **Fake-working code.** `Math.random()` returning numbers presented
     as real metrics, `setTimeout(() => fakeData)` simulating a backend,
     comments like `// TODO`, `// FIXME`, `// HACK`, `// XXX`, `// stub`,
     `// placeholder`, `// mock` left in shipped TS/TSX/JS code.

  3. **Shims and polyfills.** Custom `.d.ts` files under apps/*/src/ that
     declare modules to paper over upstream packages. The right fix is
     a pinned/forked/replaced dependency, not a local declaration shim.

  4. **Disabled lockfile invariants.** `pnpm.overrides` removed without
     a paired security advisory note in the commit, or `audit:dev`
     reverted to swallow exit codes.

If any check fails, prints the exact file:line and the rule, and exits
non-zero. CI and local gates run this; both must agree.

Why this exists: previously `audit:dev` was wired with `pnpm audit --dev || echo
'... not blocking'`. The `||` swallowed the non-zero exit code so 20
real CVEs (1 low, 2 moderate, 17 high) sailed through pre-push without
ever blocking a commit. That bypass was the symptom; this script is
the antibody — it scans every package.json and shell script for the
same pattern so a future copy-paste can't reintroduce it.
"""
from __future__ import annotations

import json
import re
import subprocess
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[2]


def tracked_files() -> list[Path]:
    res = subprocess.run(
        ["git", "ls-files"], cwd=ROOT, text=True, capture_output=True, check=True
    )
    return [Path(line) for line in res.stdout.splitlines() if line]


# ---------------------------------------------------------------------------
# Check 1: no quality-gate bypass patterns inside scripts run by gates
# ---------------------------------------------------------------------------

# Anchor: only flag inside `scripts` blocks of package.json files. Other
# fields (descriptions, etc.) are free-form and may legitimately mention
# `|| true` in documentation.

GATE_BYPASS_PATTERNS = [
    # `... || echo '...'`  — silences a failed command with a printed message
    (re.compile(r"\|\|\s*echo\b"), "uses '|| echo' to swallow a non-zero exit"),
    # `... || true`        — silences any failure
    (re.compile(r"\|\|\s*true\b"), "uses '|| true' to swallow a non-zero exit"),
    # `... || :`           — same as `|| true` but trickier to grep
    (re.compile(r"\|\|\s*:\s*(?:&&|;|$)"), "uses '|| :' to swallow a non-zero exit"),
    # `--no-verify`        — bypasses git hooks
    (re.compile(r"--no-verify\b"), "uses '--no-verify' to bypass git hooks"),
    # `--force` on git push — destroys remote history
    (
        re.compile(r"git\s+push.*--force\b"),
        "uses 'git push --force' (history destruction)",
    ),
    # `set +e`             — disables fail-fast in middle of a script
    (re.compile(r"^\s*set\s+\+e\b"), "uses 'set +e' (disables fail-fast)"),
]

# Allowlist: this script is the rule, so it can NAME the patterns it bans.
ALLOWLISTED_PATHS = {
    Path("scripts/quality/check_honesty.py"),
    Path("scripts/quality/CHECK_HONESTY.md"),
}

# Files whose `scripts.*` entries are quality gates and must not bypass.
QUALITY_GATE_PACKAGE_JSONS = [Path("package.json")]

# Quality-gate script names that absolutely must not bypass.
GATE_SCRIPT_NAMES = {
    "audit:prod",
    "audit:dev",
    "audit:deps",
    "build",
    "lint",
    "typecheck",
    "deps:check",
    "security:check",
    "quality:pre-commit",
    "quality:pre-push",
    "honesty:check",
}


def check_quality_gate_bypass() -> list[str]:
    violations: list[str] = []

    # 1a. Inspect package.json scripts for bypass patterns.
    for rel in QUALITY_GATE_PACKAGE_JSONS:
        path = ROOT / rel
        if not path.exists():
            continue
        try:
            data = json.loads(path.read_text(encoding="utf-8"))
        except json.JSONDecodeError as e:
            violations.append(f"{rel}: invalid JSON ({e})")
            continue
        scripts = data.get("scripts", {})
        for name, cmd in scripts.items():
            if name not in GATE_SCRIPT_NAMES:
                continue
            for pattern, why in GATE_BYPASS_PATTERNS:
                if pattern.search(cmd):
                    violations.append(
                        f"{rel}:scripts.{name}: {why} -> {cmd!r}"
                    )

    # 1b. Inspect shell scripts under .githooks/ — they must not bypass.
    for rel in tracked_files():
        if not rel.parts:
            continue
        if rel.parts[0] not in {".githooks", "scripts"}:
            continue
        if rel in ALLOWLISTED_PATHS:
            continue
        if rel.suffix not in {"", ".sh"}:
            continue
        path = ROOT / rel
        if not path.exists():
            continue
        try:
            text = path.read_text(encoding="utf-8")
        except UnicodeDecodeError:
            continue
        for line_no, line in enumerate(text.splitlines(), start=1):
            for pattern, why in GATE_BYPASS_PATTERNS:
                if pattern.search(line):
                    violations.append(f"{rel}:{line_no}: {why} -> {line.strip()!r}")
    return violations


# ---------------------------------------------------------------------------
# Check 2: no fake-working code in shipped TS/TSX/JS
# ---------------------------------------------------------------------------

FAKE_CODE_DIRS = ("apps/web/src", "apps/workspace/src", "apps/api/src", "apps/mobile/app", "apps/mobile/lib")
FAKE_CODE_EXTS = {".ts", ".tsx", ".js", ".jsx"}

FAKE_CODE_PATTERNS = [
    # Comment markers that signal half-done work in shipped code.
    # Must be line-leading whitespace + `//` to avoid matching `http://placeholder`
    # inside string literals.
    (re.compile(r"(?m)^\s*//\s*(TODO|FIXME|HACK|XXX|STUB|PLACEHOLDER|MOCK)\b", re.I),
     "left a TODO/FIXME/HACK/XXX/STUB/PLACEHOLDER/MOCK comment"),
    # Math.random or crypto.randomUUID returning user-visible "metrics"
    # is a fake-data tell. We ban Math.random outright in shipped code;
    # legitimate uses (cryptographic salts, IDs) should use crypto APIs.
    (re.compile(r"\bMath\.random\s*\("),
     "uses Math.random() in shipped code (fake-data risk; use crypto.randomUUID for IDs)"),
    # setTimeout that returns mock data (heuristic: a setTimeout literal
    # body containing `mock`/`fake`/`stub` strings).
    (re.compile(r"setTimeout\s*\([^)]*\b(mock|fake|stub|placeholder)\b", re.I),
     "setTimeout simulating a fake async backend"),
]

# Allowlist: legitimate uses (test files, dev-only utilities).
FAKE_CODE_PATH_ALLOWLIST_PATTERNS = [
    re.compile(r"\.test\.(ts|tsx|js|jsx)$"),
    re.compile(r"\.spec\.(ts|tsx|js|jsx)$"),
    re.compile(r"__tests__/"),
    re.compile(r"__mocks__/"),
]


def check_fake_code() -> list[str]:
    violations: list[str] = []
    for rel in tracked_files():
        if rel.suffix not in FAKE_CODE_EXTS:
            continue
        if not any(str(rel).startswith(d) for d in FAKE_CODE_DIRS):
            continue
        if any(p.search(str(rel)) for p in FAKE_CODE_PATH_ALLOWLIST_PATTERNS):
            continue
        path = ROOT / rel
        if not path.exists():
            continue
        try:
            text = path.read_text(encoding="utf-8")
        except UnicodeDecodeError:
            continue
        for line_no, line in enumerate(text.splitlines(), start=1):
            for pattern, why in FAKE_CODE_PATTERNS:
                if pattern.search(line):
                    violations.append(f"{rel}:{line_no}: {why}")
    return violations


# ---------------------------------------------------------------------------
# Check 3: no .d.ts shims under apps/*/src/
# ---------------------------------------------------------------------------

# A shim is a .d.ts file that declares an external module — i.e. it's
# papering over a published npm package whose own types are broken.
# Project-internal ambient declarations (e.g. CSS modules, *.svg) are
# legitimate and don't trigger this.

SHIM_DECLARE_RE = re.compile(r'declare\s+module\s+["\']([^"\']+)["\']')
SHIM_INTERNAL_PREFIXES = ("*.", "@/", ".", "./", "../")


def check_shim_dts() -> list[str]:
    violations: list[str] = []
    for rel in tracked_files():
        if rel.suffix != ".ts" or not str(rel).endswith(".d.ts"):
            continue
        # Only flag shims under app source directories, not tooling.
        if not any(str(rel).startswith(d) for d in FAKE_CODE_DIRS):
            continue
        path = ROOT / rel
        if not path.exists():
            continue
        try:
            text = path.read_text(encoding="utf-8")
        except UnicodeDecodeError:
            continue
        for match in SHIM_DECLARE_RE.finditer(text):
            module_name = match.group(1)
            # Skip ambient-asset declarations (those start with * or look
            # like local file globs) — they aren't shims for npm packages.
            if module_name.startswith(SHIM_INTERNAL_PREFIXES):
                continue
            # Anything else is a declaration for an external package =
            # a shim for broken upstream types. That's banned in production.
            line_no = text[: match.start()].count("\n") + 1
            violations.append(
                f"{rel}:{line_no}: declares module {module_name!r} (shim for upstream — "
                "pin/fork/remove the dep instead)"
            )
    return violations


# ---------------------------------------------------------------------------
# Check 4: pnpm.overrides invariant — known security pins must remain
# ---------------------------------------------------------------------------

# When we patched the 20 transitive CVEs (commit 5ebc8e8), we added
# pnpm.overrides for 6 packages. If those overrides ever go missing
# without `pnpm audit` proving the upstream chain has caught up, the
# vulns silently return. This guards the invariant.

REQUIRED_OVERRIDES = {
    "@xmldom/xmldom",
    "diff",
    "minimatch",
    "node-forge",
    "tar",
    "yaml",
}


def check_pnpm_overrides() -> list[str]:
    violations: list[str] = []
    pkg = ROOT / "package.json"
    if not pkg.exists():
        return violations
    data = json.loads(pkg.read_text(encoding="utf-8"))
    overrides = data.get("pnpm", {}).get("overrides", {})
    missing = REQUIRED_OVERRIDES - set(overrides.keys())
    if missing:
        violations.append(
            "package.json:pnpm.overrides: missing required security pins for "
            f"{sorted(missing)} (added in commit 5ebc8e8 to patch 20 transitive CVEs; "
            "remove only if `pnpm audit --dev` proves the upstream chain has caught up)."
        )
    return violations


# ---------------------------------------------------------------------------
# Driver
# ---------------------------------------------------------------------------


def main() -> int:
    sections: list[tuple[str, list[str]]] = [
        ("Quality-gate bypass", check_quality_gate_bypass()),
        ("Fake-working code", check_fake_code()),
        ("Shim .d.ts under apps/*/src/", check_shim_dts()),
        ("Required pnpm.overrides (security pins)", check_pnpm_overrides()),
    ]
    total = sum(len(v) for _, v in sections)
    if total == 0:
        print("honesty:check OK — no gate bypasses, fake code, shims, or missing security overrides.")
        return 0
    sys.stderr.write("honesty:check FAILED:\n")
    for name, violations in sections:
        if not violations:
            continue
        sys.stderr.write(f"\n[{name}]\n")
        for v in violations:
            sys.stderr.write(f"  - {v}\n")
    sys.stderr.write(
        "\nFix the root cause. Do NOT add an allowlist entry to silence a check; "
        "the rule exists because the alternative was a real production incident.\n"
    )
    return 1


if __name__ == "__main__":
    sys.exit(main())