scripts/ci_secret_scan.py

#!/usr/bin/env python3
"""Fail-closed repository secret scan with explicit fixture exclusions."""
from __future__ import annotations

import argparse
import os
import re
import sys
from pathlib import Path

PATTERNS = (
    re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"),
    re.compile(
        r"\b(?i:api[_-]?key|api[_-]?secret|access[_-]?token|password)"
        r"\s*[:=]\s*['\"]?"
        r"(?!(?:[A-Z][A-Z0-9_]{7,}\b|example\b|placeholder\b|redacted\b|test\b))"
        r"[A-Za-z0-9_./+\-=]{16,}"
    ),
    re.compile(r"\b(?:hf_|sk-|xox[baprs]-)[A-Za-z0-9_-]{24,}\b"),
)
SKIP_DIRS = {".git", ".pytest_cache", "__pycache__", ".venv", "node_modules"}
SKIP_SUFFIXES = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".zip", ".gz", ".pyc", ".sqlite3"}
FIXTURE_PREFIXES = {"hermes_overlay/tests/", "scripts/ci_secret_scan.py", "scripts/backup_allowlist.py", "scripts/verify_required_failures.py"}


def scan(root: Path, *, allow_fixtures: bool) -> list[str]:
    findings: list[str] = []
    for path in root.rglob("*"):
        if not path.is_file() or any(part in SKIP_DIRS for part in path.parts) or path.suffix.lower() in SKIP_SUFFIXES:
            continue
        rel = path.relative_to(root).as_posix()
        if allow_fixtures and any(rel.startswith(prefix) for prefix in FIXTURE_PREFIXES):
            continue
        if path.name.startswith(".env") and path.name != ".env.example":
            findings.append(f"denied secret filename: {rel}")
            continue
        try:
            text = path.read_text(encoding="utf-8", errors="ignore")[:512_000]
        except OSError:
            continue
        for pattern in PATTERNS:
            if pattern.search(text):
                findings.append(f"secret-like content: {rel}")
                break
    return findings


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("path", nargs="?", default=str(Path(__file__).resolve().parents[1]))
    parser.add_argument("--no-fixture-exclusions", action="store_true")
    args = parser.parse_args()
    root = Path(args.path).resolve()
    findings = scan(root, allow_fixtures=not args.no_fixture_exclusions)
    for finding in findings:
        print(f"ERROR: {finding}")
    print(f"secret_scan_findings={len(findings)}")
    return 1 if findings else 0


if __name__ == "__main__":
    raise SystemExit(main())