Blame for scripts/security_scan.sh

#!/usr/bin/env bash
# ADR-0014 security scan — this repo's copy of the shared definition (authored in
# DecoupleRpy_Agent; ported from pdac-subtype-agent). Same stages everywhere, run by
# the pre-push hook and `make security-scan`:
#
#   1. Dependencies   — pip-audit over the FULL resolved dependency tree (+ SBOM)
#   2. Static         — bandit over the app code (baseline-suppressed accepted findings)
#   3. Secrets        — gitleaks over the working tree AND full git history
#   4. Sandbox image  — trivy over docker/sandbox.Dockerfile (skips where absent)
#
# Differences from the sibling copies, both deliberate:
#
#   - Stage 1 audits the RESOLVED tree, not the lines in requirements.txt. The
#     siblings run `pip-audit --no-deps`, which is right for them because their
#     requirements.txt is a compiled lockfile pinning every transitive dependency.
#     Here it is a short hand-written list, so --no-deps would audit only the top-level
#     pins and never see a transitive CVE — exactly the class that hit anyio on
#     2026-09-21. So a hand-written file is first resolved with uv for the Space's own
#     python_version, plus the gradio HF installs from README `sdk_version`; a file that
#     is already a uv lock is audited as-is.
#   - bandit targets are discovered, not listed: `src/` plus every root-level module,
#     so a new top-level file cannot silently fall outside the scan.
#
# Not audited: HF's generic SDK base layer (datasets, huggingface-hub, …), which HF
# installs and caches outside this repo. Anything we depend on through it that
# matters must be floored in requirements.in (see its security-floors block).
#
# Design goals (unchanged from the shared definition):
#   - No global installs, no repo mutation beyond security/ artifacts.
#   - Graceful skip (not failure) when an optional scanner is absent; loud about it.
#     SECURITY_SCAN_STRICT=1 turns skips into failures.
#   - Artifacts land in $SECURITY_SCAN_OUT (default: security/). The pre-push hook
#     scans a throwaway checkout of the PUSHED commit and points this at the real
#     repo, so reports and the cached audit venv survive the checkout's removal.
#
# Exit code: non-zero if ANY enabled stage reports findings.
# Usage:  scripts/security_scan.sh [requirements.txt]
set -uo pipefail

cd "$(dirname "$0")/.."
REQ="${1:-requirements.txt}"
OUT="${SECURITY_SCAN_OUT:-security}"
mkdir -p "$OUT"
STAMP="$(date -u +%Y%m%dT%H%M%SZ)"
STRICT="${SECURITY_SCAN_STRICT:-0}"

RC=0
SKIPS=()
TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT

note()  { printf '\n\033[1m==> %s\033[0m\n' "$*"; }
fail()  { printf '\033[31mFINDINGS:\033[0m %s\n' "$*"; RC=1; }
skip()  { printf '\033[33mSKIP:\033[0m %s\n' "$*"; SKIPS+=("$*"); [[ "$STRICT" == "1" ]] && RC=1; return 0; }
ok()    { printf '\033[32mOK:\033[0m %s\n' "$*"; }

[[ -f "$REQ" ]] || { echo "error: '$REQ' not found" >&2; exit 2; }

# README front matter is what HF builds from — read it rather than restating it.
front() { awk '/^---$/{n++; next} n==1' README.md 2>/dev/null | sed -n "s/^$1:[[:space:]]*//p" | tr -d '"' | head -1; }
PYV="$(front python_version)"; PYV="${PYV:-3.10}"   # HF's default when unset
SDKV="$(front sdk_version)"

# --------------------------------------------------------------------------- #
# Stage 1a — the full dependency lock that stage 1 audits
# --------------------------------------------------------------------------- #
note "dependency tree (python $PYV, linux${SDKV:+, gradio $SDKV from sdk_version})"
LOCK="$TMP/lock.txt"
if head -3 "$REQ" | grep -q 'uv pip compile'; then
  cp "$REQ" "$LOCK"; ok "$REQ is already a compiled lock — auditing it as-is"
elif command -v uv >/dev/null 2>&1; then
  { cat "$REQ"; [[ -n "$SDKV" ]] && echo "gradio[oauth,mcp]==$SDKV"; } > "$TMP/in.txt"
  if uv pip compile "$TMP/in.txt" -o "$LOCK" --python-version "$PYV" \
       --python-platform linux --no-header --quiet 2>"$TMP/uv.err"; then
    ok "resolved $(grep -cE '^[A-Za-z0-9_.-]+==' "$LOCK") packages (incl. transitive)"
  else
    fail "could not resolve $REQ for python $PYV:"; sed 's/^/         /' "$TMP/uv.err"; LOCK=""
  fi
else
  skip "dependency resolution — uv not found; transitive deps will NOT be audited"
  cp "$REQ" "$LOCK"
fi

# --------------------------------------------------------------------------- #
# Stage 1b — CycloneDX SBOM (of the resolved tree)
# --------------------------------------------------------------------------- #
note "SBOM (CycloneDX) -> $OUT/sbom.json"
if [[ -z "$LOCK" ]]; then
  skip "SBOM — no resolved dependency tree"
elif command -v uvx >/dev/null 2>&1; then
  uvx --from cyclonedx-bom cyclonedx-py requirements "$LOCK" --of JSON -o "$OUT/sbom.json" \
    >/dev/null 2>&1 && ok "SBOM written" || fail "SBOM generation errored"
else
  skip "SBOM — uvx not found"
fi

# --------------------------------------------------------------------------- #
# Stage 1c — pip-audit (known CVEs)
# --------------------------------------------------------------------------- #
# biodata-registry is a first-party wheel hosted on HF, not a PyPI package, so it has
# no advisories to look up; strip it (and any URL/git line) before auditing.
note "pip-audit (known CVEs; first-party HF-wheel dep excluded)"
PYBIN=""
for cand in python3.13 python3.12 python3.11 python3; do
  command -v "$cand" >/dev/null 2>&1 || continue
  if "$cand" -m venv "$TMP/probe-$cand" >/dev/null 2>&1; then PYBIN="$cand"; break; fi
done
if [[ -z "$LOCK" ]]; then
  : # already reported as a finding above
elif [[ -z "$PYBIN" ]]; then
  skip "pip-audit — no Python >=3.11 with a working venv found"
else
  grep -vE 'git\+|biodata-registry|https?://' "$LOCK" > "$TMP/audit.txt"
  VENV="$OUT/.audit-venv"
  if [[ ! -x "$VENV/bin/pip-audit" ]]; then
    echo "    setting up audit venv ($PYBIN)"
    "$PYBIN" -m venv "$VENV" && "$VENV/bin/python" -m pip install -q --upgrade pip pip-audit
  fi
  # Recorded, expiring acceptances (security/accepted-vulns.txt; justified in
  # security/ACCEPTED-FINDINGS.md). An EXPIRED entry fails the scan: re-triage, never extend quietly.
  IGNORES=()
  if [[ -f security/accepted-vulns.txt ]]; then
    TODAY="$(date -u +%Y-%m-%d)"
    while read -r vid expires _rest; do
      [[ -z "${vid:-}" || "$vid" == \#* ]] && continue
      exp="${expires#expires=}"
      if [[ "$exp" < "$TODAY" ]]; then
        fail "accepted finding $vid expired on $exp — re-triage it (security/ACCEPTED-FINDINGS.md)"
        continue
      fi
      IGNORES+=(--ignore-vuln "$vid")
    done < security/accepted-vulns.txt
    [[ ${#IGNORES[@]} -gt 0 ]] && echo "    accepting $(( ${#IGNORES[@]} / 2 )) recorded finding(s) until their expiry (security/accepted-vulns.txt)"
  fi
  # `${IGNORES[@]+…}`: macOS ships bash 3.2, where an empty array under `set -u` is unbound.
  "$VENV/bin/python" -m pip_audit --requirement "$TMP/audit.txt" --no-deps --disable-pip \
    ${IGNORES[@]+"${IGNORES[@]}"} --format columns 2>/dev/null | tee "$OUT/pip-audit-$STAMP.txt"
  AUDIT_RC=${PIPESTATUS[0]}
  if [[ "$AUDIT_RC" -eq 0 ]]; then
    ok "pip-audit — no known vulnerabilities in the resolved tree"
  else
    fail "pip-audit reported vulnerabilities (see $OUT/pip-audit-$STAMP.txt)."
    echo "         Fix by pinning or flooring the package in requirements.txt, or record a"
    echo "         justified acceptance in security/ACCEPTED-FINDINGS.md."
  fi
fi

# --------------------------------------------------------------------------- #
# Stage 2 — bandit (static analysis, baseline-suppressed)
# --------------------------------------------------------------------------- #
TARGETS=(); [[ -d src ]] && TARGETS+=(src)
while IFS= read -r f; do TARGETS+=("$f"); done < <(git ls-files '*.py' 2>/dev/null | grep -v /)
note "bandit (static analysis over: ${TARGETS[*]})"
BASELINE="security/bandit-baseline.json"
BANDIT=(); if command -v bandit >/dev/null 2>&1; then BANDIT=(bandit); \
  elif command -v uvx >/dev/null 2>&1; then BANDIT=(uvx bandit); fi
if [[ ${#BANDIT[@]} -eq 0 ]]; then
  skip "bandit — not installed and uvx unavailable"
elif [[ ${#TARGETS[@]} -eq 0 ]]; then
  skip "bandit — no Python code found"
else
  BARGS=(-r "${TARGETS[@]}" -ll -c bandit.yaml)
  [[ -f "$BASELINE" ]] && BARGS+=(-b "$BASELINE")
  "${BANDIT[@]}" "${BARGS[@]}" -f json -o "$OUT/bandit-$STAMP.json" -q 2>/dev/null
  BRC=$?
  "${BANDIT[@]}" "${BARGS[@]}" -q 2>/dev/null | tail -n 25 || true
  if [[ "$BRC" -eq 0 ]]; then
    ok "bandit — no new findings beyond the accepted baseline"
  else
    fail "bandit reported NEW findings (see $OUT/bandit-$STAMP.json). Fix, or if"
    echo "         intended, annotate inline (# nosec Bxxx), document it in"
    echo "         security/ACCEPTED-FINDINGS.md, and run: make security-baseline"
  fi
fi

# --------------------------------------------------------------------------- #
# Stage 3 — gitleaks (secrets: working tree AND full history)
# --------------------------------------------------------------------------- #
note "gitleaks (secrets — working tree + full git history)"
if command -v gitleaks >/dev/null 2>&1; then
  gitleaks detect --config .gitleaks.toml --redact --no-banner --log-level warn \
    --report-format json --report-path "$OUT/gitleaks-$STAMP.json"
  if [[ $? -eq 0 ]]; then
    ok "gitleaks — no secrets detected in tree or history"
  else
    fail "gitleaks detected potential secrets (see $OUT/gitleaks-$STAMP.json)."
    echo "         If a real secret leaked, ROTATE it (HF_TOKEN / research_agent_token /"
    echo "         ANTHROPIC_API_KEY are HF Space secrets) — removing the commit is not enough."
  fi
else
  skip "gitleaks — not installed (brew install gitleaks)"
fi

# --------------------------------------------------------------------------- #
# Stage 4 — trivy (sandbox image) — N/A unless the repo has one
# --------------------------------------------------------------------------- #
note "trivy (sandbox Dockerfile)"
DOCKERFILE="docker/sandbox.Dockerfile"
if [[ ! -f "$DOCKERFILE" ]]; then
  skip "trivy — $DOCKERFILE not found (no sandboxed-exec image in this repo)"
elif command -v trivy >/dev/null 2>&1; then
  trivy config --severity HIGH,CRITICAL --exit-code 1 \
    --format json --output "$OUT/trivy-$STAMP.json" "$DOCKERFILE"
  [[ $? -eq 0 ]] && ok "trivy — no HIGH/CRITICAL Dockerfile misconfigurations" \
                 || fail "trivy flagged the sandbox Dockerfile (see $OUT/trivy-$STAMP.json)."
else
  skip "trivy — not installed (brew install trivy)"
fi

# --------------------------------------------------------------------------- #
# Summary
# --------------------------------------------------------------------------- #
note "summary"
if [[ ${#SKIPS[@]} -gt 0 ]]; then
  echo "Skipped stages:"; for s in "${SKIPS[@]}"; do echo "  - $s"; done
fi
if [[ "$RC" -eq 0 ]]; then
  ok "security scan passed (all enabled stages clean)."
else
  echo "Security scan FAILED (rc=$RC) — triage above. Artifacts in $OUT/."
fi
exit "$RC"