scripts/audit_prompts.py

"""Find LLM prompt-like strings under app/ not referenced in PROMPTS_INVENTORY.md."""
from __future__ import annotations

import re
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
INV = (ROOT / "PROMPTS_INVENTORY.md").read_text(encoding="utf-8")

MARKERS = (
    "chat.completions",
    "SystemMessage",
    "HumanMessage",
    "ChatPromptTemplate",
    'role": "system"',
    "messages=[",
)


def main() -> None:
    missing: list[tuple[str, str]] = []
    for py in (ROOT / "app").rglob("*.py"):
        if "tests" in py.parts:
            continue
        text = py.read_text(encoding="utf-8", errors="ignore")
        if not any(m in text for m in MARKERS):
            continue
        for m in re.finditer(r'"""(.*?)"""', text, re.DOTALL):
            s = m.group(1).strip()
            if len(s) < 60:
                continue
            if not any(
                tok in s
                for tok in (
                    "You are",
                    "STRICT",
                    "RICS",
                    "Return ONLY",
                    "MANDATORY",
                    "Validate",
                    "Expand",
                    "merge",
                    "MRICS",
                    "JSON",
                )
            ):
                continue
            probe = s[:50].replace("\n", " ")
            if probe in INV or s[:80] in INV:
                continue
            missing.append((str(py.relative_to(ROOT)), probe))

    print(f"Potentially missing blocks: {len(missing)}")
    for path, probe in missing[:40]:
        print(f"  {path}: {probe}...")


if __name__ == "__main__":
    main()