File size: 7,634 Bytes
6c59ea7
3d002b7
6c59ea7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3d002b7
6c59ea7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3d002b7
6c59ea7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
"""
cli.py: Command-line interface.

Uses argparse only (no third-party CLI dependency) so the tool runs anywhere
Python does. Entry points:

    llm-scan run --target stub --out ./reports
    llm-scan list-probes
    llm-scan version

``run`` produces the full deliverable set in ``--out``:
  report.json, report.html, model_card.md, risk_register.csv

and exits non-zero when findings at/above ``--fail-on`` are present, which is the
hook CI uses to block a release.
"""

from __future__ import annotations

import argparse
import sys
from pathlib import Path
from typing import List, Optional

from . import __version__
from .engine import Scanner, available_categories, load_probes
from .governance import write_governance_package
from .models import Severity
from .providers import get_provider
from .reporting import summary_table, write_html_report, write_json_report

EXIT_OK = 0
EXIT_FINDINGS = 2  # threshold exceeded, distinct from generic error (1)
EXIT_ERROR = 1


def _print_summary(result, out_dir: Path) -> None:
    sc = result.severity_counts()
    print()
    print(f"Scan complete: target={result.target}  probes={result.total_probes}")
    print(f"Findings: {result.total_findings}  (pass rate {result.pass_rate:.0%})")
    print(
        "  Critical={CRITICAL}  High={HIGH}  Medium={MEDIUM}  Low={LOW}".format(**sc)
    )
    print()
    print("Artifacts written to", out_dir.resolve())
    for name in ("report.json", "report.html", "model_card.md", "risk_register.csv"):
        print(f"  - {out_dir / name}")
    print()


def cmd_run(args: argparse.Namespace) -> int:
    out_dir = Path(args.out)
    out_dir.mkdir(parents=True, exist_ok=True)

    categories = (
        [c.strip() for c in args.categories.split(",") if c.strip()]
        if args.categories
        else None
    )

    try:
        provider = get_provider(args.target)
    except (ValueError, RuntimeError) as exc:
        print(f"error: {exc}", file=sys.stderr)
        return EXIT_ERROR

    try:
        probes = load_probes(
            Path(args.probe_dir) if args.probe_dir else None, categories
        )
    except (FileNotFoundError, ValueError) as exc:
        print(f"error: {exc}", file=sys.stderr)
        return EXIT_ERROR

    scanner = Scanner(provider, probes=probes, scanner_version=__version__)
    result = scanner.run()

    write_json_report(result, out_dir / "report.json")
    write_html_report(result, out_dir / "report.html")
    if not args.no_governance:
        write_governance_package(result, out_dir)

    if not args.quiet:
        _print_summary(result, out_dir)

    # CI gate: fail if any finding is at/above the threshold.
    threshold = Severity.from_str(args.fail_on)
    highest = result.highest_severity()
    if highest is not None and highest.value >= threshold.value:
        if not args.quiet:
            print(
                f"FAIL: highest severity {highest.name} >= threshold "
                f"{threshold.name}.",
                file=sys.stderr,
            )
        return EXIT_FINDINGS
    return EXIT_OK


def cmd_list_probes(args: argparse.Namespace) -> int:
    probe_dir = Path(args.probe_dir) if args.probe_dir else None
    try:
        probes = load_probes(probe_dir)
    except (FileNotFoundError, ValueError) as exc:
        print(f"error: {exc}", file=sys.stderr)
        return EXIT_ERROR

    print(f"{len(probes)} probes across {len(available_categories(probe_dir))} categories:\n")
    current = None
    for p in probes:
        if p.category != current:
            current = p.category
            print(f"[{p.category}]")
        print(f"  {p.id:<8} {p.severity.name:<8} {p.name}")
    return EXIT_OK


def cmd_serve(args: argparse.Namespace) -> int:
    """Launch the offline report viewer (FastAPI) in the browser.

    Lazily imports uvicorn + the viewer so the core scanner keeps zero hard
    dependency on the web stack. Install it with ``pip install ".[viewer]"``.
    """
    try:
        import uvicorn  # type: ignore
    except ImportError:
        print(
            "error: the report viewer needs FastAPI + uvicorn. Install with "
            '`pip install "llm-security-scanner[viewer]"` (or `pip install '
            "fastapi uvicorn`), then re-run `llm-scan serve`.",
            file=sys.stderr,
        )
        return EXIT_ERROR

    print(
        f"LLM Security Scanner viewer → http://{args.host}:{args.port}\n"
        f"  Running a scan against target '{args.target}' on first request.\n"
        "  Press Ctrl+C to stop."
    )
    # Point the viewer at the requested target via the env var it reads.
    import os

    os.environ["LLM_SCAN_VIEWER_TARGET"] = args.target
    uvicorn.run(
        "llm_security_scanner.viewer:app",
        host=args.host,
        port=args.port,
        log_level="warning",
    )
    return EXIT_OK


def cmd_version(args: argparse.Namespace) -> int:
    print(f"llm-security-scanner {__version__}")
    return EXIT_OK


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(
        prog="llm-scan",
        description="Security-test an LLM endpoint and generate a governance package.",
    )
    parser.add_argument(
        "--version", action="version", version=f"llm-security-scanner {__version__}"
    )
    sub = parser.add_subparsers(dest="command")

    run = sub.add_parser("run", help="Run a scan and write the report + governance package.")
    run.add_argument(
        "--target",
        default="stub",
        help="Target to scan: 'stub' (offline, default) or 'openai' (uses OPENAI_API_KEY).",
    )
    run.add_argument(
        "--out",
        default="./reports",
        help="Output directory for artifacts (default: ./reports).",
    )
    run.add_argument(
        "--categories",
        default=None,
        help="Comma-separated subset of probe categories (default: all).",
    )
    run.add_argument(
        "--probe-dir",
        default=None,
        help="Custom directory of YAML probe packs (default: built-in packs).",
    )
    run.add_argument(
        "--fail-on",
        default="CRITICAL",
        help="Exit non-zero if a finding at/above this severity is present "
        "(CRITICAL/HIGH/MEDIUM/LOW). Default: CRITICAL.",
    )
    run.add_argument(
        "--no-governance",
        action="store_true",
        help="Skip generating the model card and risk register.",
    )
    run.add_argument("--quiet", action="store_true", help="Suppress summary output.")
    run.set_defaults(func=cmd_run)

    lst = sub.add_parser("list-probes", help="List the loaded probe battery.")
    lst.add_argument("--probe-dir", default=None, help="Custom probe pack directory.")
    lst.set_defaults(func=cmd_list_probes)

    srv = sub.add_parser(
        "serve",
        help="Launch the offline web report viewer (needs the [viewer] extra).",
    )
    srv.add_argument("--host", default="127.0.0.1", help="Bind host (default: 127.0.0.1).")
    srv.add_argument("--port", type=int, default=8000, help="Bind port (default: 8000).")
    srv.add_argument(
        "--target",
        default="stub",
        help="Target to scan for the demo: 'stub' (offline, default) or 'openai'.",
    )
    srv.set_defaults(func=cmd_serve)

    ver = sub.add_parser("version", help="Print version.")
    ver.set_defaults(func=cmd_version)

    return parser


def main(argv: Optional[List[str]] = None) -> int:
    parser = build_parser()
    args = parser.parse_args(argv)
    if not getattr(args, "command", None):
        parser.print_help()
        return EXIT_OK
    return args.func(args)


if __name__ == "__main__":
    sys.exit(main())