Grimxlock's picture
Full source Space: real Python lab runs in-browser via Pyodide (lab-source.zip + modules/src tree), live suite, live validators
08e2243 verified
Raw History Blame Contribute Delete
6.79 kB
"""Command-line interface."""
import argparse
import json
import sys
from datetime import datetime, timezone
from pathlib import Path
from .runner import run
def _load_report(path):
"""Load one saved scan report, raising ValueError with a readable
message when the file is missing, unreadable, or malformed."""
try:
data = json.loads(Path(path).read_text(encoding="utf-8"))
except OSError as exc:
raise ValueError(f"cannot read report {path}: {exc}") from exc
except json.JSONDecodeError as exc:
raise ValueError(f"report {path} is not valid JSON: {exc}") from exc
if not isinstance(data, dict):
raise ValueError(f"report {path} must be a JSON object")
results = data.get("results", [])
if not isinstance(results, list):
raise ValueError(f"report {path}: 'results' must be a list")
for item in results:
if not isinstance(item, dict) or not isinstance(item.get("probe_id"), str):
raise ValueError(
f"report {path}: each result needs a string 'probe_id'")
evaluation = item.get("evaluation")
if not isinstance(evaluation, dict) or not isinstance(
evaluation.get("status"), str):
raise ValueError(
f"report {path}: result {item.get('probe_id')!r} needs "
"'evaluation' with a string 'status'")
return data
def compare_reports(paths, output):
"""Combine per-provider scan reports into one comparison.
Prints a probe-by-provider status matrix and writes a consolidated
JSON file. A provider that was scanned twice (different models)
gets a second column named provider/model.
"""
providers = []
by_probe = {}
for path in paths:
try:
data = _load_report(path)
except ValueError as exc:
print(f"error: {exc}", file=sys.stderr)
return 2
provider = data.get("provider")
if not isinstance(provider, str) or not provider:
provider = Path(path).stem
model = data.get("model")
if not isinstance(model, str):
model = None
column = provider
suffix = 0
while any(p["column"] == column for p in providers):
suffix += 1
base = f"{provider}/{model}" if model else f"{provider}/{path}"
column = base if suffix == 1 else f"{base}#{suffix}"
providers.append({"column": column, "provider": provider,
"model": model, "path": str(path)})
for item in data.get("results", []):
by_probe.setdefault(item["probe_id"], {})[column] = {
"status": item["evaluation"]["status"],
"model": model,
"error": item.get("error"),
}
comparison = {
"schema_version": "1.0",
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
"providers": providers,
"probes": by_probe,
}
header = "Probe".ljust(22) + "".join(
p["column"].ljust(22) for p in providers
)
print(header)
for probe_id in sorted(by_probe):
row = "".join(
by_probe[probe_id].get(p["column"], {}).get("status", "-").ljust(22)
for p in providers
)
print(probe_id.ljust(22) + row)
out = Path(output)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(
json.dumps(comparison, indent=2, ensure_ascii=False) + "\n",
encoding="utf-8",
)
print(f"Saved comparison: {out}")
return 0
def main(argv=None):
parser = argparse.ArgumentParser(
description=(
"Run authorized instruction-boundary probes "
"against an AI API, or compare saved scan reports."
)
)
parser.add_argument(
"--provider",
choices=("openai", "anthropic", "gemini", "openai-compatible"),
help="Provider to scan (run mode).",
)
parser.add_argument(
"--model",
help="Optional provider model identifier.",
)
parser.add_argument(
"--base-url",
default=None,
help="OpenAI-compatible API base URL "
"(openai-compatible provider only).",
)
parser.add_argument(
"--api-key-env",
default="OPENAI_COMPAT_API_KEY",
help="Environment variable holding the compatible endpoint API "
"key; may be empty for local servers "
"(openai-compatible provider only).",
)
parser.add_argument(
"--output",
default="openmind-scan-report.json",
help="Report destination (run mode).",
)
sub = parser.add_subparsers(dest="cmd")
compare = sub.add_parser(
"compare",
help="Compare saved scan reports across providers.",
)
compare.add_argument("reports", nargs="+")
compare.add_argument(
"--output",
default="reports/comparison.json",
help="Consolidated comparison destination.",
)
args = parser.parse_args(argv)
if args.cmd == "compare":
return compare_reports(args.reports, args.output)
if not args.provider:
parser.error("--provider is required")
if args.provider == "openai-compatible":
import os
if not (args.base_url or os.getenv("OPENAI_COMPAT_BASE_URL")):
parser.error(
"Set --base-url or OPENAI_COMPAT_BASE_URL for this provider"
)
if not (args.model or os.getenv("OPENAI_COMPAT_MODEL")):
parser.error("Set --model or OPENAI_COMPAT_MODEL for this provider")
report, path = None, None
try:
report, path = run(
args.provider,
args.model,
args.output,
base_url=args.base_url,
api_key_env=args.api_key_env,
)
except OSError as exc:
print(f"error: cannot write report: {exc}", file=sys.stderr)
return 2
counts = {}
for item in report["results"]:
status = item["evaluation"]["status"]
counts[status] = counts.get(status, 0) + 1
print(f"Saved evidence report: {path}")
print(
"Result counts:",
", ".join(
f"{key}={value}"
for key, value in sorted(counts.items())
),
)
if any(
item.get("error")
or item.get("provider_response_status") in {
"incomplete", "empty", "not_received", "partial"
}
for item in report["results"]
):
print(
"warning: one or more provider responses were incomplete, empty, partial, "
"or not received; inconclusive results require review. Inspect "
"the JSON evidence report.",
file=sys.stderr,
)
return 1
return 0