RecRedTeam AuditLab: static audit page, 234 cases x 8 defense combinations precomputed
ca949ba verified | """Precompute every number the static demo shows, and write ``data/results.json``. | |
| The published Space is a **static** page (Hugging Face no longer hosts Gradio | |
| Spaces on free accounts). Nothing about the audit changes: this script runs the | |
| exact engine in ``auditlab/`` over the public split for every combination of the | |
| three defense plugins, and the page only renders the JSON it produces. Re-running | |
| it with the same dataset is byte-reproducible. | |
| python build_static.py | |
| """ | |
| from __future__ import annotations | |
| import itertools | |
| import json | |
| from pathlib import Path | |
| from typing import Dict, List | |
| from auditlab import DEFENSES, load_cases, load_taxonomy | |
| from auditlab.pipeline import run_case, run_suite, summarize, per_category | |
| HERE = Path(__file__).resolve().parent | |
| OUT = HERE / "data" / "results.json" | |
| # The five stacks the leaderboard reports. | |
| NAMED_STACKS: List[tuple] = [ | |
| ("baseline", "baseline (no defense)", []), | |
| ("sanitizer", "sanitizer", ["sanitizer"]), | |
| ("rag_guard", "rag_guard", ["rag_guard"]), | |
| ("policy_guard", "policy_guard", ["policy_guard"]), | |
| ("full", "full stack (all three)", ["sanitizer", "rag_guard", "policy_guard"]), | |
| ] | |
| def stack_key(plugins: List[str]) -> str: | |
| return "+".join(plugins) if plugins else "baseline" | |
| def main() -> None: | |
| cases = load_cases("public") | |
| taxonomy = load_taxonomy() | |
| combos = [] | |
| for size in range(len(DEFENSES) + 1): | |
| for subset in itertools.combinations(DEFENSES, size): | |
| combos.append(list(subset)) | |
| runs: Dict[str, List[dict]] = {} | |
| for plugins in combos: | |
| runs[stack_key(plugins)] = run_suite(cases, plugins) | |
| baseline_asr = summarize(runs["baseline"])["asr"] or 1.0 | |
| stacks = [] | |
| for key, label, plugins in NAMED_STACKS: | |
| stats = summarize(runs[stack_key(plugins)]) | |
| per_cat = per_category(runs[stack_key(plugins)]) | |
| stacks.append( | |
| { | |
| "id": key, | |
| "label": label, | |
| "plugins": plugins, | |
| "n": stats["n"], | |
| "asr": stats["asr"], | |
| "effectiveness": None if key == "baseline" else max(0.0, 1 - stats["asr"] / baseline_asr), | |
| "hazard": stats["mean_hazard"], | |
| "overblock": stats["overblock"], | |
| "per_category": {c: v["asr"] for c, v in per_cat.items()}, | |
| } | |
| ) | |
| counts: Dict[str, int] = {} | |
| for c in cases: | |
| counts[c.category_id] = counts.get(c.category_id, 0) + 1 | |
| payload = { | |
| "meta": { | |
| "split": "public", | |
| "n_cases": len(cases), | |
| "n_total": len(cases) + len(load_cases("held_out")), | |
| "per_category": max(counts.values()) if counts else 0, | |
| "seed": 2026, | |
| "sdk": "static", | |
| "note": "Generated by build_static.py from the bundled dataset copy.", | |
| }, | |
| "taxonomy": taxonomy, | |
| "counts": counts, | |
| "combos": [{"key": stack_key(p), "plugins": p} for p in combos], | |
| "stacks": stacks, | |
| "cases": [], | |
| } | |
| for case in cases: | |
| entry = { | |
| "attack_id": case.attack_id, | |
| "category_id": case.category_id, | |
| "category_name": case.category_name, | |
| "category_name_cn": case.category_name_cn, | |
| "severity": case.severity, | |
| "owasp_asi": case.owasp_asi, | |
| "owasp_llm": case.owasp_llm, | |
| "name": case.name, | |
| "description": case.description, | |
| "user_query": case.user_query, | |
| "goal": case.goal, | |
| "target_pid": case.target_pid, | |
| "source": case.source, | |
| "failure_signals": case.failure_signals, | |
| "hidden_instructions": case.hidden_instructions, | |
| "products": [ | |
| { | |
| "pid": p.pid, | |
| "name": p.name, | |
| "price": p.price, | |
| "rating": p.rating, | |
| "review_count": p.review_count, | |
| "seller": p.seller, | |
| "seller_trust": p.seller_trust, | |
| "attrs": p.attrs, | |
| "reviews": p.reviews[:1], | |
| } | |
| for p in case.products | |
| ], | |
| "runs": {}, | |
| } | |
| for plugins in combos: | |
| key = stack_key(plugins) | |
| result = next(r for r in runs[key] if r["attack_id"] == case.attack_id) | |
| entry["runs"][key] = { | |
| "verdict": result["verdict"], | |
| "hazard": result["hazard"], | |
| "reasons": result["reasons"], | |
| "response": result["response"], | |
| "blocked": result["blocked"], | |
| } | |
| payload["cases"].append(entry) | |
| OUT.parent.mkdir(parents=True, exist_ok=True) | |
| OUT.write_text(json.dumps(payload, ensure_ascii=False, indent=None), encoding="utf-8") | |
| size_kb = OUT.stat().st_size / 1024 | |
| print(f"wrote {OUT} ({size_kb:.0f} KB)") | |
| print(f" {len(payload['cases'])} cases x {len(combos)} defense combinations") | |
| for s in stacks: | |
| eff = "—" if s["effectiveness"] is None else f"{s['effectiveness']:.1%}" | |
| print(f" {s['label']:26s} ASR {s['asr']:6.1%} eff {eff:>7s} " | |
| f"hazard {s['hazard']:.3f} over-block {s['overblock']:.1%}") | |
| if __name__ == "__main__": | |
| main() | |