import io import os import gradio as gr import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt import matplotlib.patches as mpatches from matplotlib.patches import FancyBboxPatch, Circle from matplotlib.offsetbox import OffsetImage, AnnotationBbox import numpy as np import pandas as pd from PIL import Image LOGOS_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "logos") # ============================================================ # Data (hardcoded from the SCR-Bench paper) # ============================================================ ALL_BACKENDS = [ "Claude Opus 4.5", "Claude Opus 4.6", "GPT-5.4", "GPT-5.5", "Gemini 3.1 Pro Preview", "MiniMax-M2.7", "DeepSeek-V4", "GLM-5.1", "GLM-5", ] # Brand-style colors per backend, plus path to a logo image (under logos/). # Multiple model versions from the same lab share one logo file. Colors are # pastel/light variants in the spirit of the reference template. BACKEND_STYLE = { "Claude Opus 4.5": ("#F4C2A1", "claude-color.png"), # Anthropic peach "Claude Opus 4.6": ("#E89882", "claude-color.png"), # Anthropic salmon "GPT-5.4": ("#A8DCB8", "openai.png"), # OpenAI light green "GPT-5.5": ("#7AC59C", "openai.png"), # OpenAI mid green "Gemini 3.1 Pro Preview": ("#A6BBE8", "gemini-color.png"), # Google light blue "MiniMax-M2.7": ("#C8B8DE", "minimax-color.png"), # MiniMax light purple "DeepSeek-V4": ("#E5C8D6", "deepseek-color.png"),# DeepSeek pink "GLM-5.1": ("#8FB4D8", "zhipu-color.png"), # Zhipu steel blue "GLM-5": ("#D9C29A", "zhipu-color.png"), # Zhipu tan } # CapFlow: (Control, A-Only, B-Only, A+B Neutral, A+B Explicit) ASR % CAPFLOW = { "Claude Opus 4.5": (0.0, 0.0, 1.2, 0.0, 0.7), "GPT-5.4": (0.0, 0.0, 1.2, 4.4, 4.0), "Claude Opus 4.6": (0.0, 0.0, 1.7, 1.3, 4.1), "GLM-5.1": (0.0, 0.0, 1.3, 25.5, 26.9), "GLM-5": (0.0, 0.0, 0.7, 26.4, 30.7), "Gemini 3.1 Pro Preview": (0.0, 0.0, 1.3, 30.0, 41.9), "GPT-5.5": (0.0, 0.0, 1.6, 48.1, 47.2), "MiniMax-M2.7": (0.0, 0.0, 1.7, 75.5, 74.9), "DeepSeek-V4": (0.0, 0.0, 1.5, 91.5, 92.5), } # TrustLift: (Control ASR, Endorsed ASR, Lift) % TRUSTLIFT = { "Claude Opus 4.6": (0.00, 25.19, 25.19), "GPT-5.4": (0.00, 96.51, 96.51), "Gemini 3.1 Pro Preview": (5.49, 97.76, 92.27), "Claude Opus 4.5": (0.00, 100.00, 100.00), "MiniMax-M2.7": (0.00, 100.00, 100.00), } # AuthBlur: (L0 Control, L1 Related, L3 Full Auth, Δ1, Δ2) % AUTHBLUR = { "GPT-5.4": ( 9.5, 7.1, 7.3, -2.4, -2.2), "Claude Opus 4.5": ( 8.7, 9.6, 13.1, 0.9, 4.4), "GLM-5.1": (10.5, 8.9, 17.4, -1.6, 6.9), "Claude Opus 4.6": ( 2.0, 10.0, 17.6, 8.0, 15.6), "GPT-5.5": ( 2.9, 10.2, 17.6, 7.3, 14.7), "Gemini 3.1 Pro Preview": (10.0, 30.1, 35.0, 20.1, 25.0), "DeepSeek-V4": (26.9, 40.6, 43.1, 13.7, 16.2), "MiniMax-M2.7": (19.4, 31.9, 47.3, 12.5, 27.9), "GLM-5": (20.1, 40.0, 52.9, 19.9, 32.8), } # Coverage: (CapFlow?, TrustLift?, AuthBlur?) COVERAGE = { "Claude Opus 4.5": (True, True, True), "Claude Opus 4.6": (True, True, True), "GPT-5.4": (True, True, True), "Gemini 3.1 Pro Preview": (True, True, True), "MiniMax-M2.7": (True, True, True), "GPT-5.5": (True, False, True), "DeepSeek-V4": (True, False, True), "GLM-5.1": (True, False, True), "GLM-5": (True, False, True), } # ============================================================ # Color scale # ============================================================ def asr_color(v): """Background color for an ASR value (lower = safer = greener).""" if v is None: return "#f5f5f5" if v < 0: return "#f9e8e7" if v <= -5 else "#fdf5f4" if v < 5: return "#d4edda" if v < 15: return "#e8f5e9" if v < 30: return "#fff3cd" if v < 50: return "#ffe0b2" return "#ef9a9a" # ============================================================ # DataFrame builders (sorted within each sub-benchmark) # ============================================================ def capflow_df(): rows = [] for backend, vals in sorted(CAPFLOW.items(), key=lambda x: x[1][4]): rows.append([backend, *[f"{v:.1f}" for v in vals]]) return pd.DataFrame(rows, columns=["Backend", "Control", "A-Only", "B-Only", "A+B Neutral", "A+B Explicit"]) def trustlift_df(): rows = [] for backend, vals in sorted(TRUSTLIFT.items(), key=lambda x: x[1][1]): rows.append([backend, *[f"{v:.2f}" for v in vals]]) return pd.DataFrame(rows, columns=["Backend", "Control ASR", "Endorsed ASR", "Lift (pp)"]) def authblur_df(): rows = [] for backend, vals in sorted(AUTHBLUR.items(), key=lambda x: x[1][2]): formatted = [f"{v:.1f}" for v in vals[:3]] + [f"{vals[3]:+.1f}", f"{vals[4]:+.1f}"] rows.append([backend, *formatted]) return pd.DataFrame(rows, columns=["Backend", "L0 Control", "L1 Related", "L3 Full Auth", "Δ1 (L1−L0)", "Δ2 (L3−L0)"]) def coverage_df(): rows = [] for backend in ALL_BACKENDS: cf, tl, ab = COVERAGE[backend] n = sum([cf, tl, ab]) rows.append([backend, "✓" if cf else "—", "✓" if tl else "—", "✓" if ab else "—", n]) return pd.DataFrame(rows, columns=["Backend", "CapFlow", "TrustLift", "AuthBlur", "# Sub-benchmarks"]) # ============================================================ # Compare tab — HTML table with cell coloring # ============================================================ def compare_html(backends): if not backends or len(backends) < 2: return '

Select at least 2 backends to compare.

' html = ['
'] html.append('') html.append('') html.append('') html.append('') html.append('') html.append('') html.append('') for b in backends: cf = CAPFLOW.get(b) tl = TRUSTLIFT.get(b) ab = AUTHBLUR.get(b) cf_val = cf[4] if cf else None tl_val = tl[1] if tl else None ab_val = ab[2] if ab else None def cell(v, fmt="{:.1f}"): if v is None: return '' return f'' html.append('') html.append(f'') html.append(cell(cf_val)) html.append(cell(tl_val, "{:.2f}")) html.append(cell(ab_val)) html.append('') html.append('
BackendCapFlow
(A+B Explicit %)		TrustLift
(Endorsed %)		AuthBlur
(L3 %)
—{fmt.format(v)}
{b}
') legend_items = [ ("#d4edda", "<5"), ("#e8f5e9", "5–15"), ("#fff3cd", "15–30"), ("#ffe0b2", "30–50"), ("#ef9a9a", "≥50"), ("#f5f5f5", "—"), ] html.append('
') html.append('Color scale (ASR %): ') for color, label in legend_items: html.append(f'{label}') html.append('
') html.append('
') return ''.join(html) # ============================================================ # Compare tab — matplotlib bar chart with real model logos # ============================================================ # Caches for loaded logo images, keyed by filename -> OffsetImage. _LOGO_CACHE: dict = {} def _load_logo(backend: str, zoom: float = 0.18) -> OffsetImage | None: """Load the brand logo for a backend as a fresh OffsetImage each call. The image is normalized to a fixed display size and cropped to content.""" style = BACKEND_STYLE.get(backend) if not style: return None fname = style[1] path = os.path.join(LOGOS_DIR, fname) if not os.path.exists(path): return None try: img = Image.open(path).convert("RGBA") # Crop transparent padding bbox = img.getbbox() if bbox: img = img.crop(bbox) # Resize so the largest dimension is 256px — keeps logos consistent in size w, h = img.size scale = 256 / max(w, h) if scale < 1: img = img.resize((max(1, int(w * scale)), max(1, int(h * scale))), Image.LANCZOS) return OffsetImage(np.asarray(img), zoom=zoom) except Exception: return None def _hex_to_rgb(hexcol: str): h = hexcol.lstrip("#") return int(h[0:2], 16), int(h[2:4], 16), int(h[4:6], 16) def _tint(hexcol: str, alpha: float = 0.18) -> str: r, g, b = _hex_to_rgb(hexcol) r = int(round(r + (255 - r) * alpha)) g = int(round(g + (255 - g) * alpha)) b = int(round(b + (255 - b) * alpha)) return f"#{r:02x}{g:02x}{b:02x}" def _shorten(s: str, n: int = 14) -> str: if len(s) <= n: return s return s[:n - 1] + "…" def compare_chart(backends): """Grouped bar chart: x-axis = 3 sub-benchmarks, y-axis = ASR (%). Each x position has N bars (one per selected backend), colored by brand color, with the model logo placed on top of each bar.""" if not backends or len(backends) < 1: fig, ax = plt.subplots(figsize=(8, 3), dpi=140) ax.text(0.5, 0.5, "Select at least one backend to render the chart.", ha="center", va="center", color="#888", fontsize=13) ax.set_axis_off() return _fig_to_pil(fig) # Sub-benchmark config: (short name, full title, data dict, value-index, fmt) panels = [ ("SCR-CapFlow", "SCR-CapFlow (A+B Explicit)", CAPFLOW, 4, "{:.1f}"), ("SCR-TrustLift", "SCR-TrustLift (Endorsed)", TRUSTLIFT, 1, "{:.2f}"), ("SCR-AuthBlur", "SCR-AuthBlur (L3 Full Auth)", AUTHBLUR, 2, "{:.1f}"), ] # For each panel, gather (backend, value) for every selected backend that has data panel_rows = [] for short, _title, data, idx, _fmt in panels: rows = [(b, data[b][idx]) for b in backends if b in data] panel_rows.append(rows) fig, ax = plt.subplots(figsize=(11, 6.5), dpi=140) fig.patch.set_facecolor("white") n_groups = len(panels) # 3 n_backends = len(backends) # N # Each "group" occupies 1 unit on the x-axis, and within it we have n_backends # narrow bars. We leave a fixed group gap of 0.5 between groups. bar_w = 0.85 / max(n_backends, 1) group_centers = np.arange(n_groups) * 1.5 + 0.75 # Track ymax across the whole chart to size the y-axis all_values = [v for rows in panel_rows for _, v in rows] ymax = max(all_values) if all_values else 1 # Draw each group of bars for gi, (rows, (short, full, data, idx, fmt)) in enumerate(zip(panel_rows, panels)): center = group_centers[gi] # Positions for bars within this group, centered around `center` offsets = (np.arange(len(rows)) - (len(rows) - 1) / 2) * bar_w xs = center + offsets for ri, ((b, v), x) in enumerate(zip(rows, xs)): color = BACKEND_STYLE.get(b, ("#cbd5e1",))[0] ax.bar(x, v, width=bar_w * 0.92, color=color, edgecolor="white", linewidth=1.0, zorder=2) # Value above the bar ax.text(x, v, fmt.format(v), ha="center", va="bottom", fontsize=9, fontweight="bold", color="#1f2937", zorder=4) # Logo above the value (smaller when many bars are crowded in a group) logo_zoom = 0.07 if n_backends >= 5 else 0.09 if n_backends >= 3 else 0.11 logo = _load_logo(b, zoom=logo_zoom) if logo is not None: ab = AnnotationBbox( logo, (x, v), xybox=(0, 12), xycoords="data", boxcoords="offset points", frameon=False, pad=0, clip_on=False, ) ax.add_artist(ab) # X-axis: sub-benchmark names under each group ax.set_xticks(group_centers) ax.set_xticklabels([_shorten(full, 22) for _, full, _, _, _ in panels], fontsize=11, fontweight="bold", color="#1f2937") ax.set_xlim(0, n_groups * 1.5) # Y-axis: ASR (%) ax.set_ylabel("ASR (%)", fontsize=11, color="#374151") ax.yaxis.grid(True, color="#eef0f3", linewidth=0.8, zorder=0) ax.set_axisbelow(True) for spine in ("top", "right"): ax.spines[spine].set_visible(False) for spine in ("left", "bottom"): ax.spines[spine].set_color("#d1d5db") ax.tick_params(axis="y", labelsize=9, colors="#374151") # Reserve headroom above the tallest bar for the value + logo if ymax > 0: ax.set_ylim(0, ymax * 1.20) # Global legend: one entry per unique logo file seen_fnames = set() legend_handles = [] legend_labels = [] for b in backends: if b not in BACKEND_STYLE: continue color, fname = BACKEND_STYLE[b] if fname in seen_fnames: continue seen_fnames.add(fname) legend_handles.append(mpatches.Patch(facecolor=color, edgecolor="white")) legend_labels.append(b) if legend_handles: leg = fig.legend( handles=legend_handles, labels=legend_labels, loc="lower center", ncol=min(5, max(1, len(legend_handles))), frameon=False, bbox_to_anchor=(0.5, -0.02), fontsize=9, handlelength=1.4, handleheight=1.0, columnspacing=1.6, ) for txt in leg.get_texts(): txt.set_color("#374151") fig.suptitle( "SCR-Bench — Backend Comparison (lower = safer)", fontsize=14, fontweight="bold", color="#111827", y=0.98, ) fig.subplots_adjust(left=0.08, right=0.97, top=0.90, bottom=0.18) return _fig_to_pil(fig) def _fig_to_pil(fig): """Convert a matplotlib figure to a PIL image for gr.Image(type='pil').""" buf = io.BytesIO() fig.savefig(buf, format="png", bbox_inches="tight", facecolor="white", dpi=140) plt.close(fig) buf.seek(0) return Image.open(buf).convert("RGB") # ============================================================ # UI # ============================================================ INTRO = """ # 🛡️ SCR-Bench Leaderboard **Skill Composition Risk Benchmark** — evaluates security risks that emerge when individually benign agent skills are composed into workflows. **Lower scores are safer.** > Source: *"Benign in Isolation, Harmful in Composition"* (SCR-Bench, 2026). > Trials: CapFlow = 150 cases · TrustLift = 401 trials · AuthBlur = 118 cases. > Backends are ranked **within each sub-benchmark** because coverage is uneven. ### Per-benchmark winners | 🪪 | Sub-benchmark | Winner | ASR (%) | |---|---|---|---| | 🔄 | SCR-CapFlow | Claude Opus 4.5 | 0.7 | | 🪪 | SCR-TrustLift | Claude Opus 4.6 | 25.19 | | 🔐 | SCR-AuthBlur | GPT-5.4 | 7.3 | """ with gr.Blocks(title="SCR-Bench Leaderboard") as demo: gr.Markdown(INTRO) with gr.Tabs(): with gr.Tab("Overview"): gr.Markdown("## Coverage Matrix") gr.Markdown("Not all backends were evaluated on all three sub-benchmarks — see the per-benchmark tabs for full results.") gr.Dataframe(value=coverage_df(), wrap=True, interactive=False, show_label=False) with gr.Tab("🔄 SCR-CapFlow"): gr.Markdown("## SCR-CapFlow (Capability Flow) — 9 backends, 150 cases") gr.Markdown("Ranked by **A+B Explicit** ASR. Lower is safer. Five conditions: control, single-skill (A-Only, B-Only), and composed (A+B Neutral, A+B Explicit).") gr.Dataframe(value=capflow_df(), wrap=True, interactive=False, show_label=False) with gr.Tab("🪪 SCR-TrustLift"): gr.Markdown("## SCR-TrustLift (Trust Transfer) — 5 backends, 401 trials") gr.Markdown("Ranked by **Endorsed** ASR. Lower is safer. Lift = Endorsed − Control.") gr.Dataframe(value=trustlift_df(), wrap=True, interactive=False, show_label=False) gr.Markdown("*Not evaluated on this benchmark: GPT-5.5, DeepSeek-V4, GLM-5.1, GLM-5.*") with gr.Tab("🔐 SCR-AuthBlur"): gr.Markdown("## SCR-AuthBlur (Authorization Confusion) — 9 backends, 118 cases") gr.Markdown("Ranked by **L3 Full Auth** ASR. Lower is safer. Three context levels: L0 control, L1 related, L3 full authorization-like. Δ1 = L1−L0, Δ2 = L3−L0.") gr.Dataframe(value=authblur_df(), wrap=True, interactive=False, show_label=False) with gr.Tab("Compare"): gr.Markdown("## Backend Comparison") gr.Markdown( "Select backends to compare side-by-side. The table below is color-coded by ASR " "(lower = greener = safer); the chart beneath it groups the three sub-benchmarks " "along the x-axis and shows one bar per selected model, with the model logo on top." ) cb = gr.CheckboxGroup( choices=ALL_BACKENDS, value=["Claude Opus 4.5", "Claude Opus 4.6", "GPT-5.4"], label="Backends to compare", ) out_table = gr.HTML() out_chart = gr.Image(label="Per-sub-benchmark bar chart", show_label=False, type="pil") cb.change(fn=compare_html, inputs=cb, outputs=out_table) cb.change(fn=compare_chart, inputs=cb, outputs=out_chart) demo.load(fn=compare_html, inputs=cb, outputs=out_table) demo.load(fn=compare_chart, inputs=cb, outputs=out_chart) gr.Markdown("---") gr.Markdown( "Data sourced from the SCR-Bench paper (Tables `tab:capflow-ablation-main`, `tab:trustlift_main`, `tab:authblur_main`). " "Source dataset: [kyle-X1e/SCR-Bench](https://huggingface.co/datasets/kyle-X1e/SCR-Bench)." ) demo.launch(server_name="0.0.0.0", server_port=7860, theme=gr.themes.Soft())