app.py

import io
import os
import gradio as gr
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
from matplotlib.patches import FancyBboxPatch, Circle
from matplotlib.offsetbox import OffsetImage, AnnotationBbox
import numpy as np
import pandas as pd
from PIL import Image

LOGOS_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "logos")

# ============================================================
# Data (hardcoded from the SCR-Bench paper)
# ============================================================

ALL_BACKENDS = [
    "Claude Opus 4.5", "Claude Opus 4.6", "GPT-5.4", "GPT-5.5",
    "Gemini 3.1 Pro Preview", "MiniMax-M2.7",
    "DeepSeek-V4", "GLM-5.1", "GLM-5",
]

# Brand-style colors per backend, plus path to a logo image (under logos/).
# Multiple model versions from the same lab share one logo file. Colors are
# pastel/light variants in the spirit of the reference template.
BACKEND_STYLE = {
    "Claude Opus 4.5":        ("#F4C2A1", "claude-color.png"),  # Anthropic peach
    "Claude Opus 4.6":        ("#E89882", "claude-color.png"),  # Anthropic salmon
    "GPT-5.4":                ("#A8DCB8", "openai.png"),        # OpenAI light green
    "GPT-5.5":                ("#7AC59C", "openai.png"),        # OpenAI mid green
    "Gemini 3.1 Pro Preview": ("#A6BBE8", "gemini-color.png"),  # Google light blue
    "MiniMax-M2.7":           ("#C8B8DE", "minimax-color.png"), # MiniMax light purple
    "DeepSeek-V4":            ("#E5C8D6", "deepseek-color.png"),# DeepSeek pink
    "GLM-5.1":                ("#8FB4D8", "zhipu-color.png"),   # Zhipu steel blue
    "GLM-5":                  ("#D9C29A", "zhipu-color.png"),   # Zhipu tan
}

# CapFlow: (Control, A-Only, B-Only, A+B Neutral, A+B Explicit) ASR %
CAPFLOW = {
    "Claude Opus 4.5":        (0.0, 0.0, 1.2,  0.0,  0.7),
    "GPT-5.4":                (0.0, 0.0, 1.2,  4.4,  4.0),
    "Claude Opus 4.6":        (0.0, 0.0, 1.7,  1.3,  4.1),
    "GLM-5.1":                (0.0, 0.0, 1.3, 25.5, 26.9),
    "GLM-5":                  (0.0, 0.0, 0.7, 26.4, 30.7),
    "Gemini 3.1 Pro Preview": (0.0, 0.0, 1.3, 30.0, 41.9),
    "GPT-5.5":                (0.0, 0.0, 1.6, 48.1, 47.2),
    "MiniMax-M2.7":           (0.0, 0.0, 1.7, 75.5, 74.9),
    "DeepSeek-V4":            (0.0, 0.0, 1.5, 91.5, 92.5),
}

# TrustLift: (Control ASR, Endorsed ASR, Lift) %
TRUSTLIFT = {
    "Claude Opus 4.6":        (0.00,  25.19,  25.19),
    "GPT-5.4":                (0.00,  96.51,  96.51),
    "Gemini 3.1 Pro Preview": (5.49,  97.76,  92.27),
    "Claude Opus 4.5":        (0.00, 100.00, 100.00),
    "MiniMax-M2.7":           (0.00, 100.00, 100.00),
}

# AuthBlur: (L0 Control, L1 Related, L3 Full Auth, Δ1, Δ2) %
AUTHBLUR = {
    "GPT-5.4":                ( 9.5,  7.1,  7.3, -2.4, -2.2),
    "Claude Opus 4.5":        ( 8.7,  9.6, 13.1,  0.9,  4.4),
    "GLM-5.1":                (10.5,  8.9, 17.4, -1.6,  6.9),
    "Claude Opus 4.6":        ( 2.0, 10.0, 17.6,  8.0, 15.6),
    "GPT-5.5":                ( 2.9, 10.2, 17.6,  7.3, 14.7),
    "Gemini 3.1 Pro Preview": (10.0, 30.1, 35.0, 20.1, 25.0),
    "DeepSeek-V4":            (26.9, 40.6, 43.1, 13.7, 16.2),
    "MiniMax-M2.7":           (19.4, 31.9, 47.3, 12.5, 27.9),
    "GLM-5":                  (20.1, 40.0, 52.9, 19.9, 32.8),
}

# Coverage: (CapFlow?, TrustLift?, AuthBlur?)
COVERAGE = {
    "Claude Opus 4.5":        (True,  True,  True),
    "Claude Opus 4.6":        (True,  True,  True),
    "GPT-5.4":                (True,  True,  True),
    "Gemini 3.1 Pro Preview": (True,  True,  True),
    "MiniMax-M2.7":           (True,  True,  True),
    "GPT-5.5":                (True,  False, True),
    "DeepSeek-V4":            (True,  False, True),
    "GLM-5.1":                (True,  False, True),
    "GLM-5":                  (True,  False, True),
}

# ============================================================
# Color scale
# ============================================================
def asr_color(v):
    """Background color for an ASR value (lower = safer = greener)."""
    if v is None:
        return "#f5f5f5"
    if v < 0:
        return "#f9e8e7" if v <= -5 else "#fdf5f4"
    if v < 5:  return "#d4edda"
    if v < 15: return "#e8f5e9"
    if v < 30: return "#fff3cd"
    if v < 50: return "#ffe0b2"
    return "#ef9a9a"


# ============================================================
# DataFrame builders (sorted within each sub-benchmark)
# ============================================================
def capflow_df():
    rows = []
    for backend, vals in sorted(CAPFLOW.items(), key=lambda x: x[1][4]):
        rows.append([backend, *[f"{v:.1f}" for v in vals]])
    return pd.DataFrame(rows, columns=["Backend", "Control", "A-Only", "B-Only", "A+B Neutral", "A+B Explicit"])


def trustlift_df():
    rows = []
    for backend, vals in sorted(TRUSTLIFT.items(), key=lambda x: x[1][1]):
        rows.append([backend, *[f"{v:.2f}" for v in vals]])
    return pd.DataFrame(rows, columns=["Backend", "Control ASR", "Endorsed ASR", "Lift (pp)"])


def authblur_df():
    rows = []
    for backend, vals in sorted(AUTHBLUR.items(), key=lambda x: x[1][2]):
        formatted = [f"{v:.1f}" for v in vals[:3]] + [f"{vals[3]:+.1f}", f"{vals[4]:+.1f}"]
        rows.append([backend, *formatted])
    return pd.DataFrame(rows, columns=["Backend", "L0 Control", "L1 Related", "L3 Full Auth", "Δ1 (L1−L0)", "Δ2 (L3−L0)"])


def coverage_df():
    rows = []
    for backend in ALL_BACKENDS:
        cf, tl, ab = COVERAGE[backend]
        n = sum([cf, tl, ab])
        rows.append([backend, "✓" if cf else "—", "✓" if tl else "—", "✓" if ab else "—", n])
    return pd.DataFrame(rows, columns=["Backend", "CapFlow", "TrustLift", "AuthBlur", "# Sub-benchmarks"])


# ============================================================
# Compare tab — HTML table with cell coloring
# ============================================================
def compare_html(backends):
    if not backends or len(backends) < 2:
        return '<p style="color:#888;padding:20px;font-family:sans-serif">Select at least 2 backends to compare.</p>'

    html = ['<div style="overflow-x:auto;font-family:-apple-system,BlinkMacSystemFont,Segoe UI,sans-serif">']
    html.append('<table style="border-collapse:collapse;font-size:14px;width:100%">')
    html.append('<thead><tr style="background:#374151;color:white">')
    html.append('<th style="padding:10px 14px;text-align:left;border:1px solid #e5e7eb">Backend</th>')
    html.append('<th style="padding:10px 14px;text-align:right;border:1px solid #e5e7eb">CapFlow<br><span style="font-size:11px;font-weight:400;opacity:0.85">(A+B Explicit %)</span></th>')
    html.append('<th style="padding:10px 14px;text-align:right;border:1px solid #e5e7eb">TrustLift<br><span style="font-size:11px;font-weight:400;opacity:0.85">(Endorsed %)</span></th>')
    html.append('<th style="padding:10px 14px;text-align:right;border:1px solid #e5e7eb">AuthBlur<br><span style="font-size:11px;font-weight:400;opacity:0.85">(L3 %)</span></th>')
    html.append('</tr></thead><tbody>')

    for b in backends:
        cf = CAPFLOW.get(b)
        tl = TRUSTLIFT.get(b)
        ab = AUTHBLUR.get(b)
        cf_val = cf[4] if cf else None
        tl_val = tl[1] if tl else None
        ab_val = ab[2] if ab else None

        def cell(v, fmt="{:.1f}"):
            if v is None:
                return '<td style="padding:8px 14px;text-align:right;border:1px solid #e5e7eb;background:#f5f5f5;color:#999">—</td>'
            return f'<td style="padding:8px 14px;text-align:right;border:1px solid #e5e7eb;background:{asr_color(v)}">{fmt.format(v)}</td>'

        html.append('<tr style="background:#fafafa">')
        html.append(f'<td style="padding:8px 14px;text-align:left;border:1px solid #e5e7eb;font-weight:600">{b}</td>')
        html.append(cell(cf_val))
        html.append(cell(tl_val, "{:.2f}"))
        html.append(cell(ab_val))
        html.append('</tr>')

    html.append('</tbody></table>')

    legend_items = [
        ("#d4edda", "<5"),
        ("#e8f5e9", "5–15"),
        ("#fff3cd", "15–30"),
        ("#ffe0b2", "30–50"),
        ("#ef9a9a", "≥50"),
        ("#f5f5f5", "—"),
    ]
    html.append('<div style="margin-top:14px;font-size:12px;color:#555">')
    html.append('<b>Color scale (ASR %):</b> ')
    for color, label in legend_items:
        html.append(f'<span style="background:{color};padding:2px 8px;margin-right:6px;border:1px solid #ddd">{label}</span>')
    html.append('</div>')

    html.append('</div>')
    return ''.join(html)


# ============================================================
# Compare tab — matplotlib bar chart with real model logos
# ============================================================
# Caches for loaded logo images, keyed by filename -> OffsetImage.
_LOGO_CACHE: dict = {}


def _load_logo(backend: str, zoom: float = 0.18) -> OffsetImage | None:
    """Load the brand logo for a backend as a fresh OffsetImage each call.
    The image is normalized to a fixed display size and cropped to content."""
    style = BACKEND_STYLE.get(backend)
    if not style:
        return None
    fname = style[1]
    path = os.path.join(LOGOS_DIR, fname)
    if not os.path.exists(path):
        return None
    try:
        img = Image.open(path).convert("RGBA")
        # Crop transparent padding
        bbox = img.getbbox()
        if bbox:
            img = img.crop(bbox)
        # Resize so the largest dimension is 256px — keeps logos consistent in size
        w, h = img.size
        scale = 256 / max(w, h)
        if scale < 1:
            img = img.resize((max(1, int(w * scale)), max(1, int(h * scale))), Image.LANCZOS)
        return OffsetImage(np.asarray(img), zoom=zoom)
    except Exception:
        return None


def _hex_to_rgb(hexcol: str):
    h = hexcol.lstrip("#")
    return int(h[0:2], 16), int(h[2:4], 16), int(h[4:6], 16)


def _tint(hexcol: str, alpha: float = 0.18) -> str:
    r, g, b = _hex_to_rgb(hexcol)
    r = int(round(r + (255 - r) * alpha))
    g = int(round(g + (255 - g) * alpha))
    b = int(round(b + (255 - b) * alpha))
    return f"#{r:02x}{g:02x}{b:02x}"


def _shorten(s: str, n: int = 14) -> str:
    if len(s) <= n:
        return s
    return s[:n - 1] + "…"


def compare_chart(backends):
    """Grouped bar chart: x-axis = 3 sub-benchmarks, y-axis = ASR (%).
    Each x position has N bars (one per selected backend), colored by brand
    color, with the model logo placed on top of each bar."""
    if not backends or len(backends) < 1:
        fig, ax = plt.subplots(figsize=(8, 3), dpi=140)
        ax.text(0.5, 0.5, "Select at least one backend to render the chart.",
                ha="center", va="center", color="#888", fontsize=13)
        ax.set_axis_off()
        return _fig_to_pil(fig)

    # Sub-benchmark config: (short name, full title, data dict, value-index, fmt)
    panels = [
        ("SCR-CapFlow",   "SCR-CapFlow (A+B Explicit)",  CAPFLOW,   4, "{:.1f}"),
        ("SCR-TrustLift", "SCR-TrustLift (Endorsed)",    TRUSTLIFT, 1, "{:.2f}"),
        ("SCR-AuthBlur",  "SCR-AuthBlur (L3 Full Auth)", AUTHBLUR,  2, "{:.1f}"),
    ]

    # For each panel, gather (backend, value) for every selected backend that has data
    panel_rows = []
    for short, _title, data, idx, _fmt in panels:
        rows = [(b, data[b][idx]) for b in backends if b in data]
        panel_rows.append(rows)

    fig, ax = plt.subplots(figsize=(11, 6.5), dpi=140)
    fig.patch.set_facecolor("white")

    n_groups = len(panels)        # 3
    n_backends = len(backends)    # N
    # Each "group" occupies 1 unit on the x-axis, and within it we have n_backends
    # narrow bars. We leave a fixed group gap of 0.5 between groups.
    bar_w = 0.85 / max(n_backends, 1)
    group_centers = np.arange(n_groups) * 1.5 + 0.75

    # Track ymax across the whole chart to size the y-axis
    all_values = [v for rows in panel_rows for _, v in rows]
    ymax = max(all_values) if all_values else 1

    # Draw each group of bars
    for gi, (rows, (short, full, data, idx, fmt)) in enumerate(zip(panel_rows, panels)):
        center = group_centers[gi]
        # Positions for bars within this group, centered around `center`
        offsets = (np.arange(len(rows)) - (len(rows) - 1) / 2) * bar_w
        xs = center + offsets

        for ri, ((b, v), x) in enumerate(zip(rows, xs)):
            color = BACKEND_STYLE.get(b, ("#cbd5e1",))[0]
            ax.bar(x, v, width=bar_w * 0.92, color=color,
                   edgecolor="white", linewidth=1.0, zorder=2)
            # Value above the bar
            ax.text(x, v, fmt.format(v),
                    ha="center", va="bottom", fontsize=9,
                    fontweight="bold", color="#1f2937", zorder=4)
            # Logo above the value (smaller when many bars are crowded in a group)
            logo_zoom = 0.07 if n_backends >= 5 else 0.09 if n_backends >= 3 else 0.11
            logo = _load_logo(b, zoom=logo_zoom)
            if logo is not None:
                ab = AnnotationBbox(
                    logo, (x, v),
                    xybox=(0, 12), xycoords="data",
                    boxcoords="offset points",
                    frameon=False, pad=0, clip_on=False,
                )
                ax.add_artist(ab)

    # X-axis: sub-benchmark names under each group
    ax.set_xticks(group_centers)
    ax.set_xticklabels([_shorten(full, 22) for _, full, _, _, _ in panels],
                       fontsize=11, fontweight="bold", color="#1f2937")
    ax.set_xlim(0, n_groups * 1.5)

    # Y-axis: ASR (%)
    ax.set_ylabel("ASR (%)", fontsize=11, color="#374151")
    ax.yaxis.grid(True, color="#eef0f3", linewidth=0.8, zorder=0)
    ax.set_axisbelow(True)
    for spine in ("top", "right"):
        ax.spines[spine].set_visible(False)
    for spine in ("left", "bottom"):
        ax.spines[spine].set_color("#d1d5db")
    ax.tick_params(axis="y", labelsize=9, colors="#374151")

    # Reserve headroom above the tallest bar for the value + logo
    if ymax > 0:
        ax.set_ylim(0, ymax * 1.20)

    # Global legend: one entry per unique logo file
    seen_fnames = set()
    legend_handles = []
    legend_labels = []
    for b in backends:
        if b not in BACKEND_STYLE:
            continue
        color, fname = BACKEND_STYLE[b]
        if fname in seen_fnames:
            continue
        seen_fnames.add(fname)
        legend_handles.append(mpatches.Patch(facecolor=color, edgecolor="white"))
        legend_labels.append(b)

    if legend_handles:
        leg = fig.legend(
            handles=legend_handles, labels=legend_labels,
            loc="lower center", ncol=min(5, max(1, len(legend_handles))),
            frameon=False, bbox_to_anchor=(0.5, -0.02),
            fontsize=9, handlelength=1.4, handleheight=1.0, columnspacing=1.6,
        )
        for txt in leg.get_texts():
            txt.set_color("#374151")

    fig.suptitle(
        "SCR-Bench — Backend Comparison (lower = safer)",
        fontsize=14, fontweight="bold", color="#111827", y=0.98,
    )
    fig.subplots_adjust(left=0.08, right=0.97, top=0.90, bottom=0.18)
    return _fig_to_pil(fig)


def _fig_to_pil(fig):
    """Convert a matplotlib figure to a PIL image for gr.Image(type='pil')."""
    buf = io.BytesIO()
    fig.savefig(buf, format="png", bbox_inches="tight", facecolor="white", dpi=140)
    plt.close(fig)
    buf.seek(0)
    return Image.open(buf).convert("RGB")


# ============================================================
# UI
# ============================================================
INTRO = """
# 🛡️ SCR-Bench Leaderboard

**Skill Composition Risk Benchmark** — evaluates security risks that emerge when individually benign agent skills are composed into workflows. **Lower scores are safer.**

> Source: *"Benign in Isolation, Harmful in Composition"* (SCR-Bench, 2026).
> Trials: CapFlow = 150 cases · TrustLift = 401 trials · AuthBlur = 118 cases.
> Backends are ranked **within each sub-benchmark** because coverage is uneven.

### Per-benchmark winners
| 🪪 | Sub-benchmark | Winner | ASR (%) |
|---|---|---|---|
| 🔄 | SCR-CapFlow | Claude Opus 4.5 | 0.7 |
| 🪪 | SCR-TrustLift | Claude Opus 4.6 | 25.19 |
| 🔐 | SCR-AuthBlur | GPT-5.4 | 7.3 |
"""

with gr.Blocks(title="SCR-Bench Leaderboard") as demo:
    gr.Markdown(INTRO)

    with gr.Tabs():
        with gr.Tab("Overview"):
            gr.Markdown("## Coverage Matrix")
            gr.Markdown("Not all backends were evaluated on all three sub-benchmarks — see the per-benchmark tabs for full results.")
            gr.Dataframe(value=coverage_df(), wrap=True, interactive=False, show_label=False)

        with gr.Tab("🔄 SCR-CapFlow"):
            gr.Markdown("## SCR-CapFlow (Capability Flow) — 9 backends, 150 cases")
            gr.Markdown("Ranked by **A+B Explicit** ASR. Lower is safer. Five conditions: control, single-skill (A-Only, B-Only), and composed (A+B Neutral, A+B Explicit).")
            gr.Dataframe(value=capflow_df(), wrap=True, interactive=False, show_label=False)

        with gr.Tab("🪪 SCR-TrustLift"):
            gr.Markdown("## SCR-TrustLift (Trust Transfer) — 5 backends, 401 trials")
            gr.Markdown("Ranked by **Endorsed** ASR. Lower is safer. Lift = Endorsed − Control.")
            gr.Dataframe(value=trustlift_df(), wrap=True, interactive=False, show_label=False)
            gr.Markdown("*Not evaluated on this benchmark: GPT-5.5, DeepSeek-V4, GLM-5.1, GLM-5.*")

        with gr.Tab("🔐 SCR-AuthBlur"):
            gr.Markdown("## SCR-AuthBlur (Authorization Confusion) — 9 backends, 118 cases")
            gr.Markdown("Ranked by **L3 Full Auth** ASR. Lower is safer. Three context levels: L0 control, L1 related, L3 full authorization-like. Δ1 = L1−L0, Δ2 = L3−L0.")
            gr.Dataframe(value=authblur_df(), wrap=True, interactive=False, show_label=False)

        with gr.Tab("Compare"):
            gr.Markdown("## Backend Comparison")
            gr.Markdown(
                "Select backends to compare side-by-side. The table below is color-coded by ASR "
                "(lower = greener = safer); the chart beneath it groups the three sub-benchmarks "
                "along the x-axis and shows one bar per selected model, with the model logo on top."
            )
            cb = gr.CheckboxGroup(
                choices=ALL_BACKENDS,
                value=["Claude Opus 4.5", "Claude Opus 4.6", "GPT-5.4"],
                label="Backends to compare",
            )
            out_table = gr.HTML()
            out_chart = gr.Image(label="Per-sub-benchmark bar chart", show_label=False, type="pil")
            cb.change(fn=compare_html, inputs=cb, outputs=out_table)
            cb.change(fn=compare_chart, inputs=cb, outputs=out_chart)
            demo.load(fn=compare_html, inputs=cb, outputs=out_table)
            demo.load(fn=compare_chart, inputs=cb, outputs=out_chart)

    gr.Markdown("---")
    gr.Markdown(
        "Data sourced from the SCR-Bench paper (Tables `tab:capflow-ablation-main`, `tab:trustlift_main`, `tab:authblur_main`). "
        "Source dataset: [kyle-X1e/SCR-Bench](https://huggingface.co/datasets/kyle-X1e/SCR-Bench)."
    )

demo.launch(server_name="0.0.0.0", server_port=7860, theme=gr.themes.Soft())