""" Glokta-lite — Gradio dashboard backed directly by the HF Dataset. Four tabs: 1. Risk Leaderboard — risk-weighted pass rates; click a row to drill into Probe Results 2. Probe Results — raw probe-level data per model 3. Compare — overall pass rate across multiple models over time 4. Run Status — per-model scan status summary No database, no API server, no attempts data. Set HF_DATASET_REPO (default: Jake/glokta-public) before running. """ import os import gradio as gr import pandas as pd import plotly.graph_objects as go import data from risks import ACTIVE_RISKS, RISK_DEFINITIONS _PROBE_DETAIL_COLS = ["Probe Name", "Category", "Detector", "Pass", "Fail", "ASR", "Pass Rate"] _RISK_CHECKBOX_CHOICES = [(v["label"], k) for k, v in RISK_DEFINITIONS.items() if v["enabled"]] _RISK_CHECKBOX_DEFAULT = ACTIVE_RISKS # --------------------------------------------------------------------------- # Display helpers # --------------------------------------------------------------------------- def _probe_row(pr: dict) -> dict: total = pr["pass_count"] + pr["fail_count"] pass_rate = pr["pass_count"] / total if total > 0 else 0.0 return { "Probe Name": pr["probe_name"], "Category": pr["probe_category"], "Detector": pr["detector"], "Pass": pr["pass_count"], "Fail": pr["fail_count"], "ASR": f"{pr['score']:.3f}" if pr.get("score") is not None else "N/A", "Pass Rate": f"{pass_rate:.1%}", } def _empty_fig(message: str) -> go.Figure: fig = go.Figure() fig.add_annotation( text=message, xref="paper", yref="paper", x=0.5, y=0.5, showarrow=False, font=dict(size=14), ) fig.update_layout(xaxis_visible=False, yaxis_visible=False) return fig # --------------------------------------------------------------------------- # DataFrame builders (mirror the main app's fetch_* functions) # --------------------------------------------------------------------------- def build_leaderboard_df(probe_category: str, model_id: str) -> pd.DataFrame: rows = data.get_leaderboard( probe_category=probe_category if probe_category and probe_category != "All" else None, model_id=model_id if model_id else None, ) if not rows: return pd.DataFrame(columns=["Model", "Provider", "Probe Category", "Pass", "Fail", "ASR", "Pass Rate"]) return pd.DataFrame([{ "Model": r["model_name"], "Provider": r["provider"], "Probe Category": r["probe_category"], "Pass": r["total_pass"], "Fail": r["total_fail"], "ASR": f"{r['score']:.3f}" if r["score"] is not None else "N/A", "Pass Rate": f"{r['pass_rate']:.1%}", "Origin": r.get("origin", "api"), } for r in rows]) def build_risk_leaderboard_df(included_risks: list[str]) -> pd.DataFrame: empty = pd.DataFrame(columns=["Model", "Provider", "Overall Pass Rate", "Coverage"]) if not included_risks: return empty rows = data.get_risk_leaderboard(included_risks) if not rows: return empty result = [] for m in rows: per_risk = m.get("per_risk", {}) n_covered = sum(1 for r in included_risks if per_risk.get(r) is not None) overall_str = f"{m['overall_pass_rate']:.1%}" if m["overall_pass_rate"] is not None else "N/A" coverage_str = f"{n_covered}/{len(included_risks)}" if n_covered < len(included_risks) else "✓" result.append({ "Model": m["model_name"], "Provider": m["provider"], "Overall Pass Rate": overall_str, "Coverage": coverage_str, }) return pd.DataFrame(result) def build_model_detail_df(model_id: str) -> tuple[pd.DataFrame, str | None]: empty = pd.DataFrame(columns=_PROBE_DETAIL_COLS) if not model_id: return empty, None detail = data.get_model_detail(model_id) if not detail or not detail.get("probe_results"): return empty, None rows = [_probe_row(pr) for pr in detail["probe_results"]] return pd.DataFrame(rows), detail.get("run_id") def build_run_summary_df() -> pd.DataFrame: rows = data.get_run_summary() if not rows: return pd.DataFrame(columns=["Model", "Provider", "Complete", "Running", "Pending", "Failed", "Latest Origin"]) return pd.DataFrame([{ "Model": r["model_name"], "Provider": r["provider"], "Complete": r["complete"], "Running": r["running"], "Pending": r["pending"], "Failed": r["failed"], "Latest Origin": r.get("latest_origin", ""), } for r in rows]) def build_run_probe_df(run_id: str) -> pd.DataFrame: empty = pd.DataFrame(columns=_PROBE_DETAIL_COLS) if not run_id: return empty rows = data.get_run_probe_results(run_id) return pd.DataFrame([_probe_row(pr) for pr in rows]) if rows else empty def make_compare_plot(model_ids: list[str], included_risks: list[str]) -> go.Figure: fig = go.Figure() plotted = 0 for model_id in model_ids: result = data.get_trends(model_id, included_risks) if not result or not result.get("points"): continue points = result["points"] dates = [p["completed_at"] for p in points] y_vals = [p.get("overall_pass_rate") for p in points] if any(v is not None for v in y_vals): fig.add_trace(go.Scatter( x=dates, y=y_vals, mode="lines+markers", name=result["model_name"], connectgaps=True, )) plotted += 1 if plotted == 0: return _empty_fig("No scan history for selected models.") fig.update_layout( title="Model Comparison — Overall Risk Pass Rate Over Time", xaxis_title="Scan Date", yaxis=dict(title="Pass Rate", range=[0, 1], tickformat=".0%"), ) return fig # --------------------------------------------------------------------------- # Gradio UI # --------------------------------------------------------------------------- def build_app() -> gr.Blocks: with gr.Blocks(title="Glokta Lite", theme=gr.themes.Soft()) as demo: gr.Markdown( f""" # Glokta Lite — LLM Security Leaderboard [Main Project](https://github.com/JakeBx/Glokta) to reference for self-hosting options Powered by [garak](https://github.com/NVIDIA/garak) · data from `{data.HF_DATASET_REPO}` """ ) current_run_id = gr.State(value=None) model_name_to_id = gr.State(value={}) with gr.Tabs() as tabs: # ---------------------------------------------------------------- # Tab 1: Risk Leaderboard # ---------------------------------------------------------------- with gr.Tab("Risk Leaderboard", id="risk_leaderboard"): gr.Markdown( "Overall pass rate = mean of per-risk pass rates for selected risks. " "Sorted safest-first. **Click a row to drill into probe results.**" ) with gr.Row(): risk_filter = gr.CheckboxGroup( label="Include Risks", choices=_RISK_CHECKBOX_CHOICES, value=_RISK_CHECKBOX_DEFAULT, scale=4, ) risk_refresh_btn = gr.Button("Refresh", scale=1, variant="secondary") risk_table = gr.Dataframe(label="Risk Leaderboard", interactive=False, wrap=True) # ---------------------------------------------------------------- # Tab 2: Probe Results # ---------------------------------------------------------------- with gr.Tab("Probe Results", id="probe_results"): with gr.Row(): category_filter = gr.Dropdown( label="Probe Category", choices=["All"], value="All", interactive=True, scale=2, ) model_filter = gr.Dropdown( label="Model", choices=[("All", "")], value="", interactive=True, scale=3, ) probe_refresh_btn = gr.Button("Refresh", scale=1, variant="secondary") leaderboard_table = gr.Dataframe(label="Probe Results", interactive=False, wrap=True) gr.Markdown("### Per-Model Probe Breakdown") gr.Markdown("*Select a model from the dropdown above. Click a row to see attempt-level detail.*") detail_table = gr.Dataframe( label="Probe Details", interactive=False, wrap=True, ) # ---------------------------------------------------------------- # Tab 3: Compare # ---------------------------------------------------------------- with gr.Tab("Compare", id="compare"): gr.Markdown( "Overall pass rate across multiple models over time. " "Risk filter affects the overall pass rate calculation." ) with gr.Row(): compare_models_input = gr.Dropdown( label="Models (select multiple)", choices=[], value=[], multiselect=True, interactive=True, scale=4, ) compare_refresh_btn = gr.Button("Refresh", scale=1, variant="secondary") compare_risk_filter = gr.CheckboxGroup( label="Risk Categories", choices=_RISK_CHECKBOX_CHOICES, value=_RISK_CHECKBOX_DEFAULT, ) compare_plot = gr.Plot(label="Model Comparison", value=_empty_fig("Select models to compare.")) # ---------------------------------------------------------------- # Tab 4: Run Status # ---------------------------------------------------------------- with gr.Tab("Run Status", id="run_status"): gr.Markdown("Per-model scan status summary.") run_summary_table = gr.Dataframe( label="Run Status by Model", interactive=False, wrap=True, ) gr.Row() with gr.Row(): reload_btn = gr.Button("Reload Data from HF", variant="primary") run_refresh_btn = gr.Button("Refresh Table", variant="secondary") # -------------------------------------------------------------------- # Event handlers # -------------------------------------------------------------------- def on_load(): try: data.load_data() except Exception as exc: print(f"[app] Data load failed: {exc}") categories = ["All"] + data.get_probe_categories() models = [(m["name"], m["id"]) for m in data.get_models()] model_choices_with_all = [("All", "")] + models name_to_id = {m["name"]: m["id"] for m in data.get_models()} return ( gr.update(choices=categories, value="All"), gr.update(choices=model_choices_with_all, value=""), build_leaderboard_df("All", ""), build_risk_leaderboard_df(_RISK_CHECKBOX_DEFAULT), name_to_id, gr.update(choices=models, value=[]), build_run_summary_df(), ) def on_reload(): try: data.load_data() except Exception as exc: print(f"[app] Reload failed: {exc}") return build_run_summary_df() def on_probe_filter_change(probe_category: str, model_id: str): df = build_leaderboard_df(probe_category, model_id) if model_id: detail_df, run_id = build_model_detail_df(model_id) else: detail_df = pd.DataFrame(columns=_PROBE_DETAIL_COLS) run_id = None return df, detail_df, run_id def on_risk_filter_change(included_risks: list[str]): return build_risk_leaderboard_df(included_risks) def on_risk_row_click(evt: gr.SelectData, risk_df: pd.DataFrame, name_to_id: dict): try: model_name = str(risk_df.iloc[evt.index[0]]["Model"]).strip() except Exception: return gr.update(), gr.update(), pd.DataFrame(columns=_PROBE_DETAIL_COLS), None, gr.update() model_id = name_to_id.get(model_name, "") leaderboard_df = build_leaderboard_df("All", model_id) detail_df, run_id = build_model_detail_df(model_id) if model_id else (pd.DataFrame(columns=_PROBE_DETAIL_COLS), None) return ( gr.update(value=model_id), # model_filter leaderboard_df, # leaderboard_table detail_df, # detail_table run_id, # current_run_id gr.update(selected="probe_results"), # tabs ) def on_compare_update(selected_values: list[str], included_risks: list[str]): if not selected_values or not included_risks: return _empty_fig("Select models to compare.") return make_compare_plot(selected_values, included_risks) # -------------------------------------------------------------------- # Wire events # -------------------------------------------------------------------- demo.load( fn=on_load, inputs=None, outputs=[ category_filter, model_filter, leaderboard_table, risk_table, model_name_to_id, compare_models_input, run_summary_table, ], ) risk_filter.change(fn=on_risk_filter_change, inputs=[risk_filter], outputs=[risk_table]) risk_refresh_btn.click(fn=on_risk_filter_change, inputs=[risk_filter], outputs=[risk_table]) risk_table.select( fn=on_risk_row_click, inputs=[risk_table, model_name_to_id], outputs=[model_filter, leaderboard_table, detail_table, current_run_id, tabs], ) probe_refresh_btn.click( fn=on_probe_filter_change, inputs=[category_filter, model_filter], outputs=[leaderboard_table, detail_table, current_run_id], ) category_filter.change( fn=on_probe_filter_change, inputs=[category_filter, model_filter], outputs=[leaderboard_table, detail_table, current_run_id], ) model_filter.change( fn=on_probe_filter_change, inputs=[category_filter, model_filter], outputs=[leaderboard_table, detail_table, current_run_id], ) compare_models_input.change( fn=on_compare_update, inputs=[compare_models_input, compare_risk_filter], outputs=[compare_plot] ) compare_risk_filter.change( fn=on_compare_update, inputs=[compare_models_input, compare_risk_filter], outputs=[compare_plot] ) compare_refresh_btn.click( fn=on_compare_update, inputs=[compare_models_input, compare_risk_filter], outputs=[compare_plot] ) reload_btn.click(fn=on_reload, inputs=None, outputs=[run_summary_table]) run_refresh_btn.click(fn=build_run_summary_df, inputs=None, outputs=[run_summary_table]) return demo if __name__ == "__main__": port = int(os.environ.get("GRADIO_SERVER_PORT", 7860)) app = build_app() app.launch(server_name="0.0.0.0", server_port=port, show_api=False)