""" edgebench — Tiny LLM benchmark results on Raspberry Pi 5. Interactive Gradio app showing cross-model and quant-sweep results. """ from html import escape from math import cos, pi, sin, sqrt import gradio as gr import pandas as pd # ── Cross-model data (all Q4_K_M, same 96 MC items: 48 MMLU-Pro + 48 MuSR) ── CROSS_MODEL = [ {"model": "LFM2.5-230M", "family": "Liquid AI", "params_m": 230, "accuracy": 0.229, "ci_low": 0.156, "ci_high": 0.323, "decode": 35.4, "prefill": 211.4, "pss_mb": 1849, "disk_mb": 153, "color": "#55D6BE"}, {"model": "LFM2.5-350M", "family": "Liquid AI", "params_m": 350, "accuracy": 0.281, "ci_low": 0.201, "ci_high": 0.378, "decode": 22.3, "prefill": 121.2, "pss_mb": 2011, "disk_mb": 229, "color": "#FF9F68"}, {"model": "OpenELM-270M", "family": "Apple", "params_m": 270, "accuracy": 0.198, "ci_low": 0.131, "ci_high": 0.289, "decode": 27.5, "prefill": 136.5, "pss_mb": 413, "disk_mb": 176, "color": "#B69CFF"}, {"model": "Qwen3-0.6B", "family": "Qwen", "params_m": 600, "accuracy": 0.135, "ci_low": 0.081, "ci_high": 0.218, "decode": 9.0, "prefill": 60.7, "pss_mb": 5306, "disk_mb": 397, "color": "#FF7087"}, {"model": "Qwen3.5-0.8B", "family": "Qwen", "params_m": 800, "accuracy": 0.250, "ci_low": 0.174, "ci_high": 0.345, "decode": 8.1, "prefill": 50.2, "pss_mb": 4245, "disk_mb": 508, "color": "#FFD166"}, ] # ── LFM2.5-230M quant sweep (full 256 items: 160 MMLU-Pro + 96 MuSR) ── QUANT_SWEEP = [ {"quant": "Q4_0", "accuracy": 0.188, "ci_low": 0.144, "ci_high": 0.240, "decode": 39.4, "prefill": 376.8, "pss_mb": 1838, "disk_mb": 149, "frontier": True}, {"quant": "Q4_K_M", "accuracy": 0.219, "ci_low": 0.172, "ci_high": 0.273, "decode": 36.4, "prefill": 219.3, "pss_mb": 1846, "disk_mb": 153, "frontier": True}, {"quant": "Q5_K_M", "accuracy": 0.184, "ci_low": 0.141, "ci_high": 0.236, "decode": 29.0, "prefill": 173.9, "pss_mb": 1873, "disk_mb": 172, "frontier": False}, {"quant": "Q6_K", "accuracy": 0.215, "ci_low": 0.169, "ci_high": 0.269, "decode": 29.4, "prefill": 204.4, "pss_mb": 1916, "disk_mb": 191, "frontier": False}, {"quant": "Q8_0", "accuracy": 0.207, "ci_low": 0.162, "ci_high": 0.261, "decode": 25.4, "prefill": 308.9, "pss_mb": 2022, "disk_mb": 247, "frontier": False}, ] BG = "#071512" PANEL = "#0D211C" PANEL_2 = "#112A23" INK = "#F5FBF8" MUTED = "#A6C0B5" GRID = "#29443A" ACCENT = "#55D6A7" FRONTIER = "#FF9F68" QUANT_OTHER = "#6ACFC7" METRIC_LABELS = { "accuracy": "Accuracy", "decode": "Decode speed (tok/s)", "prefill": "Prefill speed (tok/s)", "disk_mb": "Model size on disk (MB)", "pss_mb": "Peak memory / PSS (MB)", } def _range_with_padding(values, fraction=0.14, floor=None, ceiling=None): low = min(values) high = max(values) span = high - low or max(abs(high), 1) low -= span * fraction high += span * fraction if floor is not None: low = max(floor, low) if ceiling is not None: high = min(ceiling, high) return low, high def _scale(value, domain_low, domain_high, range_low, range_high): if domain_high == domain_low: return (range_low + range_high) / 2 ratio = (value - domain_low) / (domain_high - domain_low) return range_low + ratio * (range_high - range_low) def _ticks(low, high, count=6): if count < 2: return [low] step = (high - low) / (count - 1) return [low + step * index for index in range(count)] def _format_tick(metric, value): if metric == "accuracy": return f"{value * 100:.0f}%" if metric in {"disk_mb", "pss_mb"}: return f"{value:,.0f}" return f"{value:.0f}" def _svg_shell(title, subtitle, inner, aria_label, footer_note, legend=""): return f"""
{escape(title)} {escape(subtitle)} {legend} {inner} {escape(footer_note)}
""" def render_cross_svg(x_metric="decode", y_metric="accuracy"): width = 1000 height = 610 left = 92 right = 58 top = 112 bottom = 92 plot_width = width - left - right plot_height = height - top - bottom x_values = [row[x_metric] for row in CROSS_MODEL] y_values = [row[y_metric] for row in CROSS_MODEL] x_low, x_high = _range_with_padding(x_values, fraction=0.18) if y_metric == "accuracy": y_low, y_high = _range_with_padding( [row["ci_low"] for row in CROSS_MODEL] + [row["ci_high"] for row in CROSS_MODEL], fraction=0.08, floor=0, ceiling=1, ) else: y_low, y_high = _range_with_padding(y_values, fraction=0.17) x_position = lambda value: _scale(value, x_low, x_high, left, left + plot_width) y_position = lambda value: _scale(value, y_low, y_high, top + plot_height, top) elements = [] for tick in _ticks(x_low, x_high): x = x_position(tick) elements.append( f'' ) elements.append( f'{escape(_format_tick(x_metric, tick))}' ) for tick in _ticks(y_low, y_high): y = y_position(tick) elements.append( f'' ) elements.append( f'{escape(_format_tick(y_metric, tick))}' ) max_disk = max(row["disk_mb"] for row in CROSS_MODEL) x_midpoint = (x_low + x_high) / 2 for row in CROSS_MODEL: x = x_position(row[x_metric]) y = y_position(row[y_metric]) radius = 11 + 18 * sqrt(row["disk_mb"] / max_disk) if y_metric == "accuracy": y_ci_low = y_position(row["ci_low"]) y_ci_high = y_position(row["ci_high"]) elements.extend( [ f'', f'', f'', ] ) tooltip = ( f"{row['model']} — accuracy {row['accuracy'] * 100:.1f}%, " f"decode {row['decode']:.1f} tok/s, prefill {row['prefill']:.1f} tok/s, " f"peak memory {row['pss_mb']:,} MB, disk {row['disk_mb']} MB" ) elements.append( f'{escape(tooltip)}' ) place_right = row[x_metric] <= x_midpoint label_x = x + radius + 9 if place_right else x - radius - 9 anchor = "start" if place_right else "end" label_y = max(top + 14, min(top + plot_height - 8, y - radius - 5)) elements.append( f'{escape(row["model"])}' ) elements.append( f'{escape(METRIC_LABELS[x_metric])}' ) elements.append( f'{escape(METRIC_LABELS[y_metric])}' ) return _svg_shell( title=f"{METRIC_LABELS[y_metric]} vs {METRIC_LABELS[x_metric]}", subtitle="Q4_K_M across 96 multiple-choice items · whiskers show 95% confidence intervals", inner="".join(elements), aria_label=f"Cross-model bubble chart comparing {METRIC_LABELS[y_metric]} with {METRIC_LABELS[x_metric]}", footer_note="Bubble area represents model size on disk", ) def _star_points(center_x, center_y, outer_radius, inner_radius): points = [] for index in range(10): angle = -pi / 2 + index * pi / 5 radius = outer_radius if index % 2 == 0 else inner_radius points.append( f"{center_x + cos(angle) * radius:.2f},{center_y + sin(angle) * radius:.2f}" ) return " ".join(points) def render_quant_svg(): width = 1000 height = 610 left = 92 right = 58 top = 112 bottom = 92 plot_width = width - left - right plot_height = height - top - bottom x_low, x_high = 23.5, 41.2 y_low, y_high = 0.12, 0.29 x_position = lambda value: _scale(value, x_low, x_high, left, left + plot_width) y_position = lambda value: _scale(value, y_low, y_high, top + plot_height, top) elements = [] for tick in _ticks(x_low, x_high): x = x_position(tick) elements.append( f'' ) elements.append( f'{tick:.0f}' ) for tick in _ticks(y_low, y_high): y = y_position(tick) elements.append( f'' ) elements.append( f'{tick * 100:.0f}%' ) frontier_rows = sorted( (row for row in QUANT_SWEEP if row["frontier"]), key=lambda row: row["decode"], ) if len(frontier_rows) >= 2: frontier_path = " ".join( ( "M" if index == 0 else "L" ) + f" {x_position(row['decode']):.2f} {y_position(row['accuracy']):.2f}" for index, row in enumerate(frontier_rows) ) elements.append( f'' ) max_disk = max(row["disk_mb"] for row in QUANT_SWEEP) for row in QUANT_SWEEP: x = x_position(row["decode"]) y = y_position(row["accuracy"]) radius = 12 + 15 * sqrt(row["disk_mb"] / max_disk) color = FRONTIER if row["frontier"] else QUANT_OTHER y_ci_low = y_position(row["ci_low"]) y_ci_high = y_position(row["ci_high"]) elements.extend( [ f'', f'', f'', ] ) tooltip = ( f"{row['quant']} — accuracy {row['accuracy'] * 100:.1f}%, " f"decode {row['decode']:.1f} tok/s, prefill {row['prefill']:.1f} tok/s, " f"peak memory {row['pss_mb']:,} MB, disk {row['disk_mb']} MB" ) if row["frontier"]: points = _star_points(x, y, radius + 4, (radius + 4) * 0.48) elements.append( f'{escape(tooltip)} · Pareto frontier' ) else: elements.append( f'{escape(tooltip)}' ) label = f"★ {row['quant']}" if row["frontier"] else row["quant"] elements.append( f'{escape(label)}' ) elements.append( f'Decode speed (tok/s)' ) elements.append( f'Accuracy' ) legend = f""" Pareto frontier Other tested quants """ return _svg_shell( title="Accuracy–speed trade-off by quantization", subtitle="LFM2.5-230M across 256 multiple-choice items", inner="".join(elements), aria_label="Quantization bubble chart comparing accuracy and decode speed for LFM2.5-230M", footer_note="Stars mark the measured Pareto frontier", legend=legend, ) def cross_model_table(): df = pd.DataFrame(CROSS_MODEL) df["accuracy"] = (df["accuracy"] * 100).round(1).astype(str) + "%" df["decode"] = df["decode"].round(1).astype(str) + " tok/s" df["pss_mb"] = df["pss_mb"].astype(int).astype(str) + " MB" df["disk_mb"] = df["disk_mb"].astype(int).astype(str) + " MB" df = df[["model", "family", "params_m", "accuracy", "decode", "pss_mb", "disk_mb"]] df.columns = ["Model", "Family", "Params (M)", "Accuracy", "Decode", "Peak PSS", "Disk"] return df def quant_sweep_table(): df = pd.DataFrame(QUANT_SWEEP) df["accuracy"] = (df["accuracy"] * 100).round(1).astype(str) + "%" df["decode"] = df["decode"].round(1).astype(str) + " tok/s" df["pss_mb"] = df["pss_mb"].astype(int).astype(str) + " MB" df["disk_mb"] = df["disk_mb"].astype(int).astype(str) + " MB" df["frontier"] = df["frontier"].map({True: "★", False: ""}) df = df[["quant", "accuracy", "decode", "pss_mb", "disk_mb", "frontier"]] df.columns = ["Quant", "Accuracy", "Decode", "Peak PSS", "Disk", "Frontier"] return df HERO = """
RASPBERRY PI 5 · LOCAL INFERENCE

edgebench

A hardware-grounded look at the speed, accuracy, and memory trade-offs of tiny local language models.

Fastest decode35.4 tok/sLFM2.5-230M
Highest accuracy28.1%LFM2.5-350M
Lowest peak memory413 MBOpenELM-270M
Best quant balanceQ4_K_MLFM2.5-230M
""" CSS = """ :root { --edge-bg: #071512; --edge-panel: #0d211c; --edge-panel-2: #112a23; --edge-border: #29443a; --edge-ink: #f5fbf8; --edge-muted: #a6c0b5; --edge-accent: #55d6a7; } body, .gradio-container, .dark { background: var(--edge-bg) !important; } .gradio-container { max-width: 1160px !important; margin: 0 auto !important; padding-bottom: 2.5rem !important; color: var(--edge-ink) !important; } .hero { padding: 1.45rem 0 1.25rem; } .eyebrow { color: var(--edge-accent); font-size: .75rem; font-weight: 760; letter-spacing: .13em; margin-bottom: .45rem; } .hero h1 { color: var(--edge-ink); font-size: clamp(2.5rem, 6vw, 4.4rem); line-height: .98; letter-spacing: -.045em; margin: 0; } .hero-copy { color: var(--edge-muted); font-size: 1.05rem; line-height: 1.55; max-width: 720px; margin: .9rem 0 1.2rem; } .metric-grid { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: .75rem; } .metric-card { background: linear-gradient(145deg, var(--edge-panel-2), var(--edge-panel)); border: 1px solid var(--edge-border); border-radius: 14px; padding: .9rem 1rem; min-height: 108px; box-shadow: 0 12px 28px rgba(0,0,0,.14); } .metric-card span, .metric-card em { display: block; color: var(--edge-muted); font-size: .77rem; font-style: normal; } .metric-card strong { display: block; color: var(--edge-ink); font-size: 1.55rem; line-height: 1.2; margin: .38rem 0 .25rem; } .metric-card small { font-size: .72rem; color: var(--edge-muted); } .accent-card { border-color: rgba(85,214,167,.7); box-shadow: inset 0 0 0 1px rgba(85,214,167,.12), 0 12px 28px rgba(0,0,0,.14); } [role="tablist"] { border-bottom-color: var(--edge-border) !important; margin-top: .35rem; } [role="tab"] { font-weight: 650 !important; } [role="tab"][aria-selected="true"] { color: var(--edge-accent) !important; border-color: var(--edge-accent) !important; } .control-panel { border: 1px solid var(--edge-border) !important; background: rgba(13,33,28,.76) !important; border-radius: 14px !important; } .vector-chart { border: 0 !important; background: transparent !important; padding: 0 !important; margin-top: .75rem; } .vector-chart > div { padding: 0 !important; } .vector-figure { width: 100%; margin: 0; line-height: 0; filter: drop-shadow(0 18px 36px rgba(0,0,0,.16)); } .vector-figure svg { width: 100%; height: auto; display: block; } .table-wrap { border-color: var(--edge-border) !important; border-radius: 14px !important; overflow: hidden !important; } .visual-note { color: var(--edge-muted); font-size: .88rem; margin: .2rem 0 .65rem; } @media (max-width: 800px) { .metric-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } } @media (max-width: 520px) { .metric-grid { grid-template-columns: 1fr; } .hero { padding-top: .75rem; } } """ with gr.Blocks( theme=gr.themes.Monochrome(primary_hue="emerald", secondary_hue="teal"), css=CSS, ) as demo: gr.HTML(HERO) with gr.Tab("Model frontier"): gr.HTML( "

Choose two metrics to explore the trade-off. " "Hover over a point for the complete measurement record.

" ) with gr.Row(elem_classes="control-panel"): x_axis = gr.Radio( choices=[ ("Decode speed", "decode"), ("Prefill speed", "prefill"), ("Model size", "disk_mb"), ("Peak memory", "pss_mb"), ], value="decode", label="Horizontal axis", interactive=True, ) y_axis = gr.Radio( choices=[ ("Accuracy", "accuracy"), ("Decode speed", "decode"), ("Peak memory", "pss_mb"), ("Model size", "disk_mb"), ], value="accuracy", label="Vertical axis", interactive=True, ) chart_out = gr.HTML( value=render_cross_svg(), elem_classes="vector-chart", ) table_out = gr.Dataframe( value=cross_model_table(), label="Model comparison", interactive=False, elem_classes="table-wrap", ) x_axis.change( fn=render_cross_svg, inputs=[x_axis, y_axis], outputs=chart_out, show_progress="hidden", ) y_axis.change( fn=render_cross_svg, inputs=[x_axis, y_axis], outputs=chart_out, show_progress="hidden", ) with gr.Tab("Quantization frontier"): gr.HTML( "

Stars identify the measured Pareto frontier: " "no other tested quant is both faster and more accurate.

" ) gr.HTML(value=render_quant_svg(), elem_classes="vector-chart") gr.Dataframe( value=quant_sweep_table(), label="Quant comparison (★ = Pareto frontier)", interactive=False, elem_classes="table-wrap", ) with gr.Tab("Methodology"): gr.Markdown( "### Methodology\n\n" "- **Hardware:** Raspberry Pi 5 (Cortex-A76, 8 GB RAM, 4 CPU threads)\n" "- **Engine:** llama.cpp (greedy decode, `--temp 0`, 16-token cap)\n" "- **Workload:** phonebench modern suite — MMLU-Pro (10-choice) + MuSR (narrative reasoning)\n" "- **Cross-model:** 96 items (48 per dataset), all Q4_K_M\n" "- **Quant sweep:** 256 items (160 + 96), LFM2.5-230M across Q4_0 → Q8_0\n" "- **Visual encoding:** bubble area represents model size on disk; whiskers show 95% confidence intervals.\n\n" "### Key findings\n\n" "- **LFM2.5-350M** leads accuracy at 28.1%.\n" "- **LFM2.5-230M** leads decode speed at 35.4 tok/s.\n" "- **OpenELM-270M** uses the least memory at 413 MB peak PSS — 4.5× less than the next model.\n" "- **Qwen3.5-0.8B** reaches 25.0% accuracy but uses 4.2 GB of RAM.\n" "- **Q4_K_M** provides the strongest measured accuracy–speed balance for LFM2.5-230M.\n\n" "### Data\n\n" "All runs were measured with [phonebench](https://github.com/basbe/evals/tree/main/phonebench). " "Linux SBC runs are device-valid but excluded from the public phone leaderboard." ) if __name__ == "__main__": demo.launch()