"""
edgebench — Tiny LLM benchmark results on Raspberry Pi 5.
Interactive Gradio app showing cross-model and quant-sweep results.
"""
from html import escape
from math import cos, pi, sin, sqrt
import gradio as gr
import pandas as pd
# ── Cross-model data (all Q4_K_M, same 96 MC items: 48 MMLU-Pro + 48 MuSR) ──
CROSS_MODEL = [
{"model": "LFM2.5-230M", "family": "Liquid AI", "params_m": 230, "accuracy": 0.229, "ci_low": 0.156, "ci_high": 0.323, "decode": 35.4, "prefill": 211.4, "pss_mb": 1849, "disk_mb": 153, "color": "#55D6BE"},
{"model": "LFM2.5-350M", "family": "Liquid AI", "params_m": 350, "accuracy": 0.281, "ci_low": 0.201, "ci_high": 0.378, "decode": 22.3, "prefill": 121.2, "pss_mb": 2011, "disk_mb": 229, "color": "#FF9F68"},
{"model": "OpenELM-270M", "family": "Apple", "params_m": 270, "accuracy": 0.198, "ci_low": 0.131, "ci_high": 0.289, "decode": 27.5, "prefill": 136.5, "pss_mb": 413, "disk_mb": 176, "color": "#B69CFF"},
{"model": "Qwen3-0.6B", "family": "Qwen", "params_m": 600, "accuracy": 0.135, "ci_low": 0.081, "ci_high": 0.218, "decode": 9.0, "prefill": 60.7, "pss_mb": 5306, "disk_mb": 397, "color": "#FF7087"},
{"model": "Qwen3.5-0.8B", "family": "Qwen", "params_m": 800, "accuracy": 0.250, "ci_low": 0.174, "ci_high": 0.345, "decode": 8.1, "prefill": 50.2, "pss_mb": 4245, "disk_mb": 508, "color": "#FFD166"},
]
# ── LFM2.5-230M quant sweep (full 256 items: 160 MMLU-Pro + 96 MuSR) ──
QUANT_SWEEP = [
{"quant": "Q4_0", "accuracy": 0.188, "ci_low": 0.144, "ci_high": 0.240, "decode": 39.4, "prefill": 376.8, "pss_mb": 1838, "disk_mb": 149, "frontier": True},
{"quant": "Q4_K_M", "accuracy": 0.219, "ci_low": 0.172, "ci_high": 0.273, "decode": 36.4, "prefill": 219.3, "pss_mb": 1846, "disk_mb": 153, "frontier": True},
{"quant": "Q5_K_M", "accuracy": 0.184, "ci_low": 0.141, "ci_high": 0.236, "decode": 29.0, "prefill": 173.9, "pss_mb": 1873, "disk_mb": 172, "frontier": False},
{"quant": "Q6_K", "accuracy": 0.215, "ci_low": 0.169, "ci_high": 0.269, "decode": 29.4, "prefill": 204.4, "pss_mb": 1916, "disk_mb": 191, "frontier": False},
{"quant": "Q8_0", "accuracy": 0.207, "ci_low": 0.162, "ci_high": 0.261, "decode": 25.4, "prefill": 308.9, "pss_mb": 2022, "disk_mb": 247, "frontier": False},
]
BG = "#071512"
PANEL = "#0D211C"
PANEL_2 = "#112A23"
INK = "#F5FBF8"
MUTED = "#A6C0B5"
GRID = "#29443A"
ACCENT = "#55D6A7"
FRONTIER = "#FF9F68"
QUANT_OTHER = "#6ACFC7"
METRIC_LABELS = {
"accuracy": "Accuracy",
"decode": "Decode speed (tok/s)",
"prefill": "Prefill speed (tok/s)",
"disk_mb": "Model size on disk (MB)",
"pss_mb": "Peak memory / PSS (MB)",
}
def _range_with_padding(values, fraction=0.14, floor=None, ceiling=None):
low = min(values)
high = max(values)
span = high - low or max(abs(high), 1)
low -= span * fraction
high += span * fraction
if floor is not None:
low = max(floor, low)
if ceiling is not None:
high = min(ceiling, high)
return low, high
def _scale(value, domain_low, domain_high, range_low, range_high):
if domain_high == domain_low:
return (range_low + range_high) / 2
ratio = (value - domain_low) / (domain_high - domain_low)
return range_low + ratio * (range_high - range_low)
def _ticks(low, high, count=6):
if count < 2:
return [low]
step = (high - low) / (count - 1)
return [low + step * index for index in range(count)]
def _format_tick(metric, value):
if metric == "accuracy":
return f"{value * 100:.0f}%"
if metric in {"disk_mb", "pss_mb"}:
return f"{value:,.0f}"
return f"{value:.0f}"
def _svg_shell(title, subtitle, inner, aria_label, footer_note, legend=""):
return f"""
"""
def render_cross_svg(x_metric="decode", y_metric="accuracy"):
width = 1000
height = 610
left = 92
right = 58
top = 112
bottom = 92
plot_width = width - left - right
plot_height = height - top - bottom
x_values = [row[x_metric] for row in CROSS_MODEL]
y_values = [row[y_metric] for row in CROSS_MODEL]
x_low, x_high = _range_with_padding(x_values, fraction=0.18)
if y_metric == "accuracy":
y_low, y_high = _range_with_padding(
[row["ci_low"] for row in CROSS_MODEL]
+ [row["ci_high"] for row in CROSS_MODEL],
fraction=0.08,
floor=0,
ceiling=1,
)
else:
y_low, y_high = _range_with_padding(y_values, fraction=0.17)
x_position = lambda value: _scale(value, x_low, x_high, left, left + plot_width)
y_position = lambda value: _scale(value, y_low, y_high, top + plot_height, top)
elements = []
for tick in _ticks(x_low, x_high):
x = x_position(tick)
elements.append(
f''
)
elements.append(
f'{escape(_format_tick(x_metric, tick))}'
)
for tick in _ticks(y_low, y_high):
y = y_position(tick)
elements.append(
f''
)
elements.append(
f'{escape(_format_tick(y_metric, tick))}'
)
max_disk = max(row["disk_mb"] for row in CROSS_MODEL)
x_midpoint = (x_low + x_high) / 2
for row in CROSS_MODEL:
x = x_position(row[x_metric])
y = y_position(row[y_metric])
radius = 11 + 18 * sqrt(row["disk_mb"] / max_disk)
if y_metric == "accuracy":
y_ci_low = y_position(row["ci_low"])
y_ci_high = y_position(row["ci_high"])
elements.extend(
[
f'',
f'',
f'',
]
)
tooltip = (
f"{row['model']} — accuracy {row['accuracy'] * 100:.1f}%, "
f"decode {row['decode']:.1f} tok/s, prefill {row['prefill']:.1f} tok/s, "
f"peak memory {row['pss_mb']:,} MB, disk {row['disk_mb']} MB"
)
elements.append(
f'{escape(tooltip)}'
)
place_right = row[x_metric] <= x_midpoint
label_x = x + radius + 9 if place_right else x - radius - 9
anchor = "start" if place_right else "end"
label_y = max(top + 14, min(top + plot_height - 8, y - radius - 5))
elements.append(
f'{escape(row["model"])}'
)
elements.append(
f'{escape(METRIC_LABELS[x_metric])}'
)
elements.append(
f'{escape(METRIC_LABELS[y_metric])}'
)
return _svg_shell(
title=f"{METRIC_LABELS[y_metric]} vs {METRIC_LABELS[x_metric]}",
subtitle="Q4_K_M across 96 multiple-choice items · whiskers show 95% confidence intervals",
inner="".join(elements),
aria_label=f"Cross-model bubble chart comparing {METRIC_LABELS[y_metric]} with {METRIC_LABELS[x_metric]}",
footer_note="Bubble area represents model size on disk",
)
def _star_points(center_x, center_y, outer_radius, inner_radius):
points = []
for index in range(10):
angle = -pi / 2 + index * pi / 5
radius = outer_radius if index % 2 == 0 else inner_radius
points.append(
f"{center_x + cos(angle) * radius:.2f},{center_y + sin(angle) * radius:.2f}"
)
return " ".join(points)
def render_quant_svg():
width = 1000
height = 610
left = 92
right = 58
top = 112
bottom = 92
plot_width = width - left - right
plot_height = height - top - bottom
x_low, x_high = 23.5, 41.2
y_low, y_high = 0.12, 0.29
x_position = lambda value: _scale(value, x_low, x_high, left, left + plot_width)
y_position = lambda value: _scale(value, y_low, y_high, top + plot_height, top)
elements = []
for tick in _ticks(x_low, x_high):
x = x_position(tick)
elements.append(
f''
)
elements.append(
f'{tick:.0f}'
)
for tick in _ticks(y_low, y_high):
y = y_position(tick)
elements.append(
f''
)
elements.append(
f'{tick * 100:.0f}%'
)
frontier_rows = sorted(
(row for row in QUANT_SWEEP if row["frontier"]),
key=lambda row: row["decode"],
)
if len(frontier_rows) >= 2:
frontier_path = " ".join(
(
"M" if index == 0 else "L"
)
+ f" {x_position(row['decode']):.2f} {y_position(row['accuracy']):.2f}"
for index, row in enumerate(frontier_rows)
)
elements.append(
f''
)
max_disk = max(row["disk_mb"] for row in QUANT_SWEEP)
for row in QUANT_SWEEP:
x = x_position(row["decode"])
y = y_position(row["accuracy"])
radius = 12 + 15 * sqrt(row["disk_mb"] / max_disk)
color = FRONTIER if row["frontier"] else QUANT_OTHER
y_ci_low = y_position(row["ci_low"])
y_ci_high = y_position(row["ci_high"])
elements.extend(
[
f'',
f'',
f'',
]
)
tooltip = (
f"{row['quant']} — accuracy {row['accuracy'] * 100:.1f}%, "
f"decode {row['decode']:.1f} tok/s, prefill {row['prefill']:.1f} tok/s, "
f"peak memory {row['pss_mb']:,} MB, disk {row['disk_mb']} MB"
)
if row["frontier"]:
points = _star_points(x, y, radius + 4, (radius + 4) * 0.48)
elements.append(
f'{escape(tooltip)} · Pareto frontier'
)
else:
elements.append(
f'{escape(tooltip)}'
)
label = f"★ {row['quant']}" if row["frontier"] else row["quant"]
elements.append(
f'{escape(label)}'
)
elements.append(
f'Decode speed (tok/s)'
)
elements.append(
f'Accuracy'
)
legend = f"""
Pareto frontier
Other tested quants
"""
return _svg_shell(
title="Accuracy–speed trade-off by quantization",
subtitle="LFM2.5-230M across 256 multiple-choice items",
inner="".join(elements),
aria_label="Quantization bubble chart comparing accuracy and decode speed for LFM2.5-230M",
footer_note="Stars mark the measured Pareto frontier",
legend=legend,
)
def cross_model_table():
df = pd.DataFrame(CROSS_MODEL)
df["accuracy"] = (df["accuracy"] * 100).round(1).astype(str) + "%"
df["decode"] = df["decode"].round(1).astype(str) + " tok/s"
df["pss_mb"] = df["pss_mb"].astype(int).astype(str) + " MB"
df["disk_mb"] = df["disk_mb"].astype(int).astype(str) + " MB"
df = df[["model", "family", "params_m", "accuracy", "decode", "pss_mb", "disk_mb"]]
df.columns = ["Model", "Family", "Params (M)", "Accuracy", "Decode", "Peak PSS", "Disk"]
return df
def quant_sweep_table():
df = pd.DataFrame(QUANT_SWEEP)
df["accuracy"] = (df["accuracy"] * 100).round(1).astype(str) + "%"
df["decode"] = df["decode"].round(1).astype(str) + " tok/s"
df["pss_mb"] = df["pss_mb"].astype(int).astype(str) + " MB"
df["disk_mb"] = df["disk_mb"].astype(int).astype(str) + " MB"
df["frontier"] = df["frontier"].map({True: "★", False: ""})
df = df[["quant", "accuracy", "decode", "pss_mb", "disk_mb", "frontier"]]
df.columns = ["Quant", "Accuracy", "Decode", "Peak PSS", "Disk", "Frontier"]
return df
HERO = """
RASPBERRY PI 5 · LOCAL INFERENCE
🍓 edgebench
A hardware-grounded look at the speed, accuracy, and memory trade-offs of tiny local language models.
Fastest decode35.4 tok/sLFM2.5-230M
Highest accuracy28.1%LFM2.5-350M
Lowest peak memory413 MBOpenELM-270M
Best quant balanceQ4_K_MLFM2.5-230M
"""
CSS = """
:root {
--edge-bg: #071512;
--edge-panel: #0d211c;
--edge-panel-2: #112a23;
--edge-border: #29443a;
--edge-ink: #f5fbf8;
--edge-muted: #a6c0b5;
--edge-accent: #55d6a7;
}
body, .gradio-container, .dark { background: var(--edge-bg) !important; }
.gradio-container {
max-width: 1160px !important;
margin: 0 auto !important;
padding-bottom: 2.5rem !important;
color: var(--edge-ink) !important;
}
.hero { padding: 1.45rem 0 1.25rem; }
.eyebrow { color: var(--edge-accent); font-size: .75rem; font-weight: 760; letter-spacing: .13em; margin-bottom: .45rem; }
.hero h1 { color: var(--edge-ink); font-size: clamp(2.5rem, 6vw, 4.4rem); line-height: .98; letter-spacing: -.045em; margin: 0; }
.hero-copy { color: var(--edge-muted); font-size: 1.05rem; line-height: 1.55; max-width: 720px; margin: .9rem 0 1.2rem; }
.metric-grid { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: .75rem; }
.metric-card {
background: linear-gradient(145deg, var(--edge-panel-2), var(--edge-panel));
border: 1px solid var(--edge-border);
border-radius: 14px;
padding: .9rem 1rem;
min-height: 108px;
box-shadow: 0 12px 28px rgba(0,0,0,.14);
}
.metric-card span, .metric-card em { display: block; color: var(--edge-muted); font-size: .77rem; font-style: normal; }
.metric-card strong { display: block; color: var(--edge-ink); font-size: 1.55rem; line-height: 1.2; margin: .38rem 0 .25rem; }
.metric-card small { font-size: .72rem; color: var(--edge-muted); }
.accent-card { border-color: rgba(85,214,167,.7); box-shadow: inset 0 0 0 1px rgba(85,214,167,.12), 0 12px 28px rgba(0,0,0,.14); }
[role="tablist"] { border-bottom-color: var(--edge-border) !important; margin-top: .35rem; }
[role="tab"] { font-weight: 650 !important; }
[role="tab"][aria-selected="true"] { color: var(--edge-accent) !important; border-color: var(--edge-accent) !important; }
.control-panel { border: 1px solid var(--edge-border) !important; background: rgba(13,33,28,.76) !important; border-radius: 14px !important; }
.vector-chart { border: 0 !important; background: transparent !important; padding: 0 !important; margin-top: .75rem; }
.vector-chart > div { padding: 0 !important; }
.vector-figure { width: 100%; margin: 0; line-height: 0; filter: drop-shadow(0 18px 36px rgba(0,0,0,.16)); }
.vector-figure svg { width: 100%; height: auto; display: block; }
.table-wrap { border-color: var(--edge-border) !important; border-radius: 14px !important; overflow: hidden !important; }
.visual-note { color: var(--edge-muted); font-size: .88rem; margin: .2rem 0 .65rem; }
@media (max-width: 800px) { .metric-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } }
@media (max-width: 520px) { .metric-grid { grid-template-columns: 1fr; } .hero { padding-top: .75rem; } }
"""
with gr.Blocks(
theme=gr.themes.Monochrome(primary_hue="emerald", secondary_hue="teal"),
css=CSS,
) as demo:
gr.HTML(HERO)
with gr.Tab("Model frontier"):
gr.HTML(
"
Choose two metrics to explore the trade-off. "
"Hover over a point for the complete measurement record.
"
)
with gr.Row(elem_classes="control-panel"):
x_axis = gr.Radio(
choices=[
("Decode speed", "decode"),
("Prefill speed", "prefill"),
("Model size", "disk_mb"),
("Peak memory", "pss_mb"),
],
value="decode",
label="Horizontal axis",
interactive=True,
)
y_axis = gr.Radio(
choices=[
("Accuracy", "accuracy"),
("Decode speed", "decode"),
("Peak memory", "pss_mb"),
("Model size", "disk_mb"),
],
value="accuracy",
label="Vertical axis",
interactive=True,
)
chart_out = gr.HTML(
value=render_cross_svg(),
elem_classes="vector-chart",
)
table_out = gr.Dataframe(
value=cross_model_table(),
label="Model comparison",
interactive=False,
elem_classes="table-wrap",
)
x_axis.change(
fn=render_cross_svg,
inputs=[x_axis, y_axis],
outputs=chart_out,
show_progress="hidden",
)
y_axis.change(
fn=render_cross_svg,
inputs=[x_axis, y_axis],
outputs=chart_out,
show_progress="hidden",
)
with gr.Tab("Quantization frontier"):
gr.HTML(
"Stars identify the measured Pareto frontier: "
"no other tested quant is both faster and more accurate.
"
)
gr.HTML(value=render_quant_svg(), elem_classes="vector-chart")
gr.Dataframe(
value=quant_sweep_table(),
label="Quant comparison (★ = Pareto frontier)",
interactive=False,
elem_classes="table-wrap",
)
with gr.Tab("Methodology"):
gr.Markdown(
"### Methodology\n\n"
"- **Hardware:** Raspberry Pi 5 (Cortex-A76, 8 GB RAM, 4 CPU threads)\n"
"- **Engine:** llama.cpp (greedy decode, `--temp 0`, 16-token cap)\n"
"- **Workload:** phonebench modern suite — MMLU-Pro (10-choice) + MuSR (narrative reasoning)\n"
"- **Cross-model:** 96 items (48 per dataset), all Q4_K_M\n"
"- **Quant sweep:** 256 items (160 + 96), LFM2.5-230M across Q4_0 → Q8_0\n"
"- **Visual encoding:** bubble area represents model size on disk; whiskers show 95% confidence intervals.\n\n"
"### Key findings\n\n"
"- **LFM2.5-350M** leads accuracy at 28.1%.\n"
"- **LFM2.5-230M** leads decode speed at 35.4 tok/s.\n"
"- **OpenELM-270M** uses the least memory at 413 MB peak PSS — 4.5× less than the next model.\n"
"- **Qwen3.5-0.8B** reaches 25.0% accuracy but uses 4.2 GB of RAM.\n"
"- **Q4_K_M** provides the strongest measured accuracy–speed balance for LFM2.5-230M.\n\n"
"### Data\n\n"
"All runs were measured with [phonebench](https://github.com/basbe/evals/tree/main/phonebench). "
"Linux SBC runs are device-valid but excluded from the public phone leaderboard."
)
if __name__ == "__main__":
demo.launch()