| """ |
| edgebench — Tiny LLM benchmark results on Raspberry Pi 5. |
| Interactive Gradio app showing cross-model and quant-sweep results. |
| """ |
|
|
| from html import escape |
| from math import cos, pi, sin, sqrt |
|
|
| import gradio as gr |
| import pandas as pd |
|
|
|
|
| |
| CROSS_MODEL = [ |
| {"model": "LFM2.5-230M", "family": "Liquid AI", "params_m": 230, "accuracy": 0.229, "ci_low": 0.156, "ci_high": 0.323, "decode": 35.4, "prefill": 211.4, "pss_mb": 1849, "disk_mb": 153, "color": "#55D6BE"}, |
| {"model": "LFM2.5-350M", "family": "Liquid AI", "params_m": 350, "accuracy": 0.281, "ci_low": 0.201, "ci_high": 0.378, "decode": 22.3, "prefill": 121.2, "pss_mb": 2011, "disk_mb": 229, "color": "#FF9F68"}, |
| {"model": "OpenELM-270M", "family": "Apple", "params_m": 270, "accuracy": 0.198, "ci_low": 0.131, "ci_high": 0.289, "decode": 27.5, "prefill": 136.5, "pss_mb": 413, "disk_mb": 176, "color": "#B69CFF"}, |
| {"model": "Qwen3-0.6B", "family": "Qwen", "params_m": 600, "accuracy": 0.135, "ci_low": 0.081, "ci_high": 0.218, "decode": 9.0, "prefill": 60.7, "pss_mb": 5306, "disk_mb": 397, "color": "#FF7087"}, |
| {"model": "Qwen3.5-0.8B", "family": "Qwen", "params_m": 800, "accuracy": 0.250, "ci_low": 0.174, "ci_high": 0.345, "decode": 8.1, "prefill": 50.2, "pss_mb": 4245, "disk_mb": 508, "color": "#FFD166"}, |
| ] |
|
|
|
|
| |
| QUANT_SWEEP = [ |
| {"quant": "Q4_0", "accuracy": 0.188, "ci_low": 0.144, "ci_high": 0.240, "decode": 39.4, "prefill": 376.8, "pss_mb": 1838, "disk_mb": 149, "frontier": True}, |
| {"quant": "Q4_K_M", "accuracy": 0.219, "ci_low": 0.172, "ci_high": 0.273, "decode": 36.4, "prefill": 219.3, "pss_mb": 1846, "disk_mb": 153, "frontier": True}, |
| {"quant": "Q5_K_M", "accuracy": 0.184, "ci_low": 0.141, "ci_high": 0.236, "decode": 29.0, "prefill": 173.9, "pss_mb": 1873, "disk_mb": 172, "frontier": False}, |
| {"quant": "Q6_K", "accuracy": 0.215, "ci_low": 0.169, "ci_high": 0.269, "decode": 29.4, "prefill": 204.4, "pss_mb": 1916, "disk_mb": 191, "frontier": False}, |
| {"quant": "Q8_0", "accuracy": 0.207, "ci_low": 0.162, "ci_high": 0.261, "decode": 25.4, "prefill": 308.9, "pss_mb": 2022, "disk_mb": 247, "frontier": False}, |
| ] |
|
|
|
|
| BG = "#071512" |
| PANEL = "#0D211C" |
| PANEL_2 = "#112A23" |
| INK = "#F5FBF8" |
| MUTED = "#A6C0B5" |
| GRID = "#29443A" |
| ACCENT = "#55D6A7" |
| FRONTIER = "#FF9F68" |
| QUANT_OTHER = "#6ACFC7" |
|
|
| METRIC_LABELS = { |
| "accuracy": "Accuracy", |
| "decode": "Decode speed (tok/s)", |
| "prefill": "Prefill speed (tok/s)", |
| "disk_mb": "Model size on disk (MB)", |
| "pss_mb": "Peak memory / PSS (MB)", |
| } |
|
|
|
|
| def _range_with_padding(values, fraction=0.14, floor=None, ceiling=None): |
| low = min(values) |
| high = max(values) |
| span = high - low or max(abs(high), 1) |
| low -= span * fraction |
| high += span * fraction |
| if floor is not None: |
| low = max(floor, low) |
| if ceiling is not None: |
| high = min(ceiling, high) |
| return low, high |
|
|
|
|
| def _scale(value, domain_low, domain_high, range_low, range_high): |
| if domain_high == domain_low: |
| return (range_low + range_high) / 2 |
| ratio = (value - domain_low) / (domain_high - domain_low) |
| return range_low + ratio * (range_high - range_low) |
|
|
|
|
| def _ticks(low, high, count=6): |
| if count < 2: |
| return [low] |
| step = (high - low) / (count - 1) |
| return [low + step * index for index in range(count)] |
|
|
|
|
| def _format_tick(metric, value): |
| if metric == "accuracy": |
| return f"{value * 100:.0f}%" |
| if metric in {"disk_mb", "pss_mb"}: |
| return f"{value:,.0f}" |
| return f"{value:.0f}" |
|
|
|
|
| def _svg_shell(title, subtitle, inner, aria_label, footer_note, legend=""): |
| return f""" |
| <figure class="vector-figure" aria-label="{escape(aria_label)}"> |
| <svg viewBox="0 0 1000 610" role="img" aria-label="{escape(aria_label)}" preserveAspectRatio="xMidYMid meet"> |
| <defs> |
| <filter id="soft-shadow" x="-30%" y="-30%" width="160%" height="160%"> |
| <feDropShadow dx="0" dy="5" stdDeviation="7" flood-color="#000000" flood-opacity="0.28"/> |
| </filter> |
| <linearGradient id="panel-glow" x1="0" y1="0" x2="1" y2="1"> |
| <stop offset="0%" stop-color="#112A23"/> |
| <stop offset="100%" stop-color="#0D211C"/> |
| </linearGradient> |
| </defs> |
| <style> |
| .svg-copy {{ font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; }} |
| .axis-label {{ fill: {MUTED}; font-size: 14px; font-weight: 600; }} |
| .tick-label {{ fill: {MUTED}; font-size: 12px; }} |
| .point-label {{ fill: {INK}; font-size: 13px; font-weight: 750; paint-order: stroke; stroke: {PANEL}; stroke-width: 5px; stroke-linejoin: round; }} |
| </style> |
| <rect x="1" y="1" width="998" height="608" rx="20" fill="url(#panel-glow)" stroke="{GRID}" stroke-width="2"/> |
| <text class="svg-copy" x="58" y="48" fill="{INK}" font-size="22" font-weight="760">{escape(title)}</text> |
| <text class="svg-copy" x="58" y="74" fill="{MUTED}" font-size="13">{escape(subtitle)}</text> |
| {legend} |
| {inner} |
| <text class="svg-copy" x="942" y="585" fill="{MUTED}" font-size="12" text-anchor="end">{escape(footer_note)}</text> |
| </svg> |
| </figure> |
| """ |
|
|
|
|
| def render_cross_svg(x_metric="decode", y_metric="accuracy"): |
| width = 1000 |
| height = 610 |
| left = 92 |
| right = 58 |
| top = 112 |
| bottom = 92 |
| plot_width = width - left - right |
| plot_height = height - top - bottom |
|
|
| x_values = [row[x_metric] for row in CROSS_MODEL] |
| y_values = [row[y_metric] for row in CROSS_MODEL] |
| x_low, x_high = _range_with_padding(x_values, fraction=0.18) |
|
|
| if y_metric == "accuracy": |
| y_low, y_high = _range_with_padding( |
| [row["ci_low"] for row in CROSS_MODEL] |
| + [row["ci_high"] for row in CROSS_MODEL], |
| fraction=0.08, |
| floor=0, |
| ceiling=1, |
| ) |
| else: |
| y_low, y_high = _range_with_padding(y_values, fraction=0.17) |
|
|
| x_position = lambda value: _scale(value, x_low, x_high, left, left + plot_width) |
| y_position = lambda value: _scale(value, y_low, y_high, top + plot_height, top) |
|
|
| elements = [] |
| for tick in _ticks(x_low, x_high): |
| x = x_position(tick) |
| elements.append( |
| f'<line x1="{x:.2f}" y1="{top}" x2="{x:.2f}" y2="{top + plot_height}" stroke="{GRID}" stroke-width="1" opacity="0.75"/>' |
| ) |
| elements.append( |
| f'<text class="svg-copy tick-label" x="{x:.2f}" y="{top + plot_height + 28}" text-anchor="middle">{escape(_format_tick(x_metric, tick))}</text>' |
| ) |
|
|
| for tick in _ticks(y_low, y_high): |
| y = y_position(tick) |
| elements.append( |
| f'<line x1="{left}" y1="{y:.2f}" x2="{left + plot_width}" y2="{y:.2f}" stroke="{GRID}" stroke-width="1" opacity="0.75"/>' |
| ) |
| elements.append( |
| f'<text class="svg-copy tick-label" x="{left - 16}" y="{y + 4:.2f}" text-anchor="end">{escape(_format_tick(y_metric, tick))}</text>' |
| ) |
|
|
| max_disk = max(row["disk_mb"] for row in CROSS_MODEL) |
| x_midpoint = (x_low + x_high) / 2 |
| for row in CROSS_MODEL: |
| x = x_position(row[x_metric]) |
| y = y_position(row[y_metric]) |
| radius = 11 + 18 * sqrt(row["disk_mb"] / max_disk) |
|
|
| if y_metric == "accuracy": |
| y_ci_low = y_position(row["ci_low"]) |
| y_ci_high = y_position(row["ci_high"]) |
| elements.extend( |
| [ |
| f'<line x1="{x:.2f}" y1="{y_ci_high:.2f}" x2="{x:.2f}" y2="{y_ci_low:.2f}" stroke="{row["color"]}" stroke-width="3" opacity="0.65"/>', |
| f'<line x1="{x - 8:.2f}" y1="{y_ci_high:.2f}" x2="{x + 8:.2f}" y2="{y_ci_high:.2f}" stroke="{row["color"]}" stroke-width="3" opacity="0.65"/>', |
| f'<line x1="{x - 8:.2f}" y1="{y_ci_low:.2f}" x2="{x + 8:.2f}" y2="{y_ci_low:.2f}" stroke="{row["color"]}" stroke-width="3" opacity="0.65"/>', |
| ] |
| ) |
|
|
| tooltip = ( |
| f"{row['model']} — accuracy {row['accuracy'] * 100:.1f}%, " |
| f"decode {row['decode']:.1f} tok/s, prefill {row['prefill']:.1f} tok/s, " |
| f"peak memory {row['pss_mb']:,} MB, disk {row['disk_mb']} MB" |
| ) |
| elements.append( |
| f'<circle cx="{x:.2f}" cy="{y:.2f}" r="{radius:.2f}" fill="{row["color"]}" fill-opacity="0.92" stroke="{INK}" stroke-width="2" filter="url(#soft-shadow)"><title>{escape(tooltip)}</title></circle>' |
| ) |
|
|
| place_right = row[x_metric] <= x_midpoint |
| label_x = x + radius + 9 if place_right else x - radius - 9 |
| anchor = "start" if place_right else "end" |
| label_y = max(top + 14, min(top + plot_height - 8, y - radius - 5)) |
| elements.append( |
| f'<text class="svg-copy point-label" x="{label_x:.2f}" y="{label_y:.2f}" text-anchor="{anchor}">{escape(row["model"])}</text>' |
| ) |
|
|
| elements.append( |
| f'<text class="svg-copy axis-label" x="{left + plot_width / 2:.2f}" y="{height - 44}" text-anchor="middle">{escape(METRIC_LABELS[x_metric])}</text>' |
| ) |
| elements.append( |
| f'<text class="svg-copy axis-label" x="27" y="{top + plot_height / 2:.2f}" text-anchor="middle" transform="rotate(-90 27 {top + plot_height / 2:.2f})">{escape(METRIC_LABELS[y_metric])}</text>' |
| ) |
|
|
| return _svg_shell( |
| title=f"{METRIC_LABELS[y_metric]} vs {METRIC_LABELS[x_metric]}", |
| subtitle="Q4_K_M across 96 multiple-choice items · whiskers show 95% confidence intervals", |
| inner="".join(elements), |
| aria_label=f"Cross-model bubble chart comparing {METRIC_LABELS[y_metric]} with {METRIC_LABELS[x_metric]}", |
| footer_note="Bubble area represents model size on disk", |
| ) |
|
|
|
|
| def _star_points(center_x, center_y, outer_radius, inner_radius): |
| points = [] |
| for index in range(10): |
| angle = -pi / 2 + index * pi / 5 |
| radius = outer_radius if index % 2 == 0 else inner_radius |
| points.append( |
| f"{center_x + cos(angle) * radius:.2f},{center_y + sin(angle) * radius:.2f}" |
| ) |
| return " ".join(points) |
|
|
|
|
| def render_quant_svg(): |
| width = 1000 |
| height = 610 |
| left = 92 |
| right = 58 |
| top = 112 |
| bottom = 92 |
| plot_width = width - left - right |
| plot_height = height - top - bottom |
|
|
| x_low, x_high = 23.5, 41.2 |
| y_low, y_high = 0.12, 0.29 |
| x_position = lambda value: _scale(value, x_low, x_high, left, left + plot_width) |
| y_position = lambda value: _scale(value, y_low, y_high, top + plot_height, top) |
|
|
| elements = [] |
| for tick in _ticks(x_low, x_high): |
| x = x_position(tick) |
| elements.append( |
| f'<line x1="{x:.2f}" y1="{top}" x2="{x:.2f}" y2="{top + plot_height}" stroke="{GRID}" stroke-width="1" opacity="0.75"/>' |
| ) |
| elements.append( |
| f'<text class="svg-copy tick-label" x="{x:.2f}" y="{top + plot_height + 28}" text-anchor="middle">{tick:.0f}</text>' |
| ) |
|
|
| for tick in _ticks(y_low, y_high): |
| y = y_position(tick) |
| elements.append( |
| f'<line x1="{left}" y1="{y:.2f}" x2="{left + plot_width}" y2="{y:.2f}" stroke="{GRID}" stroke-width="1" opacity="0.75"/>' |
| ) |
| elements.append( |
| f'<text class="svg-copy tick-label" x="{left - 16}" y="{y + 4:.2f}" text-anchor="end">{tick * 100:.0f}%</text>' |
| ) |
|
|
| frontier_rows = sorted( |
| (row for row in QUANT_SWEEP if row["frontier"]), |
| key=lambda row: row["decode"], |
| ) |
| if len(frontier_rows) >= 2: |
| frontier_path = " ".join( |
| ( |
| "M" if index == 0 else "L" |
| ) |
| + f" {x_position(row['decode']):.2f} {y_position(row['accuracy']):.2f}" |
| for index, row in enumerate(frontier_rows) |
| ) |
| elements.append( |
| f'<path d="{frontier_path}" fill="none" stroke="{FRONTIER}" stroke-width="4" stroke-linecap="round" opacity="0.5"/>' |
| ) |
|
|
| max_disk = max(row["disk_mb"] for row in QUANT_SWEEP) |
| for row in QUANT_SWEEP: |
| x = x_position(row["decode"]) |
| y = y_position(row["accuracy"]) |
| radius = 12 + 15 * sqrt(row["disk_mb"] / max_disk) |
| color = FRONTIER if row["frontier"] else QUANT_OTHER |
| y_ci_low = y_position(row["ci_low"]) |
| y_ci_high = y_position(row["ci_high"]) |
|
|
| elements.extend( |
| [ |
| f'<line x1="{x:.2f}" y1="{y_ci_high:.2f}" x2="{x:.2f}" y2="{y_ci_low:.2f}" stroke="{color}" stroke-width="3" opacity="0.65"/>', |
| f'<line x1="{x - 8:.2f}" y1="{y_ci_high:.2f}" x2="{x + 8:.2f}" y2="{y_ci_high:.2f}" stroke="{color}" stroke-width="3" opacity="0.65"/>', |
| f'<line x1="{x - 8:.2f}" y1="{y_ci_low:.2f}" x2="{x + 8:.2f}" y2="{y_ci_low:.2f}" stroke="{color}" stroke-width="3" opacity="0.65"/>', |
| ] |
| ) |
|
|
| tooltip = ( |
| f"{row['quant']} — accuracy {row['accuracy'] * 100:.1f}%, " |
| f"decode {row['decode']:.1f} tok/s, prefill {row['prefill']:.1f} tok/s, " |
| f"peak memory {row['pss_mb']:,} MB, disk {row['disk_mb']} MB" |
| ) |
| if row["frontier"]: |
| points = _star_points(x, y, radius + 4, (radius + 4) * 0.48) |
| elements.append( |
| f'<polygon points="{points}" fill="{color}" stroke="{INK}" stroke-width="2" filter="url(#soft-shadow)"><title>{escape(tooltip)} · Pareto frontier</title></polygon>' |
| ) |
| else: |
| elements.append( |
| f'<circle cx="{x:.2f}" cy="{y:.2f}" r="{radius:.2f}" fill="{color}" fill-opacity="0.92" stroke="{INK}" stroke-width="2" filter="url(#soft-shadow)"><title>{escape(tooltip)}</title></circle>' |
| ) |
|
|
| label = f"★ {row['quant']}" if row["frontier"] else row["quant"] |
| elements.append( |
| f'<text class="svg-copy point-label" x="{x + radius + 10:.2f}" y="{y - radius - 4:.2f}">{escape(label)}</text>' |
| ) |
|
|
| elements.append( |
| f'<text class="svg-copy axis-label" x="{left + plot_width / 2:.2f}" y="{height - 44}" text-anchor="middle">Decode speed (tok/s)</text>' |
| ) |
| elements.append( |
| f'<text class="svg-copy axis-label" x="27" y="{top + plot_height / 2:.2f}" text-anchor="middle" transform="rotate(-90 27 {top + plot_height / 2:.2f})">Accuracy</text>' |
| ) |
|
|
| legend = f""" |
| <g class="svg-copy" transform="translate(708 38)"> |
| <polygon points="0,-8 2,-2 8,-2 3,2 5,8 0,4 -5,8 -3,2 -8,-2 -2,-2" fill="{FRONTIER}"/> |
| <text x="14" y="4" fill="{MUTED}" font-size="12">Pareto frontier</text> |
| <circle cx="135" cy="0" r="6" fill="{QUANT_OTHER}"/> |
| <text x="147" y="4" fill="{MUTED}" font-size="12">Other tested quants</text> |
| </g> |
| """ |
|
|
| return _svg_shell( |
| title="Accuracy–speed trade-off by quantization", |
| subtitle="LFM2.5-230M across 256 multiple-choice items", |
| inner="".join(elements), |
| aria_label="Quantization bubble chart comparing accuracy and decode speed for LFM2.5-230M", |
| footer_note="Stars mark the measured Pareto frontier", |
| legend=legend, |
| ) |
|
|
|
|
| def cross_model_table(): |
| df = pd.DataFrame(CROSS_MODEL) |
| df["accuracy"] = (df["accuracy"] * 100).round(1).astype(str) + "%" |
| df["decode"] = df["decode"].round(1).astype(str) + " tok/s" |
| df["pss_mb"] = df["pss_mb"].astype(int).astype(str) + " MB" |
| df["disk_mb"] = df["disk_mb"].astype(int).astype(str) + " MB" |
| df = df[["model", "family", "params_m", "accuracy", "decode", "pss_mb", "disk_mb"]] |
| df.columns = ["Model", "Family", "Params (M)", "Accuracy", "Decode", "Peak PSS", "Disk"] |
| return df |
|
|
|
|
| def quant_sweep_table(): |
| df = pd.DataFrame(QUANT_SWEEP) |
| df["accuracy"] = (df["accuracy"] * 100).round(1).astype(str) + "%" |
| df["decode"] = df["decode"].round(1).astype(str) + " tok/s" |
| df["pss_mb"] = df["pss_mb"].astype(int).astype(str) + " MB" |
| df["disk_mb"] = df["disk_mb"].astype(int).astype(str) + " MB" |
| df["frontier"] = df["frontier"].map({True: "★", False: ""}) |
| df = df[["quant", "accuracy", "decode", "pss_mb", "disk_mb", "frontier"]] |
| df.columns = ["Quant", "Accuracy", "Decode", "Peak PSS", "Disk", "Frontier"] |
| return df |
|
|
|
|
| HERO = """ |
| <section class="hero"> |
| <div class="eyebrow">RASPBERRY PI 5 · LOCAL INFERENCE</div> |
| <h1><span aria-hidden="true">🍓</span> edgebench</h1> |
| <p class="hero-copy">A hardware-grounded look at the speed, accuracy, and memory trade-offs of tiny local language models.</p> |
| <div class="metric-grid"> |
| <article class="metric-card"><span>Fastest decode</span><strong>35.4 <small>tok/s</small></strong><em>LFM2.5-230M</em></article> |
| <article class="metric-card"><span>Highest accuracy</span><strong>28.1<small>%</small></strong><em>LFM2.5-350M</em></article> |
| <article class="metric-card"><span>Lowest peak memory</span><strong>413 <small>MB</small></strong><em>OpenELM-270M</em></article> |
| <article class="metric-card accent-card"><span>Best quant balance</span><strong>Q4_K_M</strong><em>LFM2.5-230M</em></article> |
| </div> |
| </section> |
| """ |
|
|
|
|
| CSS = """ |
| :root { |
| --edge-bg: #071512; |
| --edge-panel: #0d211c; |
| --edge-panel-2: #112a23; |
| --edge-border: #29443a; |
| --edge-ink: #f5fbf8; |
| --edge-muted: #a6c0b5; |
| --edge-accent: #55d6a7; |
| } |
| |
| body, .gradio-container, .dark { background: var(--edge-bg) !important; } |
| |
| .gradio-container { |
| max-width: 1160px !important; |
| margin: 0 auto !important; |
| padding-bottom: 2.5rem !important; |
| color: var(--edge-ink) !important; |
| } |
| |
| .hero { padding: 1.45rem 0 1.25rem; } |
| .eyebrow { color: var(--edge-accent); font-size: .75rem; font-weight: 760; letter-spacing: .13em; margin-bottom: .45rem; } |
| .hero h1 { color: var(--edge-ink); font-size: clamp(2.5rem, 6vw, 4.4rem); line-height: .98; letter-spacing: -.045em; margin: 0; } |
| .hero-copy { color: var(--edge-muted); font-size: 1.05rem; line-height: 1.55; max-width: 720px; margin: .9rem 0 1.2rem; } |
| |
| .metric-grid { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: .75rem; } |
| .metric-card { |
| background: linear-gradient(145deg, var(--edge-panel-2), var(--edge-panel)); |
| border: 1px solid var(--edge-border); |
| border-radius: 14px; |
| padding: .9rem 1rem; |
| min-height: 108px; |
| box-shadow: 0 12px 28px rgba(0,0,0,.14); |
| } |
| .metric-card span, .metric-card em { display: block; color: var(--edge-muted); font-size: .77rem; font-style: normal; } |
| .metric-card strong { display: block; color: var(--edge-ink); font-size: 1.55rem; line-height: 1.2; margin: .38rem 0 .25rem; } |
| .metric-card small { font-size: .72rem; color: var(--edge-muted); } |
| .accent-card { border-color: rgba(85,214,167,.7); box-shadow: inset 0 0 0 1px rgba(85,214,167,.12), 0 12px 28px rgba(0,0,0,.14); } |
| |
| [role="tablist"] { border-bottom-color: var(--edge-border) !important; margin-top: .35rem; } |
| [role="tab"] { font-weight: 650 !important; } |
| [role="tab"][aria-selected="true"] { color: var(--edge-accent) !important; border-color: var(--edge-accent) !important; } |
| |
| .control-panel { border: 1px solid var(--edge-border) !important; background: rgba(13,33,28,.76) !important; border-radius: 14px !important; } |
| .vector-chart { border: 0 !important; background: transparent !important; padding: 0 !important; margin-top: .75rem; } |
| .vector-chart > div { padding: 0 !important; } |
| .vector-figure { width: 100%; margin: 0; line-height: 0; filter: drop-shadow(0 18px 36px rgba(0,0,0,.16)); } |
| .vector-figure svg { width: 100%; height: auto; display: block; } |
| .table-wrap { border-color: var(--edge-border) !important; border-radius: 14px !important; overflow: hidden !important; } |
| .visual-note { color: var(--edge-muted); font-size: .88rem; margin: .2rem 0 .65rem; } |
| |
| @media (max-width: 800px) { .metric-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } } |
| @media (max-width: 520px) { .metric-grid { grid-template-columns: 1fr; } .hero { padding-top: .75rem; } } |
| """ |
|
|
|
|
| with gr.Blocks( |
| theme=gr.themes.Monochrome(primary_hue="emerald", secondary_hue="teal"), |
| css=CSS, |
| ) as demo: |
| gr.HTML(HERO) |
|
|
| with gr.Tab("Model frontier"): |
| gr.HTML( |
| "<p class='visual-note'>Choose two metrics to explore the trade-off. " |
| "Hover over a point for the complete measurement record.</p>" |
| ) |
| with gr.Row(elem_classes="control-panel"): |
| x_axis = gr.Radio( |
| choices=[ |
| ("Decode speed", "decode"), |
| ("Prefill speed", "prefill"), |
| ("Model size", "disk_mb"), |
| ("Peak memory", "pss_mb"), |
| ], |
| value="decode", |
| label="Horizontal axis", |
| interactive=True, |
| ) |
| y_axis = gr.Radio( |
| choices=[ |
| ("Accuracy", "accuracy"), |
| ("Decode speed", "decode"), |
| ("Peak memory", "pss_mb"), |
| ("Model size", "disk_mb"), |
| ], |
| value="accuracy", |
| label="Vertical axis", |
| interactive=True, |
| ) |
|
|
| chart_out = gr.HTML( |
| value=render_cross_svg(), |
| elem_classes="vector-chart", |
| ) |
| table_out = gr.Dataframe( |
| value=cross_model_table(), |
| label="Model comparison", |
| interactive=False, |
| elem_classes="table-wrap", |
| ) |
|
|
| x_axis.change( |
| fn=render_cross_svg, |
| inputs=[x_axis, y_axis], |
| outputs=chart_out, |
| show_progress="hidden", |
| ) |
| y_axis.change( |
| fn=render_cross_svg, |
| inputs=[x_axis, y_axis], |
| outputs=chart_out, |
| show_progress="hidden", |
| ) |
|
|
| with gr.Tab("Quantization frontier"): |
| gr.HTML( |
| "<p class='visual-note'>Stars identify the measured Pareto frontier: " |
| "no other tested quant is both faster and more accurate.</p>" |
| ) |
| gr.HTML(value=render_quant_svg(), elem_classes="vector-chart") |
| gr.Dataframe( |
| value=quant_sweep_table(), |
| label="Quant comparison (★ = Pareto frontier)", |
| interactive=False, |
| elem_classes="table-wrap", |
| ) |
|
|
| with gr.Tab("Methodology"): |
| gr.Markdown( |
| "### Methodology\n\n" |
| "- **Hardware:** Raspberry Pi 5 (Cortex-A76, 8 GB RAM, 4 CPU threads)\n" |
| "- **Engine:** llama.cpp (greedy decode, `--temp 0`, 16-token cap)\n" |
| "- **Workload:** phonebench modern suite — MMLU-Pro (10-choice) + MuSR (narrative reasoning)\n" |
| "- **Cross-model:** 96 items (48 per dataset), all Q4_K_M\n" |
| "- **Quant sweep:** 256 items (160 + 96), LFM2.5-230M across Q4_0 → Q8_0\n" |
| "- **Visual encoding:** bubble area represents model size on disk; whiskers show 95% confidence intervals.\n\n" |
| "### Key findings\n\n" |
| "- **LFM2.5-350M** leads accuracy at 28.1%.\n" |
| "- **LFM2.5-230M** leads decode speed at 35.4 tok/s.\n" |
| "- **OpenELM-270M** uses the least memory at 413 MB peak PSS — 4.5× less than the next model.\n" |
| "- **Qwen3.5-0.8B** reaches 25.0% accuracy but uses 4.2 GB of RAM.\n" |
| "- **Q4_K_M** provides the strongest measured accuracy–speed balance for LFM2.5-230M.\n\n" |
| "### Data\n\n" |
| "All runs were measured with [phonebench](https://github.com/basbe/evals/tree/main/phonebench). " |
| "Linux SBC runs are device-valid but excluded from the public phone leaderboard." |
| ) |
|
|
|
|
| if __name__ == "__main__": |
| demo.launch() |
|
|
