#!/usr/bin/env python3 """ dispatchAI Small Model Olympics — Community Leaderboard A Gradio Space where tiny models compete on quality, speed, size, and efficiency. """ import gradio as gr import pandas as pd import json import os # Brand colors INK = "#0A0F1A" OFF_WHITE = "#F5F7FA" ELECTRIC_BLUE = "#2E6BFF" CYAN = "#1FE0E6" # Initial leaderboard data (dispatchAI models as the starting lineup) LEADERBOARD_DATA = [ # Sprint category (< 500M) {"model": "dispatchAI/SmolLM2-135M-Instruct-mobile", "params_m": 135, "category": "Sprint", "size_mb": 270, "quality": 0.52, "cpu_tps": 45.2, "phone_tps": 18.3}, {"model": "dispatchAI/SmolLM2-360M-Instruct-mobile", "params_m": 360, "category": "Sprint", "size_mb": 720, "quality": 0.61, "cpu_tps": 28.5, "phone_tps": 12.1}, {"model": "dispatchAI/EmbeddingGemma-300M-mobile", "params_m": 300, "category": "Sprint", "size_mb": 600, "quality": 0.58, "cpu_tps": 32.0, "phone_tps": 14.5}, # Middle category (500M - 1B) {"model": "dispatchAI/Qwen2.5-0.5B-Instruct-mobile-int4", "params_m": 500, "category": "Middle", "size_mb": 350, "quality": 0.65, "cpu_tps": 22.3, "phone_tps": 9.8}, {"model": "dispatchAI/Qwen2.5-0.5B-Coder-mobile", "params_m": 500, "category": "Middle", "size_mb": 350, "quality": 0.63, "cpu_tps": 22.0, "phone_tps": 9.5}, {"model": "dispatchAI/Qwen2.5-0.5B-Chinese-mobile", "params_m": 500, "category": "Middle", "size_mb": 350, "quality": 0.60, "cpu_tps": 21.5, "phone_tps": 9.2}, {"model": "dispatchAI/Llama-3.2-1B-Instruct-mobile", "params_m": 1000, "category": "Middle", "size_mb": 700, "quality": 0.68, "cpu_tps": 15.2, "phone_tps": 6.5}, {"model": "dispatchAI/TinyLlama-1.1B-Chat-mobile-int4", "params_m": 1100, "category": "Middle", "size_mb": 450, "quality": 0.59, "cpu_tps": 14.8, "phone_tps": 6.2}, {"model": "dispatchAI/MiniCPM5-1B-mobile", "params_m": 1000, "category": "Middle", "size_mb": 700, "quality": 0.64, "cpu_tps": 15.0, "phone_tps": 6.3}, # Distance category (1B - 2B) {"model": "dispatchAI/Qwen2.5-1.5B-Instruct-mobile-int4", "params_m": 1500, "category": "Distance", "size_mb": 900, "quality": 0.72, "cpu_tps": 10.5, "phone_tps": 4.2}, {"model": "dispatchAI/SmolLM2-1.7B-Instruct-mobile", "params_m": 1700, "category": "Distance", "size_mb": 1100, "quality": 0.70, "cpu_tps": 9.2, "phone_tps": 3.5}, {"model": "dispatchAI/Gemma-2-2B-IT-mobile", "params_m": 2000, "category": "Distance", "size_mb": 1300, "quality": 0.74, "cpu_tps": 8.0, "phone_tps": 3.0}, {"model": "dispatchAI/Phi-3.5-mini-Instruct-mobile", "params_m": 2000, "category": "Distance", "size_mb": 1300, "quality": 0.73, "cpu_tps": 8.2, "phone_tps": 3.1}, # Relay category (Merged) {"model": "dispatchAI/Qwen2.5-0.5B-CodeInstruct-mobile", "params_m": 500, "category": "Relay", "size_mb": 900, "quality": 0.66, "cpu_tps": 22.0, "phone_tps": 9.5}, ] df = pd.DataFrame(LEADERBOARD_DATA) # Calculate efficiency score: quality per 100MB df["efficiency"] = (df["quality"] / (df["size_mb"] / 100)).round(3) def get_leaderboard(category="All", sort_by="efficiency"): """Filter and sort the leaderboard.""" data = df.copy() if category != "All": data = data[data["category"] == category] data = data.sort_values(sort_by, ascending=sort_by == "size_mb") return data[["model", "params_m", "category", "size_mb", "quality", "cpu_tps", "phone_tps", "efficiency"]] def submit_model(model_id, params_m, category, size_mb, quality, cpu_tps, phone_tps): """Submit a new model to the leaderboard (adds to in-memory list).""" global df new_row = { "model": model_id, "params_m": int(params_m) if params_m else 0, "category": category, "size_mb": int(size_mb) if size_mb else 0, "quality": float(quality) if quality else 0, "cpu_tps": float(cpu_tps) if cpu_tps else 0, "phone_tps": float(phone_tps) if phone_tps else 0, } new_row["efficiency"] = round(new_row["quality"] / (new_row["size_mb"] / 100), 3) if new_row["size_mb"] > 0 else 0 df = pd.concat([df, pd.DataFrame([new_row])], ignore_index=True) return get_leaderboard(category="All"), f"✅ Submitted {model_id}! It's now on the leaderboard." # Custom CSS for dispatchAI branding custom_css = """ .gradio-container { background: #0A0F1A !important; color: #F5F7FA !important; } .main-panel { background: #0A0F1A !important; } .gr-dataframe table { background: #111827 !important; color: #F5F7FA !important; } .gr-dataframe th { background: #1a2332 !important; color: #1FE0E6 !important; } .gr-button { background: linear-gradient(135deg, #2E6BFF, #1FE0E6) !important; color: #0A0F1A !important; font-weight: bold !important; } h1, h2, h3 { color: #1FE0E6 !important; } """ with gr.Blocks(css=custom_css, title="Small Model Olympics") as demo: gr.Markdown(""" # 🏆 Small Model Olympics **The community leaderboard for tiny AI models (sub-2B parameters).** Models compete on quality, speed, size, and efficiency. Smaller is mightier. Hosted by [dispatchAI](https://huggingface.co/dispatchAI) | [Submit your model](https://huggingface.co/spaces/dispatchAI/small-model-olympics/discussions) """) with gr.Tabs(): with gr.TabItem("Leaderboard"): with gr.Row(): category_filter = gr.Dropdown( choices=["All", "Sprint", "Middle", "Distance", "Relay"], value="All", label="Category" ) sort_by = gr.Dropdown( choices=["efficiency", "quality", "cpu_tps", "phone_tps", "size_mb"], value="efficiency", label="Sort by" ) refresh_btn = gr.Button("Refresh", variant="primary") leaderboard_table = gr.Dataframe( value=get_leaderboard(), headers=["model", "params_m", "category", "size_mb", "quality", "cpu_tps", "phone_tps", "efficiency"], label="Leaderboard", interactive=False, wrap=True, ) refresh_btn.click( fn=get_leaderboard, inputs=[category_filter, sort_by], outputs=leaderboard_table ) category_filter.change( fn=get_leaderboard, inputs=[category_filter, sort_by], outputs=leaderboard_table ) sort_by.change( fn=get_leaderboard, inputs=[category_filter, sort_by], outputs=leaderboard_table ) with gr.TabItem("Submit Model"): gr.Markdown(""" ### Submit your model Enter your model details below. All fields are required. **Categories:** - **Sprint** — Under 500M parameters - **Middle** — 500M to 1B - **Distance** — 1B to 2B - **Relay** — Merged/composed models """) with gr.Row(): model_id = gr.Textbox(label="HuggingFace Model ID", placeholder="org/model-name") params_m = gr.Textbox(label="Parameters (M)", placeholder="500") with gr.Row(): category = gr.Dropdown(choices=["Sprint", "Middle", "Distance", "Relay"], value="Middle", label="Category") size_mb = gr.Textbox(label="Model Size (MB)", placeholder="350") with gr.Row(): quality = gr.Textbox(label="Quality Score (0-1)", placeholder="0.65") cpu_tps = gr.Textbox(label="CPU Tokens/sec", placeholder="22.0") with gr.Row(): phone_tps = gr.Textbox(label="Phone Tokens/sec", placeholder="9.5") submit_btn = gr.Button("Submit", variant="primary") submit_status = gr.Textbox(label="Status", interactive=False) submit_btn.click( fn=submit_model, inputs=[model_id, params_m, category, size_mb, quality, cpu_tps, phone_tps], outputs=[leaderboard_table, submit_status] ) with gr.TabItem("About"): gr.Markdown(""" ## About the Small Model Olympics ### Scoring | Metric | Description | |--------|-------------| | **Quality** | Unique word ratio on standard prompts (0-1 scale) | | **CPU Speed** | Tokens/sec on CPU (112-core workstation) | | **Phone Speed** | Tokens/sec on Samsung S20 FE (Snapdragon 865) | | **Size** | Model file size in MB (smaller = better) | | **Efficiency** | Quality per 100MB — the ultimate mobile metric | ### Categories - **Sprint** (< 500M) — The tiniest of the tiny - **Middle** (500M-1B) — The mobile sweet spot - **Distance** (1B-2B) — Pushing the mobile limit - **Relay** (Merged) — Compositions of multiple models ### How to evaluate your model ```python from transformers import AutoModelForCausalLM, AutoTokenizer import torch model = AutoModelForCausalLM.from_pretrained("your-model", torch_dtype=torch.float16) tok = AutoTokenizer.from_pretrained("your-model") prompt = "Explain what AI is in simple terms:" inputs = tok(prompt, return_tensors="pt") import time t0 = time.time() with torch.no_grad(): out = model.generate(**inputs, max_new_tokens=50, do_sample=True, temperature=0.7) elapsed = time.time() - t0 text = tok.decode(out[0], skip_special_tokens=True) words = text.split() unique_ratio = len(set(words)) / len(words) if words else 0 tps = (out.shape[1] - inputs["input_ids"].shape[1]) / elapsed print(f"Quality: {unique_ratio:.2f}, Speed: {tps:.1f} t/s") ``` ### Citation ```bibtex @misc{dispatchai_olympics_2026, title={Small Model Olympics: Community Leaderboard for Sub-2B Models}, author={Aljallaf Alzaabi, Omar Abdulla Jasem}, year={2026}, url={https://huggingface.co/spaces/dispatchAI/small-model-olympics} } ``` --- *Dispatch AI (FZE), Sharjah SRTI Free Zone, License No. 10818.* """) if __name__ == "__main__": demo.launch()