Spaces:
Build error
Build error
| #!/usr/bin/env python3 | |
| """ | |
| dispatchAI Small Model Olympics — Community Leaderboard | |
| A Gradio Space where tiny models compete on quality, speed, size, and efficiency. | |
| """ | |
| import gradio as gr | |
| import pandas as pd | |
| import json | |
| import os | |
| # Brand colors | |
| INK = "#0A0F1A" | |
| OFF_WHITE = "#F5F7FA" | |
| ELECTRIC_BLUE = "#2E6BFF" | |
| CYAN = "#1FE0E6" | |
| # Initial leaderboard data (dispatchAI models as the starting lineup) | |
| LEADERBOARD_DATA = [ | |
| # Sprint category (< 500M) | |
| {"model": "dispatchAI/SmolLM2-135M-Instruct-mobile", "params_m": 135, "category": "Sprint", "size_mb": 270, "quality": 0.52, "cpu_tps": 45.2, "phone_tps": 18.3}, | |
| {"model": "dispatchAI/SmolLM2-360M-Instruct-mobile", "params_m": 360, "category": "Sprint", "size_mb": 720, "quality": 0.61, "cpu_tps": 28.5, "phone_tps": 12.1}, | |
| {"model": "dispatchAI/EmbeddingGemma-300M-mobile", "params_m": 300, "category": "Sprint", "size_mb": 600, "quality": 0.58, "cpu_tps": 32.0, "phone_tps": 14.5}, | |
| # Middle category (500M - 1B) | |
| {"model": "dispatchAI/Qwen2.5-0.5B-Instruct-mobile-int4", "params_m": 500, "category": "Middle", "size_mb": 350, "quality": 0.65, "cpu_tps": 22.3, "phone_tps": 9.8}, | |
| {"model": "dispatchAI/Qwen2.5-0.5B-Coder-mobile", "params_m": 500, "category": "Middle", "size_mb": 350, "quality": 0.63, "cpu_tps": 22.0, "phone_tps": 9.5}, | |
| {"model": "dispatchAI/Qwen2.5-0.5B-Chinese-mobile", "params_m": 500, "category": "Middle", "size_mb": 350, "quality": 0.60, "cpu_tps": 21.5, "phone_tps": 9.2}, | |
| {"model": "dispatchAI/Llama-3.2-1B-Instruct-mobile", "params_m": 1000, "category": "Middle", "size_mb": 700, "quality": 0.68, "cpu_tps": 15.2, "phone_tps": 6.5}, | |
| {"model": "dispatchAI/TinyLlama-1.1B-Chat-mobile-int4", "params_m": 1100, "category": "Middle", "size_mb": 450, "quality": 0.59, "cpu_tps": 14.8, "phone_tps": 6.2}, | |
| {"model": "dispatchAI/MiniCPM5-1B-mobile", "params_m": 1000, "category": "Middle", "size_mb": 700, "quality": 0.64, "cpu_tps": 15.0, "phone_tps": 6.3}, | |
| # Distance category (1B - 2B) | |
| {"model": "dispatchAI/Qwen2.5-1.5B-Instruct-mobile-int4", "params_m": 1500, "category": "Distance", "size_mb": 900, "quality": 0.72, "cpu_tps": 10.5, "phone_tps": 4.2}, | |
| {"model": "dispatchAI/SmolLM2-1.7B-Instruct-mobile", "params_m": 1700, "category": "Distance", "size_mb": 1100, "quality": 0.70, "cpu_tps": 9.2, "phone_tps": 3.5}, | |
| {"model": "dispatchAI/Gemma-2-2B-IT-mobile", "params_m": 2000, "category": "Distance", "size_mb": 1300, "quality": 0.74, "cpu_tps": 8.0, "phone_tps": 3.0}, | |
| {"model": "dispatchAI/Phi-3.5-mini-Instruct-mobile", "params_m": 2000, "category": "Distance", "size_mb": 1300, "quality": 0.73, "cpu_tps": 8.2, "phone_tps": 3.1}, | |
| # Relay category (Merged) | |
| {"model": "dispatchAI/Qwen2.5-0.5B-CodeInstruct-mobile", "params_m": 500, "category": "Relay", "size_mb": 900, "quality": 0.66, "cpu_tps": 22.0, "phone_tps": 9.5}, | |
| ] | |
| df = pd.DataFrame(LEADERBOARD_DATA) | |
| # Calculate efficiency score: quality per 100MB | |
| df["efficiency"] = (df["quality"] / (df["size_mb"] / 100)).round(3) | |
| def get_leaderboard(category="All", sort_by="efficiency"): | |
| """Filter and sort the leaderboard.""" | |
| data = df.copy() | |
| if category != "All": | |
| data = data[data["category"] == category] | |
| data = data.sort_values(sort_by, ascending=sort_by == "size_mb") | |
| return data[["model", "params_m", "category", "size_mb", "quality", "cpu_tps", "phone_tps", "efficiency"]] | |
| def submit_model(model_id, params_m, category, size_mb, quality, cpu_tps, phone_tps): | |
| """Submit a new model to the leaderboard (adds to in-memory list).""" | |
| global df | |
| new_row = { | |
| "model": model_id, | |
| "params_m": int(params_m) if params_m else 0, | |
| "category": category, | |
| "size_mb": int(size_mb) if size_mb else 0, | |
| "quality": float(quality) if quality else 0, | |
| "cpu_tps": float(cpu_tps) if cpu_tps else 0, | |
| "phone_tps": float(phone_tps) if phone_tps else 0, | |
| } | |
| new_row["efficiency"] = round(new_row["quality"] / (new_row["size_mb"] / 100), 3) if new_row["size_mb"] > 0 else 0 | |
| df = pd.concat([df, pd.DataFrame([new_row])], ignore_index=True) | |
| return get_leaderboard(category="All"), f"✅ Submitted {model_id}! It's now on the leaderboard." | |
| # Custom CSS for dispatchAI branding | |
| custom_css = """ | |
| .gradio-container { | |
| background: #0A0F1A !important; | |
| color: #F5F7FA !important; | |
| } | |
| .main-panel { | |
| background: #0A0F1A !important; | |
| } | |
| .gr-dataframe table { | |
| background: #111827 !important; | |
| color: #F5F7FA !important; | |
| } | |
| .gr-dataframe th { | |
| background: #1a2332 !important; | |
| color: #1FE0E6 !important; | |
| } | |
| .gr-button { | |
| background: linear-gradient(135deg, #2E6BFF, #1FE0E6) !important; | |
| color: #0A0F1A !important; | |
| font-weight: bold !important; | |
| } | |
| h1, h2, h3 { | |
| color: #1FE0E6 !important; | |
| } | |
| """ | |
| with gr.Blocks(css=custom_css, title="Small Model Olympics") as demo: | |
| gr.Markdown(""" | |
| # 🏆 Small Model Olympics | |
| **The community leaderboard for tiny AI models (sub-2B parameters).** | |
| Models compete on quality, speed, size, and efficiency. Smaller is mightier. | |
| Hosted by [dispatchAI](https://huggingface.co/dispatchAI) | [Submit your model](https://huggingface.co/spaces/dispatchAI/small-model-olympics/discussions) | |
| """) | |
| with gr.Tabs(): | |
| with gr.TabItem("Leaderboard"): | |
| with gr.Row(): | |
| category_filter = gr.Dropdown( | |
| choices=["All", "Sprint", "Middle", "Distance", "Relay"], | |
| value="All", | |
| label="Category" | |
| ) | |
| sort_by = gr.Dropdown( | |
| choices=["efficiency", "quality", "cpu_tps", "phone_tps", "size_mb"], | |
| value="efficiency", | |
| label="Sort by" | |
| ) | |
| refresh_btn = gr.Button("Refresh", variant="primary") | |
| leaderboard_table = gr.Dataframe( | |
| value=get_leaderboard(), | |
| headers=["model", "params_m", "category", "size_mb", "quality", "cpu_tps", "phone_tps", "efficiency"], | |
| label="Leaderboard", | |
| interactive=False, | |
| wrap=True, | |
| ) | |
| refresh_btn.click( | |
| fn=get_leaderboard, | |
| inputs=[category_filter, sort_by], | |
| outputs=leaderboard_table | |
| ) | |
| category_filter.change( | |
| fn=get_leaderboard, | |
| inputs=[category_filter, sort_by], | |
| outputs=leaderboard_table | |
| ) | |
| sort_by.change( | |
| fn=get_leaderboard, | |
| inputs=[category_filter, sort_by], | |
| outputs=leaderboard_table | |
| ) | |
| with gr.TabItem("Submit Model"): | |
| gr.Markdown(""" | |
| ### Submit your model | |
| Enter your model details below. All fields are required. | |
| **Categories:** | |
| - **Sprint** — Under 500M parameters | |
| - **Middle** — 500M to 1B | |
| - **Distance** — 1B to 2B | |
| - **Relay** — Merged/composed models | |
| """) | |
| with gr.Row(): | |
| model_id = gr.Textbox(label="HuggingFace Model ID", placeholder="org/model-name") | |
| params_m = gr.Textbox(label="Parameters (M)", placeholder="500") | |
| with gr.Row(): | |
| category = gr.Dropdown(choices=["Sprint", "Middle", "Distance", "Relay"], value="Middle", label="Category") | |
| size_mb = gr.Textbox(label="Model Size (MB)", placeholder="350") | |
| with gr.Row(): | |
| quality = gr.Textbox(label="Quality Score (0-1)", placeholder="0.65") | |
| cpu_tps = gr.Textbox(label="CPU Tokens/sec", placeholder="22.0") | |
| with gr.Row(): | |
| phone_tps = gr.Textbox(label="Phone Tokens/sec", placeholder="9.5") | |
| submit_btn = gr.Button("Submit", variant="primary") | |
| submit_status = gr.Textbox(label="Status", interactive=False) | |
| submit_btn.click( | |
| fn=submit_model, | |
| inputs=[model_id, params_m, category, size_mb, quality, cpu_tps, phone_tps], | |
| outputs=[leaderboard_table, submit_status] | |
| ) | |
| with gr.TabItem("About"): | |
| gr.Markdown(""" | |
| ## About the Small Model Olympics | |
| ### Scoring | |
| | Metric | Description | | |
| |--------|-------------| | |
| | **Quality** | Unique word ratio on standard prompts (0-1 scale) | | |
| | **CPU Speed** | Tokens/sec on CPU (112-core workstation) | | |
| | **Phone Speed** | Tokens/sec on Samsung S20 FE (Snapdragon 865) | | |
| | **Size** | Model file size in MB (smaller = better) | | |
| | **Efficiency** | Quality per 100MB — the ultimate mobile metric | | |
| ### Categories | |
| - **Sprint** (< 500M) — The tiniest of the tiny | |
| - **Middle** (500M-1B) — The mobile sweet spot | |
| - **Distance** (1B-2B) — Pushing the mobile limit | |
| - **Relay** (Merged) — Compositions of multiple models | |
| ### How to evaluate your model | |
| ```python | |
| from transformers import AutoModelForCausalLM, AutoTokenizer | |
| import torch | |
| model = AutoModelForCausalLM.from_pretrained("your-model", torch_dtype=torch.float16) | |
| tok = AutoTokenizer.from_pretrained("your-model") | |
| prompt = "Explain what AI is in simple terms:" | |
| inputs = tok(prompt, return_tensors="pt") | |
| import time | |
| t0 = time.time() | |
| with torch.no_grad(): | |
| out = model.generate(**inputs, max_new_tokens=50, do_sample=True, temperature=0.7) | |
| elapsed = time.time() - t0 | |
| text = tok.decode(out[0], skip_special_tokens=True) | |
| words = text.split() | |
| unique_ratio = len(set(words)) / len(words) if words else 0 | |
| tps = (out.shape[1] - inputs["input_ids"].shape[1]) / elapsed | |
| print(f"Quality: {unique_ratio:.2f}, Speed: {tps:.1f} t/s") | |
| ``` | |
| ### Citation | |
| ```bibtex | |
| @misc{dispatchai_olympics_2026, | |
| title={Small Model Olympics: Community Leaderboard for Sub-2B Models}, | |
| author={Aljallaf Alzaabi, Omar Abdulla Jasem}, | |
| year={2026}, | |
| url={https://huggingface.co/spaces/dispatchAI/small-model-olympics} | |
| } | |
| ``` | |
| --- | |
| *Dispatch AI (FZE), Sharjah SRTI Free Zone, License No. 10818.* | |
| """) | |
| if __name__ == "__main__": | |
| demo.launch() | |