Spaces:
Running
Running
| """Smaug-Flash Explorer. | |
| Interactive companion Space for `abacusai/Smaug-Flash` — an agentic-coding | |
| fine-tune of `deepseek-ai/DeepSeek-V4-Flash-0731` (304 B params, 1 M context). | |
| What it does: | |
| * Overview — specs, checkpoint anatomy, training approach, limitations. | |
| * Benchmarks — digitized model-card numbers as interactive Plotly charts. | |
| * Prompt encoder— the model ships no Jinja chat template; this runs the model's | |
| own `encoding/encoding_dsv4.py` so you can see the exact prompt | |
| string (and parse a completion back). | |
| * Playground — chat against any OpenAI-compatible endpoint you point it at | |
| (your own vLLM / SGLang / llama.cpp deployment, or the HF | |
| router). This Space cannot host the weights: 167 GB across 48 | |
| shards, and nothing on the HF inference router serves it. | |
| * Deploy — copy-paste serving recipes + a weight-memory calculator that | |
| maps the estimate onto real GPU tiers. | |
| No GPU work happens here, so a ZeroGPU no-op keeps the runtime happy without | |
| burning anybody's quota. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import os | |
| from typing import Any, Dict, Iterator, List, Optional, Tuple | |
| import gradio as gr | |
| import plotly.graph_objects as go | |
| # --- ZeroGPU ----------------------------------------------------------------- | |
| # `import spaces` must happen before any CUDA-touching import; this app never | |
| # touches CUDA, but the ZeroGPU runtime requires at least one decorated | |
| # function to exist. It is never called, so no quota is consumed. | |
| try: | |
| import spaces # noqa: F401 | |
| HAS_ZEROGPU = True | |
| def _zerogpu_heartbeat() -> str: | |
| """No-op probe: proves the ZeroGPU runtime is attached. Never invoked.""" | |
| return "zero-gpu-runtime-present" | |
| except Exception: # spaces not installed (e.g. running locally) — fine | |
| HAS_ZEROGPU = False | |
| def _zerogpu_heartbeat() -> str: | |
| """No-op probe (ZeroGPU runtime not attached).""" | |
| return "no-zero-gpu-runtime" | |
| # --- Vendored model-side encoding -------------------------------------------- | |
| # encoding/encoding_dsv4.py from the abacusai/Smaug-Flash repo (MIT), unmodified. | |
| try: | |
| from encoding_dsv4 import ( # type: ignore | |
| REASONING_EFFORT_PROMPTS, | |
| bos_token, | |
| encode_messages, | |
| parse_message_from_completion_text, | |
| ) | |
| HAS_ENCODER = True | |
| except Exception as exc: # pragma: no cover | |
| HAS_ENCODER = False | |
| ENCODER_ERROR = f"{type(exc).__name__}: {exc}" | |
| REASONING_EFFORT_PROMPTS = {"low": "", "high": "", "max": ""} | |
| # --- Static model facts ------------------------------------------------------ | |
| MODEL_ID = "abacusai/Smaug-Flash" | |
| BASE_ID = "deepseek-ai/DeepSeek-V4-Flash-0731" | |
| MODEL_URL = f"https://huggingface.co/{MODEL_ID}" | |
| BASE_URL = f"https://huggingface.co/{BASE_ID}" | |
| ENCODING_URL = f"{MODEL_URL}/blob/main/encoding/encoding_dsv4.py" | |
| FIG_BARS = f"{MODEL_URL}/resolve/main/figures/smaug_flash_vs_base_bars.png" | |
| FIG_LIVEBENCH = f"{MODEL_URL}/resolve/main/figures/smaug_flash_livebench_categories.png" | |
| # from config.json (unchanged from the base release) | |
| N_LAYERS = 43 | |
| N_ROUTED_EXPERTS = 256 | |
| N_SHARED_EXPERTS = 1 | |
| MOE_INTERMEDIATE = 2048 | |
| HIDDEN_SIZE = 4096 | |
| VOCAB_SIZE = 129280 | |
| MAX_POSITIONS = 1_048_576 | |
| # from the Hub safetensors metadata (dtype parameter counts) | |
| TOTAL_PARAMS = 304_180_418_494 # 304.18 B | |
| CHECKPOINT_GB = 166.89 # bytes on the Hub, decimal GB | |
| CHECKPOINT_SHARDS = 48 | |
| EXPERT_PARAMS = N_LAYERS * (N_ROUTED_EXPERTS + N_SHARED_EXPERTS) * 3 * HIDDEN_SIZE * MOE_INTERMEDIATE | |
| OTHER_PARAMS = TOTAL_PARAMS - EXPERT_PARAMS | |
| # Smaug-Flash vs DeepSeek-V4-Flash-0731, digitized from the model card's | |
| # figures/smaug_flash_vs_base_bars.png. Starred base scores are vendor-reported | |
| # numbers for the identical public task set (per the card's footnote). | |
| BENCHMARKS: List[Dict[str, Any]] = [ | |
| {"name": "LiveBench overall-7", "smaug": 77.4, "base": 74.2, "starred": False}, | |
| {"name": "LiveBench agentic coding", "smaug": 61.1, "base": 46.8, "starred": False}, | |
| {"name": "Terminal Bench 2.1 (DeepSeek harness)", "smaug": 76.4, "base": 66.3, "starred": False}, | |
| {"name": "Terminal Bench 2.1 (terminus-2)", "smaug": 70.8, "base": 68.2, "starred": False}, | |
| {"name": "AutomationBench Public", "smaug": 38.8, "base": 25.1, "starred": True}, | |
| {"name": "NL2Repo-Bench", "smaug": 73.3, "base": 54.2, "starred": True}, | |
| {"name": "DeepSWE v1.1", "smaug": 56.6, "base": 54.4, "starred": True}, | |
| {"name": "AA-LCR (long-context, 100 q)", "smaug": 74.0, "base": 75.5, "starred": False}, | |
| ] | |
| # digitized from figures/smaug_flash_livebench_categories.png | |
| LIVEBENCH: List[Dict[str, Any]] = [ | |
| {"name": "Agentic Coding", "smaug": 61.1, "base": 46.8}, | |
| {"name": "Instruction Following", "smaug": 69.3, "base": 65.5}, | |
| {"name": "Mathematics", "smaug": 90.1, "base": 86.8}, | |
| {"name": "Coding", "smaug": 77.1, "base": 75.0}, | |
| {"name": "Language", "smaug": 79.2, "base": 79.2}, | |
| {"name": "Data Analysis", "smaug": 79.1, "base": 79.3}, | |
| {"name": "Reasoning", "smaug": 86.2, "base": 86.6}, | |
| {"name": "Overall", "smaug": 77.4, "base": 74.2}, | |
| ] | |
| HARDWARE: List[Dict[str, Any]] = [ | |
| {"name": "cpu-basic", "gpu": 0.0, "usd_per_hour": 0.0, "note": "2 vCPU / 16 GB — no GPU"}, | |
| {"name": "zero-a10g (ZeroGPU)", "gpu": 96.0, "usd_per_hour": 0.0, "note": "96 GB full MIG, dynamic, this Space"}, | |
| {"name": "t4-small", "gpu": 16.0, "usd_per_hour": 0.40, "note": "1x T4 16 GB"}, | |
| {"name": "a10g-largex2", "gpu": 48.0, "usd_per_hour": 3.00, "note": "2x A10G"}, | |
| {"name": "a10g-largex4", "gpu": 96.0, "usd_per_hour": 5.00, "note": "4x A10G"}, | |
| {"name": "a100-large", "gpu": 80.0, "usd_per_hour": 2.50, "note": "1x A100 80 GB"}, | |
| {"name": "a100x4", "gpu": 320.0, "usd_per_hour": 10.00, "note": "4x A100 80 GB"}, | |
| {"name": "a100x8", "gpu": 640.0, "usd_per_hour": 20.00, "note": "8x A100 80 GB"}, | |
| {"name": "rtx-pro-6000x2", "gpu": 192.0, "usd_per_hour": 5.50, "note": "2x RTX PRO 6000 96 GB"}, | |
| {"name": "rtx-pro-6000x4", "gpu": 384.0, "usd_per_hour": 11.00, "note": "4x RTX PRO 6000 96 GB"}, | |
| {"name": "rtx-pro-6000x8", "gpu": 768.0, "usd_per_hour": 22.00, "note": "8x RTX PRO 6000 96 GB"}, | |
| {"name": "l40sx8", "gpu": 384.0, "usd_per_hour": 23.50, "note": "8x L40S 48 GB"}, | |
| ] | |
| VLLM_CMD = """vllm serve abacusai/Smaug-Flash \\ | |
| --trust-remote-code --kv-cache-dtype fp8 --block-size 256 \\ | |
| --data-parallel-size 4 --enable-expert-parallel \\ | |
| --moe-backend deep_gemm_mega_moe \\ | |
| --attention-config '{"use_fp4_indexer_cache": true}' \\ | |
| --speculative-config '{"method":"dspark","num_speculative_tokens":7,"draft_sample_method":"greedy"}'""" | |
| SGLANG_CMD = """sglang serve \\ | |
| --trust-remote-code \\ | |
| --model-path abacusai/Smaug-Flash \\ | |
| --tp 4 \\ | |
| --enable-dp-attention \\ | |
| --moe-runner-backend flashinfer_mxfp4 \\ | |
| --speculative-algorithm DSPARK \\ | |
| --mem-fraction-static 0.90 \\ | |
| --chunked-prefill-size 4096 \\ | |
| --swa-full-tokens-ratio 0.1""" | |
| ENCODE_SNIPPET = '''from encoding_dsv4 import encode_messages, parse_message_from_completion_text | |
| import transformers | |
| messages = [ | |
| {"role": "system", "content": "You are a helpful assistant."}, | |
| {"role": "user", "content": "Write a Python one-liner that reverses a string."}, | |
| ] | |
| prompt = encode_messages(messages, thinking_mode="thinking", reasoning_effort="max") | |
| tokenizer = transformers.AutoTokenizer.from_pretrained("abacusai/Smaug-Flash") | |
| tokens = tokenizer.encode(prompt)''' | |
| EXAMPLE_CHAT = json.dumps( | |
| [ | |
| {"role": "system", "content": "You are a helpful assistant."}, | |
| {"role": "user", "content": "What is 2+2? Think it through."}, | |
| ], | |
| indent=2, | |
| ) | |
| EXAMPLE_AGENTIC = json.dumps( | |
| [ | |
| {"role": "system", "content": "You are an autonomous coding agent. Use the tools you are given."}, | |
| {"role": "user", "content": "Add a --verbose flag to cmd/cli.py and run the tests."}, | |
| { | |
| "role": "assistant", | |
| "content": "I'll read the file first.", | |
| "reasoning_content": "Need the current CLI definition before editing.", | |
| "tool_calls": [ | |
| { | |
| "id": "call_1", | |
| "type": "function", | |
| "function": {"name": "read_file", "arguments": {"path": "cmd/cli.py"}}, | |
| } | |
| ], | |
| }, | |
| {"role": "tool", "tool_call_id": "call_1", "content": "def main():\n pass\n"}, | |
| {"role": "user", "content": "Also keep the exit code at 0 on success."}, | |
| ], | |
| indent=2, | |
| ) | |
| EXAMPLE_TOOLS = json.dumps( | |
| [ | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "read_file", | |
| "description": "Read a UTF-8 text file from the repo.", | |
| "parameters": { | |
| "type": "object", | |
| "properties": {"path": {"type": "string", "description": "repo-relative path"}}, | |
| "required": ["path"], | |
| }, | |
| }, | |
| } | |
| ], | |
| indent=2, | |
| ) | |
| # --- Optional tokenizer (tokenizers only: no torch, no transformers) --------- | |
| TOKENIZER = None | |
| TOKENIZER_NOTE = "tokenizer.json not available — token counts hidden" | |
| def _load_tokenizer() -> Any: | |
| try: | |
| from tokenizers import Tokenizer # type: ignore | |
| from huggingface_hub import hf_hub_download # type: ignore | |
| path = hf_hub_download(MODEL_ID, "tokenizer.json") | |
| return Tokenizer.from_file(path) | |
| except Exception as exc: # pragma: no cover | |
| globals()["TOKENIZER_NOTE"] = f"tokenizer unavailable ({type(exc).__name__}) — token counts hidden" | |
| return None | |
| TOKENIZER = _load_tokenizer() | |
| def _count_tokens(text: str) -> Optional[int]: | |
| if TOKENIZER is None: | |
| return None | |
| try: | |
| return len(TOKENIZER.encode(text).ids) | |
| except Exception: | |
| return None | |
| def _human_bytes(num: float) -> str: | |
| for unit in ("B", "KB", "MB", "GB", "TB"): | |
| if abs(num) < 1024.0: | |
| return f"{num:,.1f} {unit}" | |
| num /= 1024.0 | |
| return f"{num:,.1f} PB" | |
| # --- Hub facts (best effort; never fatal) ------------------------------------ | |
| HUB_FACTS_NOTE = "live Hub stats unavailable — showing values captured when the Space was written" | |
| def _hub_facts() -> Dict[str, Any]: | |
| try: | |
| from huggingface_hub import HfApi # type: ignore | |
| info = HfApi().model_info(MODEL_ID, files_metadata=True) | |
| total_bytes = sum(f.size or 0 for f in (info.siblings or [])) | |
| return { | |
| "downloads": info.downloads, | |
| "likes": info.likes, | |
| "last_modified": str(info.lastModified)[:10], | |
| "storage": total_bytes, | |
| "live": True, | |
| } | |
| except Exception: | |
| return {"live": False} | |
| HUB_FACTS = _hub_facts() | |
| # ============================================================ | |
| # Overview | |
| # ============================================================ | |
| def overview_markdown() -> str: | |
| """Render the Overview tab: what the model is, its specs, and its limits.""" | |
| if HUB_FACTS.get("live"): | |
| stats = ( | |
| f"**Live Hub stats:** {HUB_FACTS['downloads']:,} downloads · " | |
| f"{HUB_FACTS['likes']:,} likes · last modified {HUB_FACTS['last_modified']} · " | |
| f"{_human_bytes(HUB_FACTS['storage'])} in the repo" | |
| ) | |
| else: | |
| stats = f"*{HUB_FACTS_NOTE}*" | |
| spec_rows = [ | |
| ("Architecture", "Mixture-of-Experts (MoE), `DeepseekV4ForCausalLM`"), | |
| ("Layers", f"{N_LAYERS}"), | |
| ("Attention", "MLA (low-rank q/o factors) + sparse token indexer, 64 heads"), | |
| ("Hidden size", f"{HIDDEN_SIZE}"), | |
| ("Routed experts", f"{N_ROUTED_EXPERTS} routed + {N_SHARED_EXPERTS} shared, 6 per token"), | |
| ("MoE intermediate", f"{MOE_INTERMEDIATE} per expert"), | |
| ("Vocabulary", f"{VOCAB_SIZE:,}"), | |
| ("Context length", f"{MAX_POSITIONS:,} tokens (1 M)"), | |
| ("Quantization", "block-FP8 attention (e4m3, 128x128) / packed-FP4 experts"), | |
| ("Speculative decoding", "DSpark multi-token module (inherited, functional)"), | |
| ("Params / checkpoint", f"**{TOTAL_PARAMS / 1e9:,.2f} B** params · **{CHECKPOINT_GB} GB** · {CHECKPOINT_SHARDS} shards"), | |
| ("Base model", f"[`{BASE_ID}`]({BASE_URL})"), | |
| ("Adaptation", "2x SFT + 1x KTO LoRA, merged as full deltas — attention only"), | |
| ("License", "MIT"), | |
| ] | |
| spec = "\n".join(f"| {k} | {v} |" for k, v in spec_rows) | |
| return f""" | |
| # 🐉 Smaug-Flash Explorer | |
| A working companion for [`{MODEL_ID}`]({MODEL_URL}) — Abacus.AI's agentic-coding fine-tune of | |
| [`{BASE_ID}`]({BASE_URL}). | |
| {stats} | |
| > **Why this Space has no live inference.** The checkpoint is **304 B parameters / 167 GB on disk** | |
| > (packed-FP4 experts + block-FP8 attention), and its Hub `inferenceProviderMapping` is **empty** — | |
| > no serverless provider serves it. HF Spaces hardware tops out at ZeroGPU's 96 GB, so the weights | |
| > cannot be loaded here. Instead: the **Benchmarks** and **Prompt encoder** tabs are real, | |
| > model-specific and run with zero GPU, and the **Playground** talks to any OpenAI-compatible | |
| > endpoint *you* point it at (your own vLLM/SGLang deployment — see **Deploy & size**). | |
| ### Model summary | |
| | Property | Value | | |
| |---|---| | |
| {spec} | |
| ### What changed vs the base model | |
| The card's headline deltas: **+14.3** LiveBench agentic coding, **+10.1** Terminal Bench 2.1 | |
| (DeepSeek harness), **+13.7** AutomationBench, **+19.1** NL2Repo-Bench, **+3.2** LiveBench overall. | |
| Training was 2 SFT steps plus a KTO step, three LoRA adapters trained in sequence and merged as | |
| full (untruncated) deltas into **only the 129 MLA attention factor matrices**. Experts, router, | |
| embeddings and the DSpark speculative module stay byte-identical to the official release, packed | |
| FP4 included — which is why *any stack that serves the base release serves Smaug-Flash unmodified*. | |
| > ⚠️ Card caution: serve in the checkpoint-native format. Re-quantizing the merged weights into a | |
| > different scheme at load time measurably degrades constrained instruction-following. | |
| ### Known behaviors (from the card) | |
| * More decisive than its base: fewer, denser turns, submits sooner. | |
| * Under tight wall-clock budgets that wins timeout-bound tasks the base loses. | |
| * Under hidden-test grading it can submit at ~98% spec coverage without spending the remaining | |
| budget on self-verification — if you reward exhaustive verification, **prompt for explicit | |
| self-testing before completion**. | |
| ### Inference gotchas worth knowing before you serve it | |
| * **No Jinja chat template ships with the repo.** You must encode messages yourself — that's what | |
| the **Prompt encoder** tab does, using the repo's own `encoding_dsv4.py`. | |
| * `reasoning_effort` ∈ {{`low`, `high`, `max`}}; **`None` is treated as `low`**. For `high`/`max` | |
| the card recommends a maximum output length of **384K** tokens. | |
| * Sampling: `temperature=1.0`, `top_p=0.95`. | |
| * Requires `trust_remote_code`. | |
| * Wall-clock task budgets are effectively *inference-compute* budgets on timeout-bounded | |
| benchmarks — benchmark on serving comparable to what you deploy. | |
| Sources: [model card]({MODEL_URL}) · [encoding reference]({ENCODING_URL}) · [base model]({BASE_URL}). | |
| """ | |
| # ============================================================ | |
| # Benchmarks | |
| # ============================================================ | |
| def benchmarks_figure() -> go.Figure: | |
| """Build the grouped Smaug-Flash vs base bar chart from the model card numbers.""" | |
| labels = [ | |
| f"{b['name']} ({b['smaug'] - b['base']:+.1f})" for b in BENCHMARKS | |
| ] | |
| fig = go.Figure() | |
| fig.add_bar( | |
| y=labels, | |
| x=[b["smaug"] for b in BENCHMARKS], | |
| name="Smaug-Flash", | |
| orientation="h", | |
| marker_color="#4f46e5", | |
| text=[f"{b['smaug']:.1f}" for b in BENCHMARKS], | |
| textposition="outside", | |
| hovertemplate="%{y}<br>Smaug-Flash: %{x:.1f}<extra></extra>", | |
| ) | |
| fig.add_bar( | |
| y=labels, | |
| x=[b["base"] for b in BENCHMARKS], | |
| name=f"{BASE_ID} (base)", | |
| orientation="h", | |
| marker_color="#f59e0b", | |
| text=[f"{b['base']:.1f}{'*' if b['starred'] else ''}" for b in BENCHMARKS], | |
| textposition="outside", | |
| hovertemplate="%{y}<br>base: %{x:.1f}<extra></extra>", | |
| ) | |
| fig.update_layout( | |
| barmode="group", | |
| title="Smaug-Flash vs DeepSeek-V4-Flash-0731 (delta in the axis label)", | |
| xaxis_title="score (higher is better)", | |
| legend_orientation="h", | |
| height=520, | |
| margin=dict(l=10, r=10, t=60, b=40), | |
| ) | |
| return fig | |
| def delta_figure() -> go.Figure: | |
| """Build the delta-only bar chart, green for gains and red for losses.""" | |
| deltas = [(b["name"], b["smaug"] - b["base"]) for b in BENCHMARKS] | |
| fig = go.Figure( | |
| go.Bar( | |
| y=[n for n, _ in deltas], | |
| x=[d for _, d in deltas], | |
| orientation="h", | |
| marker_color=["#16a34a" if d >= 0 else "#dc2626" for _, d in deltas], | |
| text=[f"{d:+.1f}" for _, d in deltas], | |
| textposition="outside", | |
| hovertemplate="%{y}: %{x:+.1f}<extra></extra>", | |
| ) | |
| ) | |
| fig.update_layout( | |
| title="Delta vs base: 7 of 8 benchmarks up, AA-LCR down 1.5", | |
| xaxis_title="Smaug-Flash minus base (points)", | |
| height=420, | |
| showlegend=False, | |
| margin=dict(l=10, r=10, t=60, b=40), | |
| ) | |
| return fig | |
| def livebench_figure() -> go.Figure: | |
| """Build the LiveBench category radar from the model card's category figure.""" | |
| cats = [c for c in LIVEBENCH if c["name"] != "Overall"] | |
| theta = [c["name"] for c in cats] + [cats[0]["name"]] | |
| fig = go.Figure() | |
| fig.add_trace( | |
| go.Scatterpolar( | |
| r=[c["smaug"] for c in cats] + [cats[0]["smaug"]], | |
| theta=theta, | |
| name="Smaug-Flash", | |
| fill="toself", | |
| line_color="#4f46e5", | |
| ) | |
| ) | |
| fig.add_trace( | |
| go.Scatterpolar( | |
| r=[c["base"] for c in cats] + [cats[0]["base"]], | |
| theta=theta, | |
| name="DSV4-Flash-0731 (base)", | |
| fill="toself", | |
| line_color="#f59e0b", | |
| opacity=0.7, | |
| ) | |
| ) | |
| fig.update_layout( | |
| polar=dict(radialaxis=dict(visible=True, range=[40, 100], ticks="")), | |
| height=520, | |
| margin=dict(l=40, r=40, t=60, b=40), | |
| title="LiveBench 2026-06-25 categories (0–100)", | |
| legend_orientation="h", | |
| ) | |
| return fig | |
| def benchmarks_markdown() -> str: | |
| """Render the benchmark tables and the caveats attached to them.""" | |
| rows = "\n".join( | |
| "| {n} | **{s:.1f}** | {b:.1f}{star} | {d:+.1f} |".format( | |
| n=b["name"], | |
| s=b["smaug"], | |
| b=b["base"], | |
| star="\\*" if b["starred"] else "", | |
| d=b["smaug"] - b["base"], | |
| ) | |
| for b in BENCHMARKS | |
| ) | |
| lb_rows = "\n".join( | |
| f"| {c['name']} | {c['smaug']:.1f} | {c['base']:.1f} | {c['smaug'] - c['base']:+.1f} |" | |
| for c in LIVEBENCH | |
| ) | |
| return f""" | |
| ### Smaug-Flash vs its base | |
| | Benchmark | Smaug-Flash | DeepSeek-V4-Flash-0731 | Δ | | |
| |---|---:|---:|---:| | |
| {rows} | |
| \\* *base score is the vendor-reported number for the identical public task set (card footnote). | |
| Unstarred rows are paired runs under the same harness.* | |
| ### LiveBench category profile (2026-06-25 release) | |
| | Category | Smaug-Flash | Base | Δ | | |
| |---|---:|---:|---:| | |
| {lb_rows} | |
| **How to read this.** Averages hide where the gain comes from: agentic coding and instruction | |
| following move a lot, while reasoning, data analysis and language are flat-to-slightly-down. That's | |
| the expected shape for an agentic-loop fine-tune of the attention factors only. | |
| **Two caveats, both from the card itself.** (1) These were measured on fresh 2026 task generations, | |
| self-served — on timeout-bounded agentic benchmarks the *serving throughput* is part of the score, | |
| so numbers are hardware-dependent. (2) The chart above is **digitized from the model card's figures** | |
| (`figures/smaug_flash_vs_base_bars.png`, `figures/smaug_flash_livebench_categories.png`); the | |
| originals are reproduced below and are the authoritative version. Where the card's prose and figure | |
| disagree slightly (NL2Repo-Bench "+19" prose vs "+19.1" in the figure), the figure value is used. | |
| ### Original figures from the model card | |
|  | |
|  | |
| """ | |
| # ============================================================ | |
| # Prompt encoder | |
| # ============================================================ | |
| def encode_prompt( | |
| messages_json: str, | |
| thinking_mode: str, | |
| reasoning_effort: str, | |
| drop_thinking: bool, | |
| tools_json: str = "", | |
| ) -> Tuple[str, str, str]: | |
| """Encode an OpenAI-format conversation with the model's own encoding_dsv4. | |
| Args: | |
| messages_json: JSON list of OpenAI-style messages (role, content, optional | |
| reasoning_content and tool_calls, plus a tool message per result). | |
| thinking_mode: "chat" closes the think block immediately; "thinking" lets | |
| the model reason inside <think>...</think>. | |
| reasoning_effort: one of "low", "high", "max" — injected only in thinking | |
| mode, and only on the first message. | |
| drop_thinking: drop reasoning_content from assistant turns before the last | |
| user message (the model's production behaviour). | |
| tools_json: optional JSON list of OpenAI-format tool schemas. | |
| Returns: | |
| Tuple of (encoded prompt, stats markdown, explanation markdown). | |
| """ | |
| if not HAS_ENCODER: | |
| return "", f"⚠️ encoder unavailable: {ENCODER_ERROR}", "" | |
| try: | |
| messages = json.loads(messages_json or "[]") | |
| except json.JSONDecodeError as exc: | |
| return "", f"⚠️ messages JSON is invalid: {exc}", "" | |
| if not isinstance(messages, list) or not messages: | |
| return "", "⚠️ provide a non-empty JSON **list** of messages.", "" | |
| tools = None | |
| if (tools_json or "").strip(): | |
| try: | |
| tools = json.loads(tools_json) | |
| except json.JSONDecodeError as exc: | |
| return "", f"⚠️ tools JSON is invalid: {exc}", "" | |
| if tools and messages: | |
| messages = [dict(m) for m in messages] | |
| messages[0] = {**messages[0], "tools": tools} | |
| try: | |
| prompt = encode_messages( | |
| messages, | |
| thinking_mode=thinking_mode, | |
| reasoning_effort=reasoning_effort, | |
| drop_thinking=drop_thinking, | |
| ) | |
| except Exception as exc: | |
| return "", f"⚠️ encoding failed — `{type(exc).__name__}: {exc}`", "" | |
| n_tokens = _count_tokens(prompt) | |
| tok_line = ( | |
| f"**Tokens:** {n_tokens:,} — {n_tokens / MAX_POSITIONS * 100:.4f}% of the 1,048,576-token context" | |
| if n_tokens is not None | |
| else f"*{TOKENIZER_NOTE}*" | |
| ) | |
| effort_text = REASONING_EFFORT_PROMPTS.get(reasoning_effort, "") if thinking_mode == "thinking" else "" | |
| effort_note = ( | |
| f"`reasoning_effort={reasoning_effort}` injected **{len(effort_text)} chars** into the prompt " | |
| f"({0 if reasoning_effort == 'low' else effort_text.count(chr(10)) - 2} lines of instruction)." | |
| if thinking_mode == "thinking" | |
| else "`reasoning_effort` is ignored in **chat** mode (the think block is closed immediately)." | |
| ) | |
| stats = f""" | |
| **Messages in:** {len(messages)} · **Thinking mode:** `{thinking_mode}` · **drop_thinking:** `{drop_thinking}` | |
| **Characters:** {len(prompt):,} · {tok_line} | |
| {effort_note} | |
| **Tools rendered:** {'yes — ' + str(len(tools)) + ' schema(s)' if tools else 'none'} | |
| """ | |
| explain = f""" | |
| #### What the encoder just did | |
| 1. Merged any `tool` messages into the preceding user turn (DeepSeek-V4 has no separate tool role | |
| slot in the prompt) and sorted tool results by call order — hence the `merge_tool_messages()` / | |
| `sort_tool_results_by_call_order()` step. | |
| 2. {'Dropped reasoning_content from assistant turns before the last user message.' if drop_thinking and thinking_mode == 'thinking' else 'Kept all reasoning_content (drop_thinking off, or chat mode).'} | |
| 3. Prepended `{bos_token}` and closed the final assistant turn with `<｜Assistant｜>`{' plus `<think>`' if thinking_mode == 'thinking' else ' plus `</think>` (chat mode)'}. | |
| The encoded string is exactly what you hand to the tokenizer / the server. Flip `thinking_mode` | |
| to `chat` and watch the `<think>` tokens disappear from the tail. | |
| """ | |
| return prompt, stats, explain | |
| def decode_completion(completion: str, thinking_mode: str) -> Tuple[str, str]: | |
| """Parse a raw model completion back into a structured assistant message. | |
| Args: | |
| completion: Raw model output for one assistant turn, including the | |
| trailing <｜end▁of▁sentence｜> token. | |
| thinking_mode: "chat" or "thinking" — must match how the output was produced. | |
| Returns: | |
| Tuple of (pretty JSON message, status markdown). | |
| """ | |
| if not HAS_ENCODER: | |
| return "", f"⚠️ encoder unavailable: {ENCODER_ERROR}" | |
| if not (completion or "").strip(): | |
| return "", "⚠️ paste a completion first (the raw text, EOS token included)." | |
| try: | |
| parsed = parse_message_from_completion_text(completion, thinking_mode=thinking_mode) | |
| except Exception as exc: | |
| return "", ( | |
| f"⚠️ parse failed — `{type(exc).__name__}: {exc}`\n\n" | |
| "The parser only accepts *well-formed* output (the card is explicit about this): the " | |
| "thinking block must be closed with `</think>` and the turn must end with " | |
| "`<｜end▁of▁sentence｜>`." | |
| ) | |
| return json.dumps(parsed, indent=2, ensure_ascii=False), "✅ parsed cleanly into OpenAI-format fields." | |
| def encoder_intro() -> str: | |
| """Explain why the encoder tab exists and what its controls mean.""" | |
| effort_rows = "\n".join( | |
| f"| `{k}` | {len(v):,} chars | {('injects the effort preamble' if v else 'no preamble — the plain thinking prompt')} |" | |
| for k, v in REASONING_EFFORT_PROMPTS.items() | |
| ) | |
| return f""" | |
| ### No chat template, so here is the real one | |
| `{MODEL_ID}` ships **no Jinja chat template** — `tokenizer.apply_chat_template()` is not an option. | |
| The repo's [`encoding/encoding_dsv4.py`]({ENCODING_URL}) (MIT) is vendored into this Space | |
| **unmodified** and runs below, so you can watch messages become the exact prompt string the model | |
| was trained on: `{bos_token}` … `<｜User｜>` … `<｜Assistant｜><think>`. | |
| | `reasoning_effort` | preamble size | effect | | |
| |---|---:|---| | |
| {effort_rows} | |
| `None` is treated as **`low`**, and `reasoning_effort` only takes effect in `thinking` mode — and | |
| only on the first message. For `high`/`max`, budget up to **384K** output tokens. | |
| """ | |
| # ============================================================ | |
| # Playground | |
| # ============================================================ | |
| PRESETS: Dict[str, Dict[str, str]] = { | |
| "Your own Smaug-Flash deployment (vLLM / SGLang)": { | |
| "base_url": "http://your-host:8000/v1", | |
| "model": MODEL_ID, | |
| "api_key": "EMPTY", | |
| "hint": "vLLM/SGLang serve OpenAI-compatible endpoints at `/v1`. The host must be reachable " | |
| "**from this Space's container** — a laptop localhost is not. Use a tunnel/public hostname, " | |
| "or run this app locally (`pip install -r requirements.txt && python app.py`).", | |
| }, | |
| "HF Inference Providers router (live sibling model)": { | |
| "base_url": "https://router.huggingface.co/v1", | |
| "model": "deepseek-ai/DeepSeek-V4.1-Flash-1017", | |
| "api_key": "", | |
| "hint": "The router is live and OpenAI-compatible, but it does **not** serve Smaug-Flash — " | |
| "point the model field at a supported Flash-class model to sanity-check the plumbing, or use " | |
| "your own deployment above for the real thing. Paste a HF token with `Inference Providers` " | |
| "permission as the key (tokens are used for the request and never stored).", | |
| }, | |
| "Ollama / LM Studio / llama.cpp (local runtime)": { | |
| "base_url": "http://127.0.0.1:11434/v1", | |
| "model": "smaug-flash", | |
| "api_key": "ollama", | |
| "hint": "Only resolves when this app runs on the same machine as the runtime.", | |
| }, | |
| } | |
| def preset_changed(preset: str) -> Tuple[str, str, str, str]: | |
| """Fill the endpoint fields from a preset and return the preset's hint. | |
| Args: | |
| preset: one of the preset labels shown in the dropdown. | |
| Returns: | |
| Tuple of (base_url, model, api_key, hint markdown). | |
| """ | |
| cfg = PRESETS.get(preset) | |
| if not cfg: | |
| return "", MODEL_ID, "", "Custom endpoint: any OpenAI-compatible `/v1` base URL works." | |
| return cfg["base_url"], cfg["model"], cfg["api_key"], cfg["hint"] | |
| def test_endpoint(base_url: str, api_key: str) -> str: | |
| """List the models an endpoint advertises, to prove the connection works. | |
| Args: | |
| base_url: OpenAI-compatible base URL, e.g. http://host:8000/v1. | |
| api_key: API key for that endpoint ("EMPTY" is fine for local vLLM/SGLang). | |
| Returns: | |
| Markdown with the endpoint's model ids, or the error that stopped it. | |
| """ | |
| if not (base_url or "").strip(): | |
| return "⚠️ set a base URL first." | |
| try: | |
| from openai import OpenAI # type: ignore | |
| client = OpenAI(base_url=base_url.strip(), api_key=(api_key or "EMPTY").strip(), timeout=30.0) | |
| ids = [m.id for m in client.models.list().data] | |
| except Exception as exc: | |
| return f"❌ could not reach `{base_url}` — `{type(exc).__name__}: {exc}`" | |
| if not ids: | |
| return f"✅ `{base_url}` answered, but advertised no models." | |
| listed = "\n".join(f"- `{i}`" for i in ids[:40]) | |
| more = f"\n\n…and {len(ids) - 40} more" if len(ids) > 40 else "" | |
| return f"✅ `{base_url}` answered with {len(ids)} model(s):\n\n{listed}{more}" | |
| def _reasoning_delta(chunk_delta: Any) -> str: | |
| """Pull a reasoning/thinking fragment out of a streaming delta, if present.""" | |
| for attr in ("reasoning_content", "reasoning"): | |
| value = getattr(chunk_delta, attr, None) | |
| if value: | |
| return value | |
| extra = getattr(chunk_delta, "model_extra", None) or {} | |
| return extra.get("reasoning_content") or "" | |
| def playground_chat( | |
| message: str, | |
| chat_vis: List[Dict[str, str]], | |
| history_state: List[Dict[str, Any]], | |
| base_url: str, | |
| api_key: str, | |
| model: str, | |
| system_prompt: str, | |
| thinking_mode: str, | |
| reasoning_effort: str, | |
| send_effort: bool, | |
| temperature: float, | |
| top_p: float, | |
| max_tokens: int, | |
| ) -> Iterator[Tuple[List[Dict[str, str]], List[Dict[str, Any]], str, str]]: | |
| """Stream one chat turn from an OpenAI-compatible Smaug-Flash endpoint. | |
| Args: | |
| message: the user's new message. | |
| chat_vis: visible conversation (Gradio messages format). | |
| history_state: full OpenAI-format conversation sent to the endpoint. | |
| base_url: OpenAI-compatible base URL of your deployment. | |
| api_key: key for that endpoint. | |
| model: served model name. | |
| system_prompt: system message (replaced on every request). | |
| thinking_mode: "chat" or "thinking" — sent as `thinking_mode` when the flag is on. | |
| reasoning_effort: low / high / max, forwarded when the endpoint supports it. | |
| send_effort: send reasoning_effort (and thinking_mode) as request parameters. | |
| temperature: sampling temperature — the card recommends 1.0. | |
| top_p: nucleus sampling — the card recommends 0.95. | |
| max_tokens: maximum output tokens (card: up to 384K for high/max effort). | |
| Yields: | |
| (visible chat, conversation state, live reasoning text, cleared input box) | |
| """ | |
| chat_vis = list(chat_vis or []) | |
| history_state = list(history_state or []) | |
| if not (message or "").strip(): | |
| yield chat_vis, history_state, "", "" | |
| return | |
| if not (base_url or "").strip() or not (model or "").strip(): | |
| chat_vis = chat_vis + [ | |
| {"role": "user", "content": message}, | |
| {"role": "assistant", "content": "⚠️ Set a **base URL** and **model** first (or pick a preset)."}, | |
| ] | |
| yield chat_vis, history_state, "", "" | |
| return | |
| chat_vis = chat_vis + [{"role": "user", "content": message}, {"role": "assistant", "content": "…"}] | |
| yield chat_vis, history_state, "", "" | |
| system_prompt = (system_prompt or "").strip() | |
| request_messages: List[Dict[str, Any]] = [] | |
| if system_prompt: | |
| request_messages.append({"role": "system", "content": system_prompt}) | |
| request_messages.extend(history_state) | |
| request_messages.append({"role": "user", "content": message}) | |
| extra_body: Dict[str, Any] = {} | |
| if send_effort: | |
| extra_body["reasoning_effort"] = reasoning_effort | |
| extra_body["thinking_mode"] = thinking_mode | |
| kwargs: Dict[str, Any] = dict( | |
| model=model.strip(), | |
| messages=request_messages, | |
| temperature=float(temperature), | |
| top_p=float(top_p), | |
| max_tokens=int(max_tokens), | |
| stream=True, | |
| ) | |
| try: | |
| from openai import OpenAI # type: ignore | |
| client = OpenAI(base_url=base_url.strip(), api_key=(api_key or "EMPTY").strip(), timeout=600.0) | |
| try: | |
| stream = client.chat.completions.create(**kwargs, extra_body=extra_body) if extra_body else client.chat.completions.create(**kwargs) | |
| except Exception as first_exc: | |
| if not extra_body: | |
| raise | |
| # Endpoint rejected the extra parameters — retry plain, and say so. | |
| note = f"_(endpoint rejected `reasoning_effort`/`thinking_mode`; retried without — {type(first_exc).__name__})_\n\n" | |
| stream = client.chat.completions.create(**kwargs) | |
| chat_vis[-1]["content"] = note | |
| yield chat_vis, history_state, "", "" | |
| answer = "" | |
| reasoning = "" | |
| for chunk in stream: | |
| if not chunk.choices: | |
| continue | |
| delta = chunk.choices[0].delta | |
| piece = _reasoning_delta(delta) | |
| if piece: | |
| reasoning += piece | |
| yield chat_vis, history_state, reasoning, "" | |
| if delta and getattr(delta, "content", None): | |
| answer += delta.content | |
| chat_vis[-1]["content"] = answer | |
| yield chat_vis, history_state, reasoning, "" | |
| if not answer and not reasoning: | |
| chat_vis[-1]["content"] = "⚠️ the endpoint streamed no content (check the model name)." | |
| yield chat_vis, history_state, "", "" | |
| return | |
| history_state.extend( | |
| [ | |
| {"role": "user", "content": message}, | |
| {"role": "assistant", "content": answer}, | |
| ] | |
| ) | |
| yield chat_vis, history_state, reasoning, "" | |
| except Exception as exc: | |
| chat_vis[-1]["content"] = ( | |
| f"❌ request failed — `{type(exc).__name__}: {exc}`\n\n" | |
| "Checklist: is the base URL reachable *from this container*? Is the model name what " | |
| "your server actually serves (`GET /v1/models` via **Test endpoint**)? Does the key " | |
| "have permission?" | |
| ) | |
| yield chat_vis, history_state, "", "" | |
| def reset_chat() -> Tuple[List[Dict[str, str]], List[Dict[str, Any]], str, str]: | |
| """Clear the visible conversation, the endpoint-side history and the reasoning pane. | |
| Returns: | |
| Empty chat, empty state, empty reasoning text, empty message box. | |
| """ | |
| return [], [], "", "" | |
| # ============================================================ | |
| # Deploy & size | |
| # ============================================================ | |
| def estimate_memory(attn_bits: str, expert_bits: str, headroom_pct: float) -> Tuple[str, go.Figure]: | |
| """Estimate weight memory for the checkpoint and map it onto real GPU tiers. | |
| Args: | |
| attn_bits: storage precision for the ~26 B non-expert parameters. | |
| expert_bits: storage precision for the ~278 B routed/shared expert parameters. | |
| headroom_pct: extra percent for KV cache, activations and framework overhead. | |
| Returns: | |
| Tuple of (markdown table of tiers with fit/no-fit, memory breakdown figure). | |
| """ | |
| bpe = {"bf16": 2.0, "fp8 (1 byte)": 1.0, "fp4/int4 packed (0.5 byte)": 0.5, "int8 (1 byte)": 1.0} | |
| a = bpe.get(attn_bits, 1.0) | |
| e = bpe.get(expert_bits, 0.5) | |
| expert_gb = EXPERT_PARAMS * e / 1e9 | |
| other_gb = OTHER_PARAMS * a / 1e9 | |
| weights_gb = expert_gb + other_gb | |
| total_gb = weights_gb * (1 + headroom_pct / 100.0) | |
| breakdown = f""" | |
| | Component | Parameters | Precision | Size | | |
| |---|---:|---|---:| | |
| | Expert FFNs ({N_ROUTED_EXPERTS}+{N_SHARED_EXPERTS} per layer × {N_LAYERS}) | {EXPERT_PARAMS / 1e9:,.1f} B | {expert_bits} | {expert_gb:,.1f} GB | | |
| | Attention + embeddings + DSpark | {OTHER_PARAMS / 1e9:,.1f} B | {attn_bits} | {other_gb:,.1f} GB | | |
| | **Weights subtotal** | {TOTAL_PARAMS / 1e9:,.1f} B | — | **{weights_gb:,.1f} GB** | | |
| | +{headroom_pct:.0f}% headroom (KV cache, activations, overhead) | | | +{total_gb - weights_gb:,.1f} GB | | |
| | **Budget** | | | **{total_gb:,.1f} GB** | | |
| Reference point: the shipped checkpoint is **{CHECKPOINT_GB} GB** on the Hub — block-FP8 attention + | |
| packed-FP4 experts, which is the row `fp8 (1 byte)` × `fp4/int4 packed (0.5 byte)` | |
| ({weights_gb:,.1f} GB here, plus index/tokenizer overhead). | |
| """ | |
| rows = [] | |
| for hw in sorted(HARDWARE, key=lambda h: h["usd_per_hour"]): | |
| if hw["gpu"] <= 0: | |
| fits = "— no GPU" | |
| else: | |
| fits = "✅ fits on paper" if hw["gpu"] >= total_gb else f"❌ short by {total_gb - hw['gpu']:,.0f} GB" | |
| cost = "free" if hw["usd_per_hour"] == 0 else f"${hw['usd_per_hour']:,.2f}/h" | |
| rows.append( | |
| f"| {hw['name']} | {hw['note']} | {hw['gpu']:,.0f} GB | {cost} | {fits} |" | |
| ) | |
| tier_table = f""" | |
| ### Does it fit anywhere on Spaces? | |
| | Flavor | Hardware | GPU VRAM | Price | vs {total_gb:,.1f} GB budget | | |
| |---|---|---:|---:|---| | |
| {chr(10).join(rows)} | |
| **Read this table with the card's deployment notes in mind.** The official recipes are | |
| `tp=4`/`data-parallel-size 4` with expert parallelism and the `flashinfer_mxfp4` / `deep_gemm_mega_moe` | |
| MoE backends — the packed-FP4 experts need kernels that only exist on recent-GPU stacks (SGLang | |
| ≥ 0.5.15 on Blackwell). Also remember the card's warning: serving it in a *different* quantization | |
| than the checkpoint-native one degrades constrained instruction-following, so "just quantize harder" | |
| is not a free win. | |
| Two things the estimate deliberately leaves out: KV-cache size (MLA's compressed ratios in | |
| `config.json` make a naive per-token number misleading — budget the headroom slider instead) and | |
| the fact that a Space's next-cheapest paid tier is billed **per hour while it runs**. | |
| """ | |
| fig = go.Figure( | |
| go.Bar( | |
| x=["weights", f"+{headroom_pct:.0f}% headroom"], | |
| y=[weights_gb, total_gb - weights_gb], | |
| marker_color=["#4f46e5", "#a5b4fc"], | |
| text=[f"{weights_gb:,.0f} GB", f"{total_gb - weights_gb:,.0f} GB"], | |
| textposition="inside", | |
| hovertemplate="%{x}: %{y:,.1f} GB<extra></extra>", | |
| ) | |
| ) | |
| fig.update_layout( | |
| title=f"Memory budget vs the 96 GB ZeroGPU ceiling (dashed)", | |
| yaxis_title="GB (decimal)", | |
| height=380, | |
| showlegend=False, | |
| margin=dict(l=10, r=10, t=60, b=40), | |
| shapes=[ | |
| dict( | |
| type="line", | |
| x0=-0.5, | |
| x1=1.5, | |
| y0=96, | |
| y1=96, | |
| line=dict(color="#dc2626", dash="dash", width=2), | |
| ) | |
| ], | |
| annotations=[dict(x=1.0, y=96, text="ZeroGPU xlarge 96 GB", showarrow=False, yshift=10)], | |
| ) | |
| return breakdown + tier_table, fig | |
| def deploy_markdown() -> str: | |
| """Render the serving recipes, hardware notes and grant pointer.""" | |
| return f""" | |
| ### Serve the real thing | |
| Any stack that runs `{BASE_ID}` runs Smaug-Flash unmodified — only the attention factors changed, | |
| so the FP4 expert kernels and DSpark speculative path are untouched. | |
| **SGLang ≥ 0.5.15 (Blackwell)** — the card's primary recipe: | |
| ```bash | |
| {SGLANG_CMD} | |
| ``` | |
| **vLLM (single 4×GB300 node)**: | |
| ```bash | |
| {VLLM_CMD} | |
| ``` | |
| **Reference transformer path** (the repo's own `inference/` folder) needs a weight conversion first: | |
| ```bash | |
| export EXPERTS=256 MP=4 CONFIG=config.json | |
| python convert.py --hf-ckpt-path ${{HF_CKPT_PATH}} --save-path ${{SAVE_PATH}} \\ | |
| --n-experts ${{EXPERTS}} --model-parallel ${{MP}} | |
| torchrun --nproc-per-node ${{MP}} generate.py --ckpt-path ${{SAVE_PATH}} --config ${{CONFIG}} --interactive | |
| ``` | |
| Serving settings the card is explicit about: `reasoning_effort=`max``, `temperature=1.0`, | |
| `top_p=0.95`, fp8 KV cache, full 1M context, and up to **384K** output tokens for high/max effort. | |
| ### You do not need a paid plan to get real inference here | |
| If you want this Space itself to run the model, the path is a | |
| [community GPU grant](https://huggingface.co/docs/hub/spaces-gpus) — open a discussion on this Space | |
| titled `Apply for a GPU community grant: Personal project` explaining that it is a public open-source | |
| demo of an MIT-licensed 304 B model that can't fit in ZeroGPU's 96 GB. Dedicated multi-GPU tiers for | |
| the `fp8 × fp4` budget above start around **$5.50/h** (2× RTX PRO 6000) and **$10/h** (4× A100). | |
| ### Cheaper alternatives that actually work today | |
| 1. **Run this Space locally** against your own server: | |
| `python app.py` → base URL `http://127.0.0.1:8000/v1` (set `GRADIO_SERVER_PORT` if you like). | |
| 2. **Tunnel a remote deployment** (cloudflared / ngrok / tailscale funnel) and paste the public URL | |
| into the Playground — no hardware change here. | |
| 3. **Quantize harder and accept the card's caveat** — the warning about re-quantizing merged weights | |
| degrading instruction-following applies to agentic/constrained tasks specifically. | |
| """ | |
| # ============================================================ | |
| # UI | |
| # ============================================================ | |
| THEME = gr.themes.Soft(primary_hue="indigo", secondary_hue="pink") | |
| # computed once for the initial render of the sizing tab | |
| _INIT_SIZE_MD, _INIT_SIZE_FIG = estimate_memory("fp8 (1 byte)", "fp4/int4 packed (0.5 byte)", 15) | |
| with gr.Blocks(theme=THEME, title="Smaug-Flash Explorer", fill_height=True) as demo: | |
| gr.Markdown( | |
| f""" | |
| # 🐉 Smaug-Flash Explorer | |
| Interactive companion for [`{MODEL_ID}`]({MODEL_URL}) — Abacus.AI's agentic-coding fine-tune of | |
| [`{BASE_ID}`]({BASE_URL}). **304 B params · 167 GB checkpoint · 1 M context · MIT.** | |
| No GPU needed for anything on this page. | |
| """ | |
| ) | |
| with gr.Tabs(): | |
| # ---------------------------------------------------------------- Overview | |
| with gr.Tab("🧭 Overview"): | |
| gr.Markdown(overview_markdown()) | |
| # -------------------------------------------------------------- Benchmarks | |
| with gr.Tab("📊 Benchmarks"): | |
| gr.Markdown(benchmarks_markdown()) | |
| with gr.Row(): | |
| with gr.Column(): | |
| gr.Plot(benchmarks_figure()) | |
| with gr.Column(): | |
| gr.Plot(livebench_figure()) | |
| gr.Plot(delta_figure()) | |
| # ---------------------------------------------------------- Prompt encoder | |
| with gr.Tab("🔤 Prompt encoder"): | |
| gr.Markdown(encoder_intro()) | |
| if not HAS_ENCODER: | |
| gr.Markdown(f"⚠️ **encoder import failed** — `{ENCODER_ERROR}`") | |
| with gr.Row(): | |
| with gr.Column(scale=5): | |
| messages_in = gr.Code( | |
| value=EXAMPLE_CHAT, | |
| language="json", | |
| label="messages — OpenAI format (reasoning_content / tool_calls allowed)", | |
| lines=18, | |
| ) | |
| with gr.Row(): | |
| thinking_mode_in = gr.Radio( | |
| ["thinking", "chat"], value="thinking", label="thinking_mode" | |
| ) | |
| effort_in = gr.Radio(["low", "high", "max"], value="max", label="reasoning_effort") | |
| drop_in = gr.Checkbox( | |
| value=True, | |
| label="drop_thinking (drop earlier reasoning, production behaviour)", | |
| ) | |
| with gr.Accordion("Tool schemas (optional JSON)", open=False): | |
| tools_in = gr.Code(value="", language="json", label="tools", lines=10) | |
| gr.Markdown( | |
| "Declaring tools switches the encoder's `drop_thinking` off by itself — " | |
| "the model was trained to keep its reasoning across tool loops. The second " | |
| "Examples row below loads a full agentic tool-call loop to try." | |
| ) | |
| encode_btn = gr.Button("Encode prompt", variant="primary") | |
| with gr.Column(scale=6): | |
| prompt_out = gr.Code(label="encoded prompt — this is what you tokenize", lines=22) | |
| stats_out = gr.Markdown() | |
| explain_out = gr.Markdown() | |
| gr.Examples( | |
| examples=[[EXAMPLE_CHAT, "thinking", "max", True, ""], [EXAMPLE_AGENTIC, "thinking", "max", True, EXAMPLE_TOOLS], [EXAMPLE_CHAT, "chat", "low", True, ""]], | |
| inputs=[messages_in, thinking_mode_in, effort_in, drop_in, tools_in], | |
| outputs=[prompt_out, stats_out, explain_out], | |
| fn=encode_prompt, | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| label="Examples: plain chat · agentic tool loop · chat mode", | |
| ) | |
| encode_btn.click( | |
| encode_prompt, | |
| inputs=[messages_in, thinking_mode_in, effort_in, drop_in, tools_in], | |
| outputs=[prompt_out, stats_out, explain_out], | |
| api_name="encode_prompt", | |
| ) | |
| gr.Markdown("---\n### Decode a completion back into OpenAI format") | |
| with gr.Row(): | |
| completion_in = gr.Textbox( | |
| label="raw model completion (include the trailing <｜end▁of▁sentence｜>)", | |
| lines=6, | |
| value="Let me add the two integers.</think>The answer is 4.<｜end▁of▁sentence｜>", | |
| ) | |
| decode_mode = gr.Radio(["thinking", "chat"], value="thinking", label="thinking_mode") | |
| decode_btn = gr.Button("Parse completion") | |
| parsed_out = gr.Code(label="parsed assistant message", language="json", lines=12) | |
| parsed_status = gr.Markdown() | |
| decode_btn.click( | |
| decode_completion, | |
| inputs=[completion_in, decode_mode], | |
| outputs=[parsed_out, parsed_status], | |
| api_name="decode_completion", | |
| ) | |
| gr.Markdown( | |
| f""" | |
| `encode_prompt` and `decode_completion` are also exposed over this Space's **API / MCP server**, so | |
| an agent can use the exact same encoding logic. The module itself is vendored unmodified from the | |
| model repo — see `encoding_dsv4.py` in this Space's Files and | |
| [`{MODEL_ID}/encoding`]({MODEL_URL}/tree/main/encoding). | |
| """ | |
| ) | |
| # --------------------------------------------------------------- Playground | |
| with gr.Tab("💬 Playground"): | |
| gr.Markdown( | |
| f""" | |
| ### Chat with Smaug-Flash — through an endpoint you own | |
| This Space has no GPU big enough for a 167 GB checkpoint, so the chat client is the real one and the | |
| model is yours: serve `{MODEL_ID}` with vLLM/SGLang (see **Deploy & size**) and paste the base URL | |
| below. Anything OpenAI-compatible works — including a model on the HF router for a plumbing check. | |
| """ | |
| ) | |
| with gr.Row(): | |
| with gr.Column(scale=4): | |
| preset_in = gr.Dropdown( | |
| list(PRESETS.keys()) + ["Custom"], | |
| value=list(PRESETS.keys())[0], | |
| label="Endpoint preset", | |
| ) | |
| base_url_in = gr.Textbox(label="Base URL", value=PRESETS[list(PRESETS.keys())[0]]["base_url"]) | |
| model_in = gr.Textbox(label="Model", value=MODEL_ID) | |
| key_in = gr.Textbox(label="API key", type="password", value="EMPTY") | |
| preset_hint = gr.Markdown(PRESETS[list(PRESETS.keys())[0]]["hint"]) | |
| test_btn = gr.Button("Test endpoint") | |
| test_out = gr.Markdown() | |
| with gr.Accordion("Sampling & reasoning", open=True): | |
| system_in = gr.Textbox( | |
| label="System prompt", | |
| value="You are Smaug-Flash, a rigorous agentic coding assistant. Verify your work before you finish.", | |
| lines=3, | |
| ) | |
| mode_in = gr.Radio(["thinking", "chat"], value="thinking", label="thinking_mode") | |
| effort_talk = gr.Radio(["low", "high", "max"], value="max", label="reasoning_effort") | |
| send_effort_in = gr.Checkbox( | |
| value=True, | |
| label="send reasoning_effort / thinking_mode as request parameters", | |
| ) | |
| temp_in = gr.Slider(0.0, 2.0, value=1.0, step=0.05, label="temperature (card: 1.0)") | |
| top_p_in = gr.Slider(0.05, 1.0, value=0.95, step=0.01, label="top_p (card: 0.95)") | |
| max_tok_in = gr.Slider(256, 384000, value=8192, step=256, label="max_tokens (card: up to 384K for high/max)") | |
| with gr.Column(scale=7): | |
| chatbot = gr.Chatbot( | |
| type="messages", | |
| label="conversation", | |
| height=520, | |
| show_label=False, | |
| ) | |
| msg_in = gr.Textbox( | |
| label="message", | |
| placeholder="Ask the model something, or paste a repo task…", | |
| lines=2, | |
| ) | |
| with gr.Row(): | |
| send_btn = gr.Button("Send", variant="primary") | |
| clear_btn = gr.Button("Clear") | |
| noop_btn = gr.Button("ZeroGPU runtime probe", visible=HAS_ZEROGPU) | |
| reasoning_out = gr.Textbox( | |
| label="live reasoning stream (reasoning_content, if the server emits it)", | |
| lines=8, | |
| interactive=False, | |
| ) | |
| state = gr.State([]) | |
| preset_in.change( | |
| preset_changed, | |
| inputs=[preset_in], | |
| outputs=[base_url_in, model_in, key_in, preset_hint], | |
| api_name=False, | |
| ) | |
| test_btn.click(test_endpoint, inputs=[base_url_in, key_in], outputs=[test_out], api_name="test_endpoint") | |
| send_btn.click( | |
| playground_chat, | |
| inputs=[ | |
| msg_in, | |
| chatbot, | |
| state, | |
| base_url_in, | |
| key_in, | |
| model_in, | |
| system_in, | |
| mode_in, | |
| effort_talk, | |
| send_effort_in, | |
| temp_in, | |
| top_p_in, | |
| max_tok_in, | |
| ], | |
| outputs=[chatbot, state, reasoning_out, msg_in], | |
| api_name="playground_chat", | |
| ) | |
| msg_in.submit( | |
| playground_chat, | |
| inputs=[ | |
| msg_in, | |
| chatbot, | |
| state, | |
| base_url_in, | |
| key_in, | |
| model_in, | |
| system_in, | |
| mode_in, | |
| effort_talk, | |
| send_effort_in, | |
| temp_in, | |
| top_p_in, | |
| max_tok_in, | |
| ], | |
| outputs=[chatbot, state, reasoning_out, msg_in], | |
| api_name=False, | |
| ) | |
| clear_btn.click(reset_chat, outputs=[chatbot, state, reasoning_out, msg_in], api_name=False) | |
| if HAS_ZEROGPU: | |
| noop_btn.click(_zerogpu_heartbeat, outputs=[reasoning_out], api_name="runtime_probe") | |
| # ------------------------------------------------------------- Deploy/size | |
| with gr.Tab("🚀 Deploy & size"): | |
| with gr.Row(): | |
| with gr.Column(scale=4): | |
| attn_bits_in = gr.Radio( | |
| ["fp8 (1 byte)", "bf16", "fp4/int4 packed (0.5 byte)"], | |
| value="fp8 (1 byte)", | |
| label="non-expert parameters (attention, embeddings, DSpark)", | |
| ) | |
| expert_bits_in = gr.Radio( | |
| ["fp4/int4 packed (0.5 byte)", "fp8 (1 byte)", "bf16"], | |
| value="fp4/int4 packed (0.5 byte)", | |
| label="expert parameters", | |
| ) | |
| headroom_in = gr.Slider(0, 60, value=15, step=5, label="headroom % (KV cache, activations, overhead)") | |
| size_btn = gr.Button("Estimate", variant="primary") | |
| with gr.Column(scale=7): | |
| size_out = gr.Markdown(_INIT_SIZE_MD) | |
| size_fig = gr.Plot(_INIT_SIZE_FIG) | |
| size_btn.click( | |
| estimate_memory, | |
| inputs=[attn_bits_in, expert_bits_in, headroom_in], | |
| outputs=[size_out, size_fig], | |
| api_name="estimate_memory", | |
| ) | |
| gr.Markdown(deploy_markdown()) | |
| gr.Markdown( | |
| f""" | |
| --- | |
| Built from the [`{MODEL_ID}`]({MODEL_URL}) model card and repository contents | |
| (`config.json`, `encoding/encoding_dsv4.py`, `figures/`), which is MIT licensed; the encoder is | |
| vendored unmodified. Benchmark values are digitized from the card's figures — the originals above | |
| remain authoritative. Not affiliated with Abacus.AI or DeepSeek-AI. | |
| """ | |
| ) | |
| if __name__ == "__main__": | |
| demo.queue(default_concurrency_limit=8).launch( | |
| mcp_server=True, | |
| server_name=os.getenv("GRADIO_SERVER_NAME", "0.0.0.0"), | |
| server_port=int(os.getenv("GRADIO_SERVER_PORT", "7860")), | |
| ) | |