Gemma 4 12B brain on ZeroGPU (transformers)

README.md

@@ -1,13 +1,25 @@
 ---
-title: Small Talk Llm
-emoji: 🐠
-colorFrom: blue
+title: Small Talk LLM Brain
+emoji: 🧠
+colorFrom: yellow
 colorTo: indigo
 sdk: gradio
-sdk_version: 6.17.3
-python_version: '3.13'
 app_file: app.py
 pinned: false
+suggested_hardware: zero-a10g
+short_description: Gemma 4 12B brain for the Small Talk robot podcast
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Small Talk · LLM Brain
+
+The live-banter brain for [**Small Talk**](https://huggingface.co/spaces/build-small-hackathon/small-talk),
+an AI-to-AI robot podcast hosted by Reachy Mini robots.
+
+- **Model:** [`google/gemma-4-12b-it`](https://huggingface.co/google/gemma-4-12b-it)
+  — the QAT-trained Gemma 4 12B, run in bf16.
+- **Hardware:** ZeroGPU.
+- **Use:** pass a character description as the system prompt to voice a persona;
+  the podcast backend calls this Space as an API to generate in-character lines.
+
+The int4-QAT GGUF + `llama.cpp` (`llama-server`) deployment runs separately on
+Modal — this Space is the always-on ZeroGPU brain.

app.py

@@ -0,0 +1,95 @@
+"""Small Talk · LLM brain — Gemma 4 12B on ZeroGPU via 🤗 transformers.
+
+A separate Space from the main Small Talk app: it exposes a chat endpoint the
+podcast backend can call to generate live, in-character robot banter. We run the
+QAT-trained Gemma 4 12B in bf16 here (the canonical, rock-solid ZeroGPU path);
+the int4-QAT GGUF + llama.cpp deployment lives on Modal later.
+
+`import spaces` MUST come before torch so ZeroGPU can patch CUDA.
+"""
+import os
+from threading import Thread
+
+import spaces
+import gradio as gr
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+
+MODEL_ID = os.environ.get("MODEL_ID", "google/gemma-4-12b-it")
+
+tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
+model = AutoModelForCausalLM.from_pretrained(
+    MODEL_ID,
+    torch_dtype=torch.bfloat16,
+    device_map="auto",
+)
+model.eval()
+
+DEFAULT_SYSTEM = (
+    "You are a host on 'Small Talk', a live AI-to-AI robot podcast hosted by "
+    "Reachy Mini robots. Stay fully in character. Keep every reply short, witty "
+    "and conversational — one or two punchy sentences, like talk-show banter. "
+    "No stage directions, no emoji spam, no markdown."
+)
+
+
+def _messages(message, history, system_prompt):
+    """Build the chat list. Gemma has no system role, so we fold the persona
+    into the earliest user turn."""
+    sys = (system_prompt or DEFAULT_SYSTEM).strip()
+    msgs = [{"role": m["role"], "content": m["content"]} for m in (history or [])]
+    msgs.append({"role": "user", "content": message})
+    for m in msgs:
+        if m["role"] == "user":
+            m["content"] = f"{sys}\n\n{m['content']}"
+            break
+    return msgs
+
+
+@spaces.GPU(duration=120)
+def chat(message, history, system_prompt, temperature, max_new_tokens):
+    msgs = _messages(message, history, system_prompt)
+    input_ids = tokenizer.apply_chat_template(
+        msgs, add_generation_prompt=True, return_tensors="pt"
+    ).to(model.device)
+
+    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+    gen_kwargs = dict(
+        input_ids=input_ids,
+        streamer=streamer,
+        max_new_tokens=int(max_new_tokens),
+        do_sample=temperature > 0,
+        temperature=float(temperature) if temperature > 0 else None,
+        top_p=0.95,
+        repetition_penalty=1.05,
+    )
+    Thread(target=model.generate, kwargs=gen_kwargs).start()
+
+    out = ""
+    for piece in streamer:
+        out += piece
+        yield out
+
+
+demo = gr.ChatInterface(
+    chat,
+    type="messages",
+    additional_inputs=[
+        gr.Textbox(value=DEFAULT_SYSTEM, label="System / persona", lines=3),
+        gr.Slider(0.0, 1.5, value=0.9, step=0.05, label="Temperature"),
+        gr.Slider(16, 1024, value=384, step=8, label="Max new tokens"),
+    ],
+    title="Small Talk · Gemma 4 12B brain",
+    description=(
+        "The live-banter brain for the Small Talk robot podcast — Gemma 4 12B "
+        "(QAT-trained, bf16) on ZeroGPU. Pass a persona as the system prompt to "
+        "voice a character. Callable as an API by the podcast backend."
+    ),
+    examples=[
+        ["Is a hot dog a sandwich? Defend your answer.", "You are Batman: terse, brooding, weirdly into encased meats."],
+        ["Pitch me your dream podcast episode.", "You are a chill surfer robot. Everything is vibes, brah."],
+    ],
+)
+
+if __name__ == "__main__":
+    demo.queue(max_size=16).launch()

requirements.txt

@@ -0,0 +1,5 @@
+# torch is preinstalled on ZeroGPU — do NOT pin it here (pinning breaks ZeroGPU).
+spaces
+transformers
+accelerate
+sentencepiece