gemma 12b

README.md

@@ -5,7 +5,7 @@ colorFrom: green
 colorTo: pink
 sdk: gradio
 sdk_version: 6.15.2
-python_version: '3.13'
+python_version: '3.12'
 app_file: app.py
 pinned: false
 license: mit

app.py

@@ -6,19 +6,31 @@ Left:  a slim, collapsible chat rail. Talk to the AI, ask for an app, then ask
 Right: the generated app, rendered live with full JavaScript.
 
 Theme: Frutiger Aero / skeuomorphic glass.
-Model: Gemma 4 31B via Ollama Cloud (prototyping).
-  Requires:  pip install -U ollama agno gradio   and   OLLAMA_API_KEY set.
+Model: Gemma 4 12B via llama.cpp.
+  Local: start `llama-server -hf ggml-org/gemma-4-12B-it-GGUF:Q4_K_M --jinja -c 4096`
+         before running this app, or install llama-cpp-python so the app can spawn it.
+  Space: the app spawns llama_cpp.server on CPU Basic unless a server is already running.
 """
 
 import os
 import re
+import sys
+import time
 import html as html_lib
+import subprocess
 
 import gradio as gr
+import requests
 from agno.agent import Agent
-from agno.models.ollama import Ollama
+from agno.models.llama_cpp import LlamaCpp
 
-OLLAMA_MODEL = os.environ.get("OLLAMA_MODEL", "gemma4:31b-cloud")
+MODEL_REPO = os.environ.get("MODEL_REPO", "ggml-org/gemma-4-12B-it-GGUF")
+MODEL_FILE = os.environ.get("MODEL_FILE", "gemma-4-12B-it-Q4_K_M.gguf")
+HOST = os.environ.get("LLAMACPP_HOST", "127.0.0.1")
+PORT = int(os.environ.get("LLAMACPP_PORT", "8080"))
+BASE_URL = os.environ.get("LLAMACPP_BASE_URL", f"http://{HOST}:{PORT}/v1")
+N_CTX = os.environ.get("LLAMACPP_CTX", "4096")
+N_THREADS = os.environ.get("LLAMACPP_THREADS", "2")
 
 SYSTEM_PROMPT = """You are Discode, a friendly expert front-end engineer who builds and edits ONE single-page web app for the user through conversation.
 
@@ -36,10 +48,55 @@ When the user asks for a change, MODIFY the current app (it will be given to you
 If the user is only chatting (greeting, a question) and not asking for an app or change, reply normally with NO code block.
 """
 
+
+def server_is_up() -> bool:
+    try:
+        return requests.get(f"{BASE_URL}/models", timeout=2).status_code == 200
+    except requests.exceptions.RequestException:
+        return False
+
+
+server_process = None
+if server_is_up():
+    print(f"[startup] Found llama.cpp server at {BASE_URL}", flush=True)
+else:
+    print("[startup] No llama.cpp server found; downloading GGUF and spawning llama_cpp.server ...", flush=True)
+    from huggingface_hub import hf_hub_download
+
+    model_path = hf_hub_download(repo_id=MODEL_REPO, filename=MODEL_FILE)
+    server_process = subprocess.Popen(
+        [
+            sys.executable,
+            "-m",
+            "llama_cpp.server",
+            "--model",
+            model_path,
+            "--host",
+            HOST,
+            "--port",
+            str(PORT),
+            "--n_ctx",
+            N_CTX,
+            "--n_threads",
+            N_THREADS,
+        ]
+    )
+
+    print("[startup] Waiting for llama.cpp server ...", flush=True)
+    for _ in range(360):
+        if server_is_up():
+            print("[startup] llama.cpp server is ready.", flush=True)
+            break
+        time.sleep(1)
+    else:
+        raise RuntimeError("llama.cpp server did not start in time")
+
+
 agent = Agent(
-    model=Ollama(id=OLLAMA_MODEL),
+    model=LlamaCpp(id=MODEL_FILE, base_url=BASE_URL, api_key="sk-no-key-needed"),
     instructions=SYSTEM_PROMPT,
     markdown=False,
+    debug_mode=True,
 )
 
 
@@ -98,11 +155,6 @@ def on_send(user_msg, messages, current_html):
     if not user_msg or not user_msg.strip():
         return messages, gr.update(), current_html, ""
 
-    if not os.environ.get("OLLAMA_API_KEY"):
-        messages.append({"role": "user", "content": user_msg})
-        messages.append({"role": "assistant", "content": "⚠️ OLLAMA_API_KEY is not set."})
-        return messages, gr.update(), current_html, ""
-
     messages.append({"role": "user", "content": user_msg})
 
     if current_html:
@@ -113,8 +165,11 @@ def on_send(user_msg, messages, current_html):
     else:
         prompt = user_msg
 
+    started = time.time()
     result = agent.run(prompt)
+    elapsed = time.time() - started
     chat_text, new_html = parse_response(result.content)
+    chat_text = f"{chat_text}\n\n_responded in {elapsed:.1f}s via Gemma 4 12B Q4 on llama.cpp_"
 
     messages.append({"role": "assistant", "content": chat_text})
 
@@ -203,7 +258,6 @@ with gr.Blocks(css=AERO_CSS, theme=gr.themes.Soft(), title="Discode") as demo:
         # Left: slim chat rail
         with gr.Column(scale=2, min_width=280, elem_id="chat-col") as chat_col:
             chatbot = gr.Chatbot(
-                type="messages",
                 height="62vh",
                 elem_id="chatbox",
                 show_label=False,

requirements.txt

@@ -1,4 +1,5 @@
 gradio
 agno
-ollama
-openai
+huggingface_hub
+llama-cpp-python[server]
+requests