Add llama.cpp intent generation and custom UI

app.py

@@ -1,38 +1,222 @@
 import os
+import json
 import time
 import tempfile
 
 import gradio as gr
 import soundfile as sf
 import torch
+
 from qwen_tts import Qwen3TTSModel
+from llama_cpp import Llama
+
+
+TTS_MODEL_ID = os.getenv("TTS_MODEL_ID", "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice")
+
+LLM_REPO_ID = os.getenv("LLM_REPO_ID", "Qwen/Qwen2.5-1.5B-Instruct-GGUF")
+LLM_FILENAME = os.getenv("LLM_FILENAME", "qwen2.5-1.5b-instruct-q4_k_m.gguf")
+
+tts_model = None
+llm_model = None
+
+
+CUSTOM_CSS = """
+:root {
+  --bg: #050816;
+  --panel: rgba(255, 255, 255, 0.075);
+  --panel-border: rgba(255, 255, 255, 0.16);
+  --text: #f8fafc;
+  --muted: #94a3b8;
+  --accent: #8b5cf6;
+  --accent-2: #06b6d4;
+}
+
+.gradio-container {
+  background:
+    radial-gradient(circle at 20% 20%, rgba(139, 92, 246, 0.30), transparent 28%),
+    radial-gradient(circle at 80% 0%, rgba(6, 182, 212, 0.24), transparent 28%),
+    linear-gradient(135deg, #050816 0%, #0f172a 55%, #111827 100%) !important;
+  color: var(--text) !important;
+  font-family: Inter, ui-sans-serif, system-ui, sans-serif !important;
+}
+
+#hero {
+  padding: 28px;
+  border: 1px solid var(--panel-border);
+  border-radius: 28px;
+  background: linear-gradient(135deg, rgba(255,255,255,0.10), rgba(255,255,255,0.04));
+  box-shadow: 0 24px 80px rgba(0,0,0,0.35);
+  backdrop-filter: blur(18px);
+}
 
+#hero h1 {
+  font-size: 42px;
+  line-height: 1.05;
+  margin-bottom: 8px;
+  letter-spacing: -0.04em;
+}
 
-MODEL_ID = os.getenv("MODEL_ID", "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice")
+#hero p {
+  color: var(--muted);
+  font-size: 16px;
+}
 
-model = None
+.badge-row {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 10px;
+  margin-top: 16px;
+}
 
+.badge {
+  padding: 8px 12px;
+  border-radius: 999px;
+  background: rgba(139, 92, 246, 0.16);
+  border: 1px solid rgba(139, 92, 246, 0.34);
+  color: #ddd6fe;
+  font-weight: 700;
+  font-size: 13px;
+}
 
-def get_model():
-    global model
+.block, .form, .panel {
+  border-radius: 22px !important;
+}
 
-    if model is not None:
-        return model
+textarea, input, select {
+  background: rgba(15, 23, 42, 0.72) !important;
+  color: var(--text) !important;
+  border-color: rgba(255,255,255,0.14) !important;
+}
+
+button.primary, button {
+  border-radius: 999px !important;
+  font-weight: 800 !important;
+}
+
+#run_llm {
+  background: linear-gradient(135deg, var(--accent), var(--accent-2)) !important;
+  color: white !important;
+  border: none !important;
+}
+
+#run_tts {
+  background: linear-gradient(135deg, #f97316, #ec4899) !important;
+  color: white !important;
+  border: none !important;
+}
+
+.footer-note {
+  color: var(--muted);
+  font-size: 13px;
+  text-align: center;
+}
+"""
+
+
+def get_tts_model():
+    global tts_model
+
+    if tts_model is not None:
+        return tts_model
 
     if torch.cuda.is_available():
-        model = Qwen3TTSModel.from_pretrained(
-            MODEL_ID,
+        tts_model = Qwen3TTSModel.from_pretrained(
+            TTS_MODEL_ID,
             device_map="cuda:0",
             dtype=torch.bfloat16,
         )
     else:
-        model = Qwen3TTSModel.from_pretrained(
-            MODEL_ID,
+        tts_model = Qwen3TTSModel.from_pretrained(
+            TTS_MODEL_ID,
             device_map="cpu",
             dtype=torch.float32,
         )
 
-    return model
+    return tts_model
+
+
+def get_llm_model():
+    global llm_model
+
+    if llm_model is not None:
+        return llm_model
+
+    # llama-cpp-python downloads the GGUF from Hugging Face.
+    # Q4_K_M is a good first compromise for CPU Spaces.
+    llm_model = Llama.from_pretrained(
+        repo_id=LLM_REPO_ID,
+        filename=LLM_FILENAME,
+        n_ctx=2048,
+        n_threads=max(2, os.cpu_count() or 2),
+        n_gpu_layers=-1 if torch.cuda.is_available() else 0,
+        verbose=True,
+    )
+
+    return llm_model
+
+
+def safe_json_loads(text):
+    try:
+        return json.loads(text)
+    except Exception:
+        return {
+            "raw_input": text,
+            "warning": "Input was not valid JSON, treated as raw text.",
+        }
+
+
+def generate_subtitle_and_instruction(intent_json_text):
+    intent = safe_json_loads(intent_json_text)
+
+    system_prompt = (
+        "You are an assistant inside an ASL-to-speech accessibility app. "
+        "Your job is to convert detected ASL glosses, pose/facial emotion, "
+        "and intent metadata into a short natural subtitle and a precise "
+        "voice instruction for a TTS model. "
+        "Return only valid JSON with exactly two keys: subtitle and voice_instruction."
+    )
+
+    user_prompt = f"""
+Input intent data:
+{json.dumps(intent, ensure_ascii=False, indent=2)}
+
+Rules:
+- Do not invent details that are not supported by the input.
+- Keep the subtitle short and natural.
+- The voice_instruction should describe tone, emotion, pace, and intensity.
+- Return only JSON.
+"""
+
+    llm = get_llm_model()
+
+    result = llm.create_chat_completion(
+        messages=[
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_prompt},
+        ],
+        temperature=0.2,
+        max_tokens=256,
+    )
+
+    content = result["choices"][0]["message"]["content"].strip()
+
+    try:
+        parsed = json.loads(content)
+    except Exception:
+        parsed = {
+            "subtitle": content,
+            "voice_instruction": "Speak clearly and naturally.",
+        }
+
+    subtitle = parsed.get("subtitle", "").strip()
+    voice_instruction = parsed.get("voice_instruction", "").strip()
+
+    if not subtitle:
+        subtitle = "I want to say something."
+    if not voice_instruction:
+        voice_instruction = "Speak clearly and naturally."
+
+    return subtitle, voice_instruction, parsed
 
 
 def generate_tts(text, language, speaker, instruction):
@@ -40,9 +224,9 @@ def generate_tts(text, language, speaker, instruction):
     instruction = (instruction or "").strip()
 
     if not text:
-        raise gr.Error("Écris une phrase à synthétiser.")
+        raise gr.Error("Aucun subtitle à synthétiser.")
 
-    tts = get_model()
+    tts = get_tts_model()
 
     wavs, sr = tts.generate_custom_voice(
         text=text,
@@ -58,45 +242,80 @@ def generate_tts(text, language, speaker, instruction):
 
     sf.write(output_path, wavs[0], sr)
 
-    intent_json = {
-        "detected_glosses": [],
-        "detected_facial_expression": "not_connected_yet",
-        "subtitle": text,
-        "voice_instruction": instruction,
-        "language": language,
-        "speaker": speaker,
-        "pipeline_stage": "tts_only_mvp",
-    }
+    return output_path
 
-    return output_path, text, intent_json
 
+DEFAULT_INTENT = {
+    "detected_glosses": ["I", "HAPPY", "SEE", "YOU"],
+    "detected_facial_expression": "happy",
+    "emotion_profile": {
+        "dominant": "joy",
+        "confidence": 0.83,
+    },
+    "communication_intent": "friendly_greeting",
+    "pipeline_stage": "mock_asl_intent_for_llama_cpp_test",
+}
 
-with gr.Blocks(title="ASL to TTS MVP") as demo:
-    gr.Markdown(
-        """
-        # ASL to TTS MVP
-
-        Première version: on teste seulement la brique TTS.
 
-        Ensuite, on branchera:
-        video ASL -> glosses -> emotion -> intent JSON -> subtitle -> voice instruction -> TTS.
+with gr.Blocks(
+    title="SignSpeak Local",
+    css=CUSTOM_CSS,
+    theme=gr.themes.Base(),
+) as demo:
+    gr.HTML(
+        """
+        <section id="hero">
+          <h1>SignSpeak Local</h1>
+          <p>
+            ASL video to expressive speech, built as a local-first accessibility pipeline.
+            Current milestone: llama.cpp intent generation + Qwen3-TTS voice synthesis.
+          </p>
+          <div class="badge-row">
+            <span class="badge">🦙 llama.cpp</span>
+            <span class="badge">🔌 local-first</span>
+            <span class="badge">🎨 custom Gradio UI</span>
+            <span class="badge">🗣️ expressive TTS</span>
+          </div>
+        </section>
         """
     )
 
     with gr.Row():
-        with gr.Column():
-            text_input = gr.Textbox(
-                label="Subtitle temporaire",
-                value="Hello, I am happy to see you today.",
+        with gr.Column(scale=1):
+            gr.Markdown("## 1. Intent input")
+
+            intent_input = gr.Textbox(
+                label="Mock intent JSON",
+                value=json.dumps(DEFAULT_INTENT, ensure_ascii=False, indent=2),
+                lines=13,
+            )
+
+            run_llm_button = gr.Button(
+                "Generate subtitle with llama.cpp",
+                elem_id="run_llm",
+            )
+
+        with gr.Column(scale=1):
+            gr.Markdown("## 2. llama.cpp output")
+
+            subtitle_output = gr.Textbox(
+                label="Subtitle",
                 lines=3,
             )
 
-            instruction_input = gr.Textbox(
+            instruction_output = gr.Textbox(
                 label="Voice instruction",
-                value="Speak with a warm, happy, expressive voice.",
-                lines=2,
+                lines=3,
             )
 
+            llm_json_output = gr.JSON(
+                label="LLM structured output",
+            )
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            gr.Markdown("## 3. Voice synthesis")
+
             language_input = gr.Dropdown(
                 label="Language",
                 choices=[
@@ -131,26 +350,42 @@ with gr.Blocks(title="ASL to TTS MVP") as demo:
                 value="Ryan",
             )
 
-            button = gr.Button("Generate speech")
+            run_tts_button = gr.Button(
+                "Generate expressive speech",
+                elem_id="run_tts",
+            )
+
+        with gr.Column(scale=1):
+            gr.Markdown("## 4. Result")
+
+            audio_output = gr.Audio(
+                label="Generated audio",
+                type="filepath",
+            )
+
+    gr.HTML(
+        """
+        <p class="footer-note">
+          Hackathon target badges: Llama Champion, Off-Brand, Off the Grid.
+        </p>
+        """
+    )
 
-        with gr.Column():
-            audio_output = gr.Audio(label="Generated audio", type="filepath")
-            subtitle_output = gr.Textbox(label="Subtitle")
-            json_output = gr.JSON(label="Intent JSON")
+    run_llm_button.click(
+        fn=generate_subtitle_and_instruction,
+        inputs=[intent_input],
+        outputs=[subtitle_output, instruction_output, llm_json_output],
+    )
 
-    button.click(
+    run_tts_button.click(
         fn=generate_tts,
         inputs=[
-            text_input,
+            subtitle_output,
             language_input,
             speaker_input,
-            instruction_input,
-        ],
-        outputs=[
-            audio_output,
-            subtitle_output,
-            json_output,
+            instruction_output,
         ],
+        outputs=[audio_output],
     )
 
 

requirements.txt

@@ -2,3 +2,5 @@ gradio
 qwen-tts
 soundfile
 torch
+llama-cpp-python
+huggingface-hub