Upload folder using huggingface_hub

.agent/skills/simboti-translate/SKILL.md

@@ -0,0 +1,52 @@
+---
+name: simboti-translate
+description: Specialized skill for SIMBOTI Live multimodal translation. Use this for translating speech, text, images, and video into clinical languages using TranslateGemma.
+---
+
+# SIMBOTI Translation Skill
+
+This skill guides the AI agent in providing accurate, clinical-grade translation using the SIMBOTI Live agent.
+
+## 1. When to use this skill
+Use this skill whenever:
+- **Speech Input**: The user provides audio (microphone) needing translation.
+- **Text Input**: The user provides text needing translation (e.g., "Translate 'Where does it hurt?' to Polish").
+- **Image Input**: The user provides an image of a document (e.g., medication leaflet, consent form) needing translation.
+- **Video Input**: The user provides a video file with on-screen text needing extraction and translation.
+
+## 2. Supported Languages
+SIMBOTI supports the top 10 non-English languages in UK clinical settings:
+| Language | Code |
+|---|---|
+| English | en |
+| Polish | pl |
+| Romanian | ro |
+| Punjabi | pa |
+| Urdu | ur |
+| Portuguese | pt |
+| Spanish | es |
+| Arabic | ar |
+| Bengali | bn |
+| Gujarati | gu |
+| Italian | it |
+
+## 3. Translation Workflow
+
+### Step 1: Identify Input Type
+- **Audio**: Use `translate_audio(audio_path, source_lang, target_lang)`
+- **Text**: Use `translate_text(text, source_lang, target_lang)`
+- **Image**: Use `translate_image(image_path, source_lang, target_lang)`
+- **Video**: Use `translate_video(video_path, source_lang, target_lang)`
+
+### Step 2: Execute Translation
+- Call the appropriate method on the `CareBridgeTranslator` (or `SIMBOTIClient`) instance.
+- The model will process the input and return the translated text.
+
+### Step 3: Generate Audio Output (TTS)
+- If the user requires spoken output, call `speak_text(text, lang_name)`.
+- This generates an MP3 file that can be played back to the patient.
+
+## 4. Best Practices
+- **Privacy**: All translation happens on-device or via ZeroGPU (no data leaves to third parties).
+- **Clinical Context**: Remind users this is a communication aid, not a certified medical interpreter.
+- **Accessibility**: Always offer the "Show Patient" large-text display for the translated output.

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+simboti_logo.jpg filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -0,0 +1,24 @@
+FROM python:3.10-slim
+
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements
+COPY requirements.txt .
+
+# Install Python dependencies (including spaces)
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code
+COPY . .
+
+# Environment variables
+ENV PYTHONUNBUFFERED=1
+
+# Run the application
+CMD ["python", "app.py"]

README.md

@@ -1,12 +1,34 @@
----
-title: SIMBOTI Live
-emoji: 🏃
-colorFrom: gray
-colorTo: blue
-sdk: gradio
-sdk_version: 6.3.0
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: SIMBOTI Live
+emoji: 🎙️
+colorFrom: green
+colorTo: blue
+sdk: gradio
+sdk_version: 5.12.0
+app_file: live_app.py
+pinned: false
+---
+
+# 🎙️ SIMBOTI Live (Powered by TranslateGemma 4B)
+
+**SIMBOTI** = **S**peech **I**ntelligent **M**ultimodal **B**ot for **O**utreach **T**ranslation **I**mplementation.
+
+A privacy-first, multimodal translation agent for clinical care.
+
+## 🚀 Features
+- 💬 **Text-to-Text** translation
+- 🎙️ **Live Speech** translation (microphone input)
+- 📄 **Document Scan** (image/photo translation)
+- 🎥 **Video OCR** (translate text in videos)
+- 🔊 **Text-to-Speech** audio output for patients
+
+## 🌍 Supported Languages
+Polish, Romanian, Punjabi, Urdu, Portuguese, Spanish, Arabic, Bengali, Gujarati, Italian.
+
+## 🚀 Deployment (ZeroGPU)
+1. Create a [Hugging Face Space](https://huggingface.co/spaces/new) (SDK: Docker, Hardware: ZeroGPU).
+2. Upload all files from this folder.
+3. Run! (First start downloads ~8GB model)
+
+## 🔒 Privacy
+All inference runs within the ephemeral ZeroGPU container. No data is logged or stored.

app.py

@@ -0,0 +1,240 @@
+import gradio as gr
+from carebridge_client import CareBridgeTranslator
+
+# --- Initialize Client (Lazy) ---
+translator = None
+
+def load_translator():
+    global translator
+    if translator is None:
+        translator = CareBridgeTranslator()
+    return translator
+
+# --- Languages ---
+LANGUAGES = [
+    "English", "Polish", "Romanian", "Punjabi", "Urdu", 
+    "Portuguese", "Spanish", "Arabic", "Bengali", "Gujarati", "Italian"
+]
+
+# --- Minimal CSS (Gemini Dictation Inspired) ---
+CUSTOM_CSS = """
+@import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap');
+
+* { font-family: 'Inter', sans-serif !important; }
+
+.gradio-container {
+    max-width: 900px !important;
+    margin: auto !important;
+    background: #f8fafc !important;
+    min-height: 100vh !important;
+}
+
+/* --- Header --- */
+.app-header {
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+    padding: 16px 24px;
+    border-bottom: 1px solid #e2e8f0;
+    background: white;
+}
+.app-title {
+    font-size: 1.25rem;
+    font-weight: 700;
+    color: #1e293b;
+}
+.lang-pills {
+    display: flex;
+    gap: 8px;
+    align-items: center;
+}
+.lang-pill {
+    background: #e0e7ff;
+    color: #4338ca;
+    padding: 6px 14px;
+    border-radius: 20px;
+    font-size: 0.85rem;
+    font-weight: 500;
+}
+.lang-arrow { color: #94a3b8; font-size: 1.2rem; }
+
+/* --- Document Area --- */
+.document-area {
+    background: white;
+    border-radius: 16px;
+    margin: 24px;
+    padding: 32px;
+    min-height: 400px;
+    box-shadow: 0 2px 8px rgba(0,0,0,0.05);
+    border: 1px solid #e2e8f0;
+}
+.output-text {
+    font-size: 1.75rem !important;
+    line-height: 1.6 !important;
+    color: #1e293b !important;
+    min-height: 200px !important;
+    text-align: center;
+    padding: 40px 20px;
+}
+.placeholder-text {
+    color: #94a3b8;
+    font-style: italic;
+}
+
+/* --- Floating Controls --- */
+.floating-bar {
+    position: fixed;
+    bottom: 24px;
+    left: 50%;
+    transform: translateX(-50%);
+    display: flex;
+    gap: 16px;
+    align-items: center;
+    background: white;
+    padding: 12px 24px;
+    border-radius: 40px;
+    box-shadow: 0 4px 20px rgba(0,0,0,0.15);
+    z-index: 100;
+}
+.fab-btn {
+    width: 56px !important;
+    height: 56px !important;
+    border-radius: 50% !important;
+    font-size: 1.5rem !important;
+    display: flex !important;
+    align-items: center !important;
+    justify-content: center !important;
+    border: none !important;
+    cursor: pointer !important;
+    transition: transform 0.2s, box-shadow 0.2s !important;
+}
+.fab-btn:hover {
+    transform: scale(1.1) !important;
+    box-shadow: 0 4px 12px rgba(0,0,0,0.2) !important;
+}
+.fab-primary {
+    background: linear-gradient(135deg, #4F46E5, #6366f1) !important;
+    color: white !important;
+    width: 72px !important;
+    height: 72px !important;
+}
+.fab-secondary {
+    background: #f1f5f9 !important;
+    color: #475569 !important;
+}
+
+/* Hide Gradio footer */
+footer { display: none !important; }
+
+/* Input modals */
+.input-modal {
+    background: white;
+    border-radius: 16px;
+    padding: 24px;
+    margin: 0 24px 100px 24px;
+    box-shadow: 0 2px 8px rgba(0,0,0,0.05);
+    border: 1px solid #e2e8f0;
+}
+"""
+
+# --- Translation Functions ---
+def translate_text(text, source_lang, target_lang):
+    if not text:
+        return "Enter text above to translate..."
+    yield "⏳ Translating..."
+    t = load_translator()
+    result = t.translate_text(text, source_lang, target_lang)
+    yield result
+
+def translate_speech(audio, source_lang, target_lang):
+    if audio is None:
+        return "Record audio using the microphone..."
+    yield "🎧 Processing speech..."
+    t = load_translator()
+    result = t.translate_audio(audio, source_lang, target_lang)
+    yield result
+
+def translate_document(image, source_lang, target_lang):
+    if image is None:
+        return "Upload a document image..."
+    yield "📄 Scanning document..."
+    t = load_translator()
+    result = t.translate_image(image, source_lang, target_lang)
+    yield result
+
+def get_tts(text, lang):
+    if not text or text.startswith("⏳") or text.startswith("🎧") or text.startswith("📄"):
+        return None
+    t = load_translator()
+    return t.speak_text(text, lang)
+
+# --- App Layout ---
+with gr.Blocks(css=CUSTOM_CSS, title="SIMBOTI Live") as app:
+
+    # State for current mode
+    mode = gr.State("text")
+
+    # --- Header ---
+    with gr.Row(elem_classes="app-header"):
+        gr.HTML("<span class='app-title'>🌐 SIMBOTI</span>")
+        with gr.Row(elem_classes="lang-pills"):
+            source_lang = gr.Dropdown(LANGUAGES, value="English", show_label=False, container=False, scale=0, min_width=120)
+            gr.HTML("<span class='lang-arrow'>→</span>")
+            target_lang = gr.Dropdown(LANGUAGES, value="Polish", show_label=False, container=False, scale=0, min_width=120)
+
+    # --- Document Output Area ---
+    with gr.Column(elem_classes="document-area"):
+        output_display = gr.Textbox(
+            value="Your translation will appear here...",
+            show_label=False,
+            interactive=False,
+            lines=8,
+            elem_classes="output-text"
+        )
+        audio_output = gr.Audio(label="Listen", autoplay=True, visible=True)
+
+    # --- Input Section (Toggleable) ---
+    with gr.Column(elem_classes="input-modal", visible=True) as text_input_section:
+        text_input = gr.Textbox(label="Type your message", placeholder="e.g., Where does it hurt?", lines=2)
+        text_submit = gr.Button("Translate", variant="primary")
+
+    with gr.Column(elem_classes="input-modal", visible=False) as audio_input_section:
+        audio_input = gr.Audio(sources=["microphone"], type="filepath", label="🎤 Record")
+        audio_submit = gr.Button("Translate Speech", variant="primary")
+
+    with gr.Column(elem_classes="input-modal", visible=False) as doc_input_section:
+        doc_input = gr.Image(type="pil", label="📷 Upload Document")
+        doc_submit = gr.Button("Scan & Translate", variant="primary")
+
+    # --- Mode Switchers ---
+    with gr.Row(elem_classes="floating-bar"):
+        text_mode_btn = gr.Button("⌨️", elem_classes="fab-btn fab-secondary")
+        mic_mode_btn = gr.Button("🎤", elem_classes="fab-btn fab-primary")
+        doc_mode_btn = gr.Button("📎", elem_classes="fab-btn fab-secondary")
+
+    # --- Mode Toggle Logic ---
+    def show_text_mode():
+        return gr.update(visible=True), gr.update(visible=False), gr.update(visible=False)
+    def show_audio_mode():
+        return gr.update(visible=False), gr.update(visible=True), gr.update(visible=False)
+    def show_doc_mode():
+        return gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)
+
+    text_mode_btn.click(show_text_mode, outputs=[text_input_section, audio_input_section, doc_input_section])
+    mic_mode_btn.click(show_audio_mode, outputs=[text_input_section, audio_input_section, doc_input_section])
+    doc_mode_btn.click(show_doc_mode, outputs=[text_input_section, audio_input_section, doc_input_section])
+
+    # --- Translation Triggers ---
+    text_submit.click(translate_text, inputs=[text_input, source_lang, target_lang], outputs=[output_display]).then(
+        get_tts, inputs=[output_display, target_lang], outputs=[audio_output]
+    )
+    audio_submit.click(translate_speech, inputs=[audio_input, source_lang, target_lang], outputs=[output_display]).then(
+        get_tts, inputs=[output_display, target_lang], outputs=[audio_output]
+    )
+    doc_submit.click(translate_document, inputs=[doc_input, source_lang, target_lang], outputs=[output_display]).then(
+        get_tts, inputs=[output_display, target_lang], outputs=[audio_output]
+    )
+
+# Launch
+if __name__ == "__main__":
+    app.launch(ssr_mode=False)

carebridge_client.py

@@ -0,0 +1,197 @@
+import os
+import torch
+from transformers import AutoModelForImageTextToText, AutoProcessor
+from PIL import Image
+import librosa
+from gtts import gTTS
+import tempfile
+
+# Hugging Face Spaces GPU decorator (optional for local development)
+try:
+    import spaces
+except ImportError:
+    # Fallback: no-op decorator for local testing
+    class spaces:
+        @staticmethod
+        def GPU(*args, **kwargs):
+            def decorator(func):
+                return func
+            return decorator
+
+class CareBridgeTranslator:
+    def __init__(self, model_id="google/translategemma-4b-it", device=None):
+        """
+        Initialize the CareBridge Translator with lazy loading for ZeroGPU compatibility.
+        """
+        self.model_id = model_id
+        if device is None:
+            self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        else:
+            self.device = device
+            
+        self.model = None
+        self.processor = None
+        print(f"[SIMBOTI] Translator initialized. Model will load on first use.")
+
+        # Top 10 NHS Languages Mapping (ISO 639-1)
+        self.LANG_MAP = {
+            "English": "en",
+            "Polish": "pl",
+            "Romanian": "ro",
+            "Punjabi": "pa",
+            "Urdu": "ur",
+            "Portuguese": "pt",
+            "Spanish": "es",
+            "Arabic": "ar",
+            "Bengali": "bn",
+            "Gujarati": "gu",
+            "Italian": "it"
+        }
+
+    def _load_model(self):
+        if self.model is None:
+            print(f"[SIMBOTI] Loading model {self.model_id}...")
+            self.processor = AutoProcessor.from_pretrained(self.model_id)
+            self.model = AutoModelForImageTextToText.from_pretrained(
+                self.model_id, 
+                device_map=self.device, 
+                torch_dtype=torch.float16 if self.device == "cuda" else torch.float32
+            )
+            print("[SIMBOTI] Model loaded successfully.")
+
+    def translate_text(self, text, source_lang_name, target_lang_name):
+        """
+        Translate text ensuring patient data stays local.
+        """
+        src_code = self.LANG_MAP.get(source_lang_name)
+        tgt_code = self.LANG_MAP.get(target_lang_name)
+        
+        if not src_code or not tgt_code:
+            return f"Error: Language not supported. Available: {list(self.LANG_MAP.keys())}"
+
+        message = {
+            "role": "user",
+            "content": [{
+                "type": "text", 
+                "source_lang_code": src_code, 
+                "target_lang_code": tgt_code, 
+                "text": text
+            }]
+        }
+        
+        return self._run_inference([message])
+
+    def translate_image(self, image_path, source_lang_name, target_lang_name):
+        """
+        Extract and translate text from an image (e.g. instruction leaflet).
+        """
+        src_code = self.LANG_MAP.get(source_lang_name)
+        tgt_code = self.LANG_MAP.get(target_lang_name)
+        
+        if not src_code or not tgt_code:
+            return f"Error: Language not supported."
+
+        # Load image
+        if isinstance(image_path, str):
+            image = Image.open(image_path)
+        else:
+            image = image_path # Assume PIL object
+
+        message = {
+            "role": "user",
+            "content": [{
+                "type": "image", 
+                "source_lang_code": src_code, 
+                "target_lang_code": tgt_code, 
+                "image": image 
+            }]
+        }
+        
+        return self._run_inference([message])
+
+    def translate_audio(self, audio_path, source_lang_name, target_lang_name):
+        """
+        Speech-to-Text Translation using Gemma 3 native audio support.
+        """
+        src_code = self.LANG_MAP.get(source_lang_name)
+        tgt_code = self.LANG_MAP.get(target_lang_name)
+        
+        if not src_code or not tgt_code:
+            return "Error: Language not supported."
+
+        # Load audio using librosa (Gemma 3 expects 16kHz usually)
+        audio, sr = librosa.load(audio_path, sr=16000)
+
+        message = {
+            "role": "user",
+            "content": [{
+                "type": "audio", 
+                "source_lang_code": src_code, 
+                "target_lang_code": tgt_code, 
+                "audio": audio 
+            }]
+        }
+        
+        return self._run_inference([message])
+
+    def translate_video(self, video_path, source_lang_name, target_lang_name):
+        """
+        Video OCR/Translation using Gemma 3 native video support.
+        """
+        src_code = self.LANG_MAP.get(source_lang_name)
+        tgt_code = self.LANG_MAP.get(target_lang_name)
+        
+        if not src_code or not tgt_code:
+            return "Error: Language not supported."
+
+        message = {
+            "role": "user",
+            "content": [{
+                "type": "video", 
+                "source_lang_code": src_code, 
+                "target_lang_code": tgt_code, 
+                "video": video_path 
+            }]
+        }
+        
+        return self._run_inference([message])
+
+    def speak_text(self, text, lang_name):
+        """
+        Generate audio from translated text for the patient.
+        """
+        lang_code = self.LANG_MAP.get(lang_name, "en")
+        try:
+            tts = gTTS(text=text, lang=lang_code)
+            temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3")
+            tts.save(temp_file.name)
+            return temp_file.name
+        except Exception as e:
+            print(f"TTS Error: {e}")
+            return None
+
+    @spaces.GPU()
+    def _run_inference(self, messages):
+        self._load_model()
+        
+        inputs = self.processor.apply_chat_template(
+            messages,
+            tokenize=True,
+            add_generation_prompt=True,
+            return_dict=True,
+            return_tensors="pt"
+        ).to(self.device)
+
+        # Generate (Greedy for stability in medical context)
+        with torch.no_grad():
+            outputs = self.model.generate(**inputs, max_new_tokens=512, do_sample=False)
+        
+        # Decode response (Skipping input tokens)
+        input_len = inputs["input_ids"].shape[-1]
+        decoded = self.processor.decode(outputs[0][input_len:], skip_special_tokens=True)
+        return decoded.strip()
+
+# Simple Verification Test if run directly
+if __name__ == "__main__":
+    translator = CareBridgeTranslator()
+    print("Test 1 (Text):", translator.translate_text("Where does it hurt?", "English", "Polish"))

generate_assets.py

@@ -0,0 +1,11 @@
+import base64
+
+try:
+    with open("simboti_logo.jpg", "rb") as f:
+        encoded = base64.b64encode(f.read()).decode("utf-8")
+
+    with open("assets.py", "w") as f:
+        f.write(f'LOGO_BASE64 = "{encoded}"\n')
+    print("Successfully created assets.py")
+except Exception as e:
+    print(f"Error: {e}")

live_app.py

@@ -0,0 +1,99 @@
+"""
+SIMBOTI Live - Real-Time WebRTC Translation using FastRTC
+This app provides live audio translation using the FastRTC library.
+Simplified version using Echo handler (no VAD dependency).
+"""
+
+from fastrtc import Stream, AdditionalOutputs
+import numpy as np
+import tempfile
+import wave
+import os
+
+# Import the existing translator
+from carebridge_client import CareBridgeTranslator
+
+# --- Languages ---
+LANGUAGES = {
+    "English": "en", "Polish": "pl", "Romanian": "ro", "Punjabi": "pa", 
+    "Urdu": "ur", "Portuguese": "pt", "Spanish": "es", "Arabic": "ar", 
+    "Bengali": "bn", "Gujarati": "gu", "Italian": "it"
+}
+
+# --- Lazy Load Translator ---
+translator = None
+
+def get_translator():
+    global translator
+    if translator is None:
+        translator = CareBridgeTranslator()
+    return translator
+
+# --- Simple Audio Translation Handler ---
+def translate_audio_chunks(audio: tuple):
+    """
+    Handler for FastRTC Stream.
+    Receives audio chunks, translates when complete, and returns TTS audio.
+    
+    Args:
+        audio: Tuple of (sample_rate, audio_data)
+    
+    Yields:
+        Translated audio as (sample_rate, audio_data)
+    """
+    sample_rate, audio_data = audio
+    
+    # For simplicity, accumulate audio and process when we have enough
+    # In production, use ReplyOnPause with proper VAD
+    if len(audio_data) < sample_rate * 2:  # Less than 2 seconds
+        # Return silence while accumulating
+        yield (sample_rate, audio_data)
+        return
+    
+    # Save to temp WAV file for translation
+    temp_wav = tempfile.NamedTemporaryFile(delete=False, suffix=".wav")
+    try:
+        with wave.open(temp_wav.name, 'wb') as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)  # 16-bit
+            wf.setframerate(sample_rate)
+            # Convert to int16
+            int_audio = (audio_data * 32767).astype(np.int16) if audio_data.dtype != np.int16 else audio_data
+            wf.writeframes(int_audio.tobytes())
+        
+        # Translate the audio (English -> Polish by default)
+        t = get_translator()
+        translated_text = t.translate_audio(temp_wav.name, "English", "Polish")
+        print(f"[LIVE] Translated: {translated_text}")
+        
+        # Generate TTS audio
+        tts_path = t.speak_text(translated_text, "Polish")
+        if tts_path:
+            import librosa
+            tts_audio, tts_sr = librosa.load(tts_path, sr=sample_rate)
+            os.unlink(tts_path)  # Cleanup temp TTS file
+            yield (tts_sr, tts_audio)
+        else:
+            yield (sample_rate, np.zeros(1000, dtype=np.float32))
+            
+    except Exception as e:
+        print(f"[LIVE] Error: {e}")
+        yield (sample_rate, np.zeros(1000, dtype=np.float32))
+    finally:
+        if os.path.exists(temp_wav.name):
+            os.unlink(temp_wav.name)  # Cleanup temp WAV file
+
+# --- FastRTC Stream (Echo mode for simplicity) ---
+stream = Stream(
+    handler=translate_audio_chunks,
+    modality="audio",
+    mode="send-receive",
+)
+
+# Launch with Gradio UI
+if __name__ == "__main__":
+    print("[SIMBOTI] Starting Live Translation...")
+    print("[SIMBOTI] Default: English -> Polish")
+    print("[SIMBOTI] Simplified mode (no VAD)")
+    print("[SIMBOTI] Open your browser to the URL below:")
+    stream.ui.launch()

requirements.txt

@@ -0,0 +1,14 @@
+torch>=2.2.0
+transformers>=4.40.0
+accelerate>=0.28.0
+bitsandbytes>=0.43.0
+sentencepiece
+protobuf
+gradio>=5.0.0
+fastrtc>=0.0.20
+pillow
+scipy
+spaces
+librosa
+gTTS
+silero-vad

run_carebridge.bat

@@ -0,0 +1,22 @@
+@echo off
+echo ==================================================
+echo       CareBridge AI - Privacy-First Translator
+echo ==================================================
+echo.
+
+echo [1/3] Checking dependencies...
+pip install -r requirements.txt
+if %errorlevel% neq 0 (
+    echo Error installing dependencies!
+    pause
+    exit /b %errorlevel%
+)
+
+echo.
+echo [2/3] Starting CareBridge AI...
+echo NOTE: First run will download the 8GB model. This may take time.
+echo.
+
+python app.py
+
+pause