app.py

# app.py — SpellTalk (polished UI for HF Spaces)

import os
os.environ["YOLO_CONFIG_DIR"] = "/tmp/Ultralytics"   # writable dir on Spaces

import time, string
from collections import deque, Counter
import cv2
import numpy as np
import gradio as gr
from ultralytics import YOLO
from huggingface_hub import hf_hub_download

# ---- FastRTC WebRTC component
from fastrtc import WebRTC

# ================== Model ==================
weights_path = hf_hub_download("atalaydenknalbant/asl-yolo-models", filename="yolo11n.pt")
model = YOLO(weights_path)
id2label = model.names
VALID = {lbl.upper() for lbl in id2label.values() if len(lbl) == 1 and lbl in string.ascii_letters}

# ================== App State (globals) ==================
CONF_THRES = 0.35
HIST_LEN = 9                 # history window (frames)
COMMIT_STREAK = 5            # frames of agreement to lock a letter
WORD_BREAK_SECONDS = 1.0

history = deque(maxlen=HIST_LEN)
current_word, sentence = [], []
last_commit_time = time.time()

HELP_CHART_URL = "https://commons.wikimedia.org/wiki/Special:FilePath/Asl_alphabet_gallaudet_ann.png"

# ================== Helpers ==================
def most_common(q):
    items = [x for x in q if x is not None]
    if not items:
        return None, 0
    c = Counter(items).most_common(1)[0]
    return c[0], c[1]

def reset_word():
    global current_word, history
    current_word = []
    history.clear()

def reset_sentence():
    global sentence, current_word, history
    sentence = []
    current_word = []
    history.clear()

# ================== Video Frame Handler ==================
def process_frame(frame: np.ndarray):
    """
    FastRTC handler: receives RGB (H, W, 3); returns annotated RGB.
    Keeps running state in globals so words persist across frames.
    """
    global history, current_word, sentence, last_commit_time

    if frame is None:
        return None

    # mirror like a selfie camera
    frame = cv2.flip(frame, 1)

    # run detection
    try:
        res = model.predict(frame, verbose=False, conf=CONF_THRES)
    except Exception as e:
        cv2.putText(frame, f"Model error: {e}", (10, 30),
                    cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 0, 255), 2)
        return frame

    letter, conf = None, 0.0
    if res and len(res[0].boxes) > 0:
        scores = res[0].boxes.conf.cpu().numpy()
        k = scores.argmax()
        idx = int(res[0].boxes.cls[k].item())
        conf = float(scores[k])
        cand = id2label.get(idx, "").upper()
        if cand in VALID:
            letter = cand

    history.append(letter)
    top, count = most_common(history)

    now = time.time()
    if top and count >= COMMIT_STREAK:
        current_word.append(top)
        history.clear()
        last_commit_time = now

    if (top is None) and (now - last_commit_time > WORD_BREAK_SECONDS) and current_word:
        sentence.append("".join(current_word))
        current_word = []
        last_commit_time = now

    # ---------- overlay ----------
    h, w = frame.shape[:2]
    overlay = frame.copy()
    cv2.rectangle(overlay, (0, 0), (w, 120), (0, 0, 0), -1)       # header bar
    frame = cv2.addWeighted(overlay, 0.55, frame, 0.45, 0)

    label = top if top else "—"
    cv2.putText(frame, f"Letter: {label}  (stability {count}/{COMMIT_STREAK})",
                (16, 64), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (255, 255, 255), 2)

    pct = int((w - 32) * (min(1.0, count/COMMIT_STREAK) if top else 0.0))
    cv2.rectangle(frame, (16, 74), (16 + pct, 84), (255, 255, 255), -1)

    word_str = "".join(current_word)
    sent_str = " ".join(sentence[-12:]) if sentence else ""
    cv2.putText(frame, f"Word: {word_str} | Sentence: {sent_str}",
                (16, 104), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 1)

    return frame

# ================== Gradio UI ==================
THEME = gr.themes.Soft(
    primary_hue="blue",
    neutral_hue="slate"
)

CSS = """
/* container width and centering */
#app {max-width: 980px; margin: 0 auto;}
/* sticky title */
.sticky-head {position: sticky; top: 0; z-index: 5; backdrop-filter: blur(6px);}
.sticky-head > * {margin: 0 !important;}
/* webcam panel */
.webrtc-wrap {display:flex; justify-content:center;}
.webrtc-wrap .wrap {width: 100%;}
/* shrink the help chart */
.help-chart img {max-height: 360px; width: auto;}
/* tighter accordions */
.gr-accordion .label {font-weight: 600;}
/* mobile niceties */
@media (max-width: 768px){
  .help-chart img {max-height: 240px;}
}
"""

with gr.Blocks(title="SpellTalk — ASL fingerspelling (live)", theme=THEME, css=CSS, elem_id="app") as demo:
    with gr.Column(elem_classes="sticky-head"):
        gr.Markdown("## 🖐️ SpellTalk – ASL fingerspelling → words (live stream)")
        gr.Markdown(
            "Click **Start camera** below. Hold one hand in frame, shoulder-high, palm per the chart. "
            "When a letter is stable, it locks into the **Word**. Pause briefly to end a word."
        )

    with gr.Row():
        with gr.Column(scale=3):
            with gr.Group(elem_classes="webrtc-wrap"):
                webrtc = WebRTC(label="Webcam (live)", mode="send-receive", modality="video")
            with gr.Row():
                clear_word = gr.Button("Clear word")
                clear_sent = gr.Button("Clear sentence")

            with gr.Accordion("Settings", open=False):
                th = gr.Slider(0.05, 0.9, value=CONF_THRES, step=0.05, label="Confidence threshold")
                hist = gr.Slider(3, 21, value=HIST_LEN, step=1, label="History length (frames)")
                streak = gr.Slider(2, 12, value=COMMIT_STREAK, step=1, label="Stability to lock (frames)")
                pause = gr.Slider(0.3, 2.5, value=WORD_BREAK_SECONDS, step=0.1, label="Pause → new word (seconds)")

        with gr.Column(scale=2):
            with gr.Group():
                gr.Markdown("**ASL alphabet (reference)**")
                gr.Image(value=HELP_CHART_URL, show_label=False, elem_classes="help-chart", interactive=False)
            gr.Markdown(
                "Tips:\n"
                "- Good, even lighting; plain background.\n"
                "- Keep hand within the frame; avoid fast motion.\n"
                "- Letters like **J**/**Z** need motion; this model is tuned mostly for A–Y."
            )

    # Wire up runtime updates for global settings
    def _set_conf(v):
        global CONF_THRES; CONF_THRES = float(v)
    def _set_hist(v):
        global HIST_LEN, history; HIST_LEN = int(v); history = deque(history, maxlen=HIST_LEN)
    def _set_streak(v):
        global COMMIT_STREAK; COMMIT_STREAK = int(v)
    def _set_pause(v):
        global WORD_BREAK_SECONDS; WORD_BREAK_SECONDS = float(v)

    th.change(_set_conf, th, None)
    hist.change(_set_hist, hist, None)
    streak.change(_set_streak, streak, None)
    pause.change(_set_pause, pause, None)

    clear_word.click(lambda: reset_word(), None, None)
    clear_sent.click(lambda: reset_sentence(), None, None)

    # Stream hookup (10 min per session keeps Spaces happy)
    webrtc.stream(fn=process_frame, inputs=[webrtc], outputs=[webrtc], time_limit=600)

if __name__ == "__main__":
    demo.queue()
    demo.launch()