Update app.py

app.py

@@ -1,5 +1,30 @@
 import gradio as gr
 from huggingface_hub import InferenceClient
+import numpy as np
+from PIL import Image
+import tflite_runtime.interpreter as tflite
+
+# Load TFLite model at startup
+interpreter = tflite.Interpreter(model_path="model.tflite")
+interpreter.allocate_tensors()
+input_details = interpreter.get_input_details()
+output_details = interpreter.get_output_details()
+
+# Load labels
+with open("labels.txt", "r") as f:
+    labels = [line.strip().split(" ", 1)[-1] for line in f.readlines()]
+
+def predict_asl(frame):
+    if frame is None:
+        return "–"
+    img = Image.fromarray(frame).resize((224, 224))
+    img_array = np.array(img, dtype=np.float32) / 255.0
+    img_array = np.expand_dims(img_array, axis=0)
+    interpreter.set_tensor(input_details[0]['index'], img_array)
+    interpreter.invoke()
+    predictions = interpreter.get_tensor(output_details[0]['index'])[0]
+    best_idx = int(np.argmax(predictions))
+    return labels[best_idx]
 
 spotify_embed = """
 <iframe data-testid="embed-iframe" 
@@ -17,194 +42,6 @@ picture-in-picture"
 loading="lazy"></iframe>
 """
 
-
-asl_html = """
-<div id="asl-wrapper" allow="camera; microphone" style="
-    background:#fff0f6;
-    border:1.5px solid #f3c4d7;
-    border-radius:16px;
-    padding:18px 20px 14px;
-    margin-bottom:18px;
-    font-family:sans-serif;
-">
-  <div style="font-weight:700;color:#9d4edd;font-size:1.05rem;margin-bottom:10px;">
-     ASL Letter Signer
-  </div>
-
-  <div style="display:flex;gap:16px;align-items:flex-start;flex-wrap:wrap;">
-    <div>
-      <div id="asl-webcam" style="
-    width:240px;
-    height:180px;
-    border-radius:10px;
-    background:#000;
-    border:2px solid #f8c8dc;
-    overflow:hidden;
-"></div>
-      <canvas id="asl-canvas" width="240" height="180" style="display:none;"></canvas>
-    </div>
-
-    <div style="flex:1;min-width:180px;">
-      <div style="font-size:0.82rem;color:#7b2cbf;margin-bottom:4px;">Detected letter</div>
-      <div id="asl-letter" style="
-          font-size:3rem;font-weight:900;color:#9d4edd;
-          background:#fff;border-radius:10px;border:1.5px solid #e0c3fc;
-          width:64px;height:64px;display:flex;align-items:center;justify-content:center;
-          margin-bottom:8px;">–</div>
-
-      <div style="font-size:0.78rem;color:#7b2cbf;margin-bottom:3px;">Confidence</div>
-      <div style="background:#f3c4d7;border-radius:8px;height:10px;width:100%;max-width:200px;margin-bottom:12px;">
-        <div id="asl-conf-bar" style="height:10px;border-radius:8px;background:#ff8fab;width:0%;transition:width 0.2s;"></div>
-      </div>
-
-      <div style="font-size:0.82rem;color:#7b2cbf;margin-bottom:3px;">Current word</div>
-      <div id="asl-word" style="
-          font-size:1.4rem;font-weight:700;color:#5c5470;
-          background:#fff;border:1.5px solid #f3c4d7;border-radius:8px;
-          padding:4px 10px;min-width:120px;min-height:36px;
-          letter-spacing:3px;margin-bottom:12px;"></div>
-
-      <div style="display:flex;gap:8px;flex-wrap:wrap;">
-        <button onclick="aslAddLetter()" style="background:#ffb3c6;color:#fff;border:none;border-radius:8px;padding:7px 14px;font-weight:600;cursor:pointer;">
-          Add Letter
-        </button>
-        <button onclick="aslAddSpace()" style="background:#e0c3fc;color:#4a4a4a;border:none;border-radius:8px;padding:7px 14px;font-weight:600;cursor:pointer;">
-          Space
-        </button>
-        <button onclick="aslSendWord()" style="background:#9d4edd;color:#fff;border:none;border-radius:8px;padding:7px 14px;font-weight:600;cursor:pointer;">
-          Send
-        </button>
-        <button onclick="aslClear()" style="background:#fff;color:#c77dff;border:1.5px solid #e0c3fc;border-radius:8px;padding:7px 14px;font-weight:600;cursor:pointer;">
-          Clear
-        </button>
-      </div>
-
-      <div style="font-size:0.74rem;color:#cdb4db;margin-top:8px;">
-        Tip: press Space to add letter, Enter to send word
-      </div>
-    </div>
-  </div>
-
-  <div id="asl-status" style="margin-top:10px;font-size:0.8rem;color:#c77dff;">
-    Click Start Camera to begin.
-  </div>
-
-  <button onclick="initASL()" style="margin-top:8px;background:#9d4edd;color:#fff;border:none;border-radius:8px;padding:8px 18px;">
-    📷 Start Camera
-  </button>
-</div>
-
-<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@latest/dist/tf.min.js"></script>
-<script src="https://cdn.jsdelivr.net/npm/@teachablemachine/image@latest/dist/teachablemachine-image.min.js"></script>
-
-<script>
-const MODEL_URL = "https://teachablemachine.withgoogle.com/models/4aHXkhLXo/";
-
-let tmModel, webcamObj;
-let currentLetter = "–";
-let currentWord = "";
-const videoContainer = document.getElementById("asl-webcam"); #new change, if breaks remove
-async function initASL() {
-  const status = document.getElementById("asl-status");
-  status.textContent = "Loading model...";
-
-  const modelURL = MODEL_URL + "model.json";
-  const metaURL = MODEL_URL + "metadata.json";
-
-  tmModel = await tmImage.load(modelURL, metaURL);
-
-  status.textContent = "Starting webcam...";
-
-  webcamObj = new tmImage.Webcam(240, 180, true);
-
-  await webcamObj.setup();   // must come first
-  await webcamObj.play();    
-
-  
-  if (!videoContainer) {
-  console.error("ASL container missing");
-  return;
-}
-
-  
-  videoContainer.innerHTML = "";
-
-  
-  await webcamObj.setup(); #new line
-  await webcamObj.play(); #new code
-
-  document.getElementById("asl-webcam").appendChild(webcamObj.canvas); #new code
-  webcamObj.canvas.style.width = "240px";
-  webcamObj.canvas.style.height = "180px";
-  webcamObj.canvas.style.borderRadius = "10px";
-
-  status.textContent = "Ready — start signing";
-
-  loop();
-}
-
-async function loop() {
-  webcamObj.update();
-  const preds = await tmModel.predict(webcamObj.canvas);
-
-  let best = preds.reduce((a,b) => a.probability > b.probability ? a : b);
-
-  currentLetter = best.className;
-
-  document.getElementById("asl-letter").textContent = currentLetter;
-  document.getElementById("asl-conf-bar").style.width =
-    Math.round(best.probability * 100) + "%";
-
-  requestAnimationFrame(loop);
-}
-
-function aslAddLetter() {
-  currentWord += currentLetter;
-  document.getElementById("asl-word").textContent = currentWord;
-}
-
-function aslAddSpace() {
-  currentWord += " ";
-  document.getElementById("asl-word").textContent = currentWord;
-}
-
-function aslClear() {
-  currentWord = "";
-  document.getElementById("asl-word").textContent = "";
-}
-
-function aslSendWord() {
-  const inputs = document.querySelectorAll("textarea");
-  let chatInput = inputs[inputs.length - 1];
-
-  const setter = Object.getOwnPropertyDescriptor(
-    window.HTMLTextAreaElement.prototype,
-    'value'
-  ).set;
-
-  setter.call(chatInput, currentWord);
-  chatInput.dispatchEvent(new Event('input', { bubbles: true }));
-
-  setTimeout(() => {
-    const btns = document.querySelectorAll("button");
-    for (let b of btns) {
-      if (b.textContent.toLowerCase().includes("submit") || b.querySelector("svg")) {
-        b.click();
-        break;
-      }
-    }
-  }, 100);
-
-  aslClear();
-}
-
-document.addEventListener("keydown", (e) => {
-  if (e.code === "Space") aslAddLetter();
-  if (e.code === "Enter") aslSendWord();
-});
-</script>
-"""
-
 theme = gr.themes.Soft().set(
     body_background_fill="#fff7fb",
     block_background_fill="#ffffffcc",
@@ -240,7 +77,6 @@ SYSTEM_MESSAGES = {
 }
 
 def respond(message, history, mode):
-
     if mode is None:
         yield "Please select a mode first.", mode
         return
@@ -270,6 +106,20 @@ def respond(message, history, mode):
         yield response, mode
 
 
+def add_letter(letter, word):
+    if letter and letter != "–":
+        return word + letter
+    return word
+
+def add_space(word):
+    return word + " "
+
+def clear_word():
+    return ""
+
+def send_word(word):
+    return word, ""
+
 
 with gr.Blocks(theme=theme) as demo:
 
@@ -300,7 +150,46 @@ with gr.Blocks(theme=theme) as demo:
     )
 
     gr.Markdown("### 🤟 ASL Input")
-    gr.HTML(asl_html)
+    with gr.Group():
+        gr.Markdown("*Sign a letter in front of your camera, then click Add Letter to build a word and Send to chat.*")
+        with gr.Row():
+            with gr.Column(scale=1):
+                asl_cam = gr.Image(
+                    sources=["webcam"],
+                    streaming=True,
+                    label="Camera",
+                    mirror_webcam=True,
+                    height=300
+                )
+            with gr.Column(scale=1):
+                asl_detected = gr.Textbox(label="Detected Letter", interactive=False, value="–")
+                asl_word_box = gr.Textbox(label="Current Word", interactive=False, value="")
+                with gr.Row():
+                    asl_add_btn = gr.Button("Add Letter", variant="primary")
+                    asl_space_btn = gr.Button("Space", variant="secondary")
+                    asl_clear_btn = gr.Button("Clear", variant="secondary")
+                asl_send_btn = gr.Button("Send to Chat ➤", variant="primary")
+
+    word_state = gr.State("")
+
+    asl_cam.stream(
+        fn=predict_asl,
+        inputs=[asl_cam],
+        outputs=[asl_detected]
+    )
+
+    asl_add_btn.click(fn=add_letter, inputs=[asl_detected, word_state], outputs=[word_state]).then(
+        fn=lambda w: w, inputs=[word_state], outputs=[asl_word_box]
+    )
+    asl_space_btn.click(fn=add_space, inputs=[word_state], outputs=[word_state]).then(
+        fn=lambda w: w, inputs=[word_state], outputs=[asl_word_box]
+    )
+    asl_clear_btn.click(fn=clear_word, outputs=[word_state]).then(
+        fn=lambda w: w, inputs=[word_state], outputs=[asl_word_box]
+    )
+    asl_send_btn.click(fn=send_word, inputs=[word_state], outputs=[chatbot.textbox, word_state]).then(
+        fn=lambda w: w, inputs=[word_state], outputs=[asl_word_box]
+    )
 
     gr.Markdown("### 🎵 Music")
     gr.HTML(spotify_embed)