add app file

glossController.py

@@ -0,0 +1,19 @@
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
+import torch
+
+def translateGloss(gloss: str, model_id: str = "rrrr66254/Glossa-BART") -> str:
+    
+    tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
+    model = AutoModelForSeq2SeqLM.from_pretrained(model_id, trust_remote_code=True)
+    model.eval()
+    if torch.cuda.is_available():
+        model = model.to("cuda")
+    
+    inputs = tokenizer(gloss, return_tensors="pt", padding=True, truncation=True)
+    if torch.cuda.is_available():
+        inputs = {k: v.to("cuda") for k,v in inputs.items()}
+    
+    outputs = model.generate(**inputs, max_new_tokens=50, do_sample=False)
+    result = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    return result
+

lettersController.py

@@ -0,0 +1,141 @@
+import cv2
+import numpy as np
+import pickle
+import tensorflow as tf
+import mediapipe as mp
+
+lettersModel = tf.keras.models.load_model('ai_model/models/detectLettersModel.keras')
+with open('ai_model/models/labelEncoder.pickle', 'rb') as f:
+    labelEncoder = pickle.load(f)
+
+lettersModel2 = tf.keras.models.load_model('ai_model/jz_model/JZModel.keras')
+with open('ai_model/jz_model/labelEncoder.pickle', 'rb') as f:
+    labelEncoder2 = pickle.load(f)
+
+numbersModel = tf.keras.models.load_model('ai_model/models/detectNumbersModel.keras')
+with open('ai_model/models/numLabelEncoder.pickle', 'rb') as f:
+    numLabelEncoder = pickle.load(f)
+
+sequenceNum = 20
+hands = mp.solutions.hands.Hands(static_image_mode=True)
+
+def detectFromImage(sequenceList):
+
+    if len(sequenceList) != sequenceNum:
+        return {'letter': '', 'confidence': 0.0}
+
+    processedSequence = []
+
+    for imagePath in sequenceList:
+        image = cv2.imread(imagePath)
+        if image is None:
+            continue 
+
+        imgRGB = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+        results = hands.process(imgRGB)
+
+        if not results.multi_hand_landmarks:
+            continue  
+
+        handLandmarks = results.multi_hand_landmarks[0]  
+
+        xList, yList = [], []
+        dataAux2 = []
+
+        for lm in handLandmarks.landmark:
+            xList.append(lm.x)
+            yList.append(lm.y)
+
+        for lm in handLandmarks.landmark:
+            dataAux2.append(lm.x - min(xList))
+            dataAux2.append(lm.y - min(yList))
+            dataAux2.append(0) 
+
+        processedSequence.append(dataAux2)
+
+    confidence2 = 0.0
+    label2 = ""
+    fallback_frame = cv2.imread(sequenceList[-1])  
+
+    # for i in range(len(processedSequence)):
+    #     if processedSequence[i] is None:
+    #         prevIdx, nextIdx = -1, -1
+            
+    #         for j in range(i - 1, -1, -1):
+    #             if processedSequence[j] is not None:
+    #                 prevIdx = j
+    #                 break
+            
+    #         for j in range(i + 1, len(processedSequence)):
+    #             if processedSequence[j] is not None:
+    #                 nextIdx = j
+    #                 break
+
+    #         if prevIdx != -1 and nextIdx != -1:
+    #             prevData = np.array(processedSequence[prevIdx])
+    #             nextData = np.array(processedSequence[nextIdx])
+    #             t = (i - prevIdx) / (nextIdx - prevIdx)
+    #             interpolatedData = prevData + (nextData - prevData) * t
+    #             processedSequence[i] = interpolatedData.tolist()
+    #         elif prevIdx != -1:
+    #             processedSequence[i] = processedSequence[prevIdx]
+    #         elif nextIdx != -1:
+    #             processedSequence[i] = processedSequence[nextIdx]
+
+    if len(processedSequence) != sequenceNum:
+        print("incomplete sequence: ", len(processedSequence))
+        return {'letter': '', 'confidenceLetter': 0.0, 'number': '', 'confidenceNumber': 0.0}
+       
+    inputData2 = np.array(processedSequence, dtype=np.float32).reshape(1, sequenceNum, 63)
+    prediction2 = lettersModel2.predict(inputData2, verbose=0)
+
+    index2 = np.argmax(prediction2, axis=1)[0]
+    confidence2 = float(np.max(prediction2))
+    label2 = labelEncoder2.inverse_transform([index2])[0]
+    print(f'Letters Model 2:{label2} at {confidence2}')
+
+    if fallback_frame is not None:
+        imgRGB = cv2.cvtColor(fallback_frame, cv2.COLOR_BGR2RGB)
+        results = hands.process(imgRGB)
+        if results.multi_hand_landmarks:
+            handLandmarks = results.multi_hand_landmarks[0]
+            xList, yList = [], []
+            dataAux = []
+
+            for lm in handLandmarks.landmark:
+                xList.append(lm.x)
+                yList.append(lm.y)
+
+            for lm in handLandmarks.landmark:
+                dataAux.append(lm.x - min(xList))
+                dataAux.append(lm.y - min(yList))
+
+            #check in letters model1
+            inputData1 = np.array(dataAux, dtype=np.float32).reshape(1, 42, 1)
+            prediction1 = lettersModel.predict(inputData1, verbose=0)
+            index1 = np.argmax(prediction1, axis=1)[0]
+            confidence1 = float(np.max(prediction1))
+            label1 = labelEncoder.inverse_transform([index1])[0]
+
+            print(f'Letters Model 1: {label1} at {confidence1}')
+
+            prediction3 = numbersModel.predict(inputData1, verbose=0)
+            index3 = np.argmax(prediction3, axis=1)[0]
+            confidence3 = float(np.max(prediction3))
+            label3 = numLabelEncoder.inverse_transform([index3])[0]
+
+            print(f'Numbers Model: {label3} at {confidence3}')
+
+            if label1==label2:
+                return {'letter': label2, 'confidenceLetter': confidence2,
+                        'number': label3, 'confidenceNumber': confidence3}
+            # elif label2=="Z" and label1=="L":
+            #     return {'letter': label2, 'confidence': confidence2}
+            # elif label2=="J" and label1=="I":
+            #     return {'letter': label2, 'confidence': confidence2}
+            else:
+                return {'letter': label1, 'confidenceLetter': confidence1
+                        , 'number': label3, 'confidenceNumber': confidence3}        
+    else:   
+        return {'letter': label2, 'confidenceLetter': confidence2
+                , 'number': '', 'confidenceNumber': 0.0}

requirements.txt

@@ -0,0 +1,17 @@
+fastapi==0.116.1
+matplotlib==3.10.6
+mediapipe==0.10.21
+numpy>=1.24.3,<2
+openai==1.106.1
+opencv_contrib_python==4.11.0.86
+opencv_python==4.11.0.86
+opencv_python_headless==4.11.0.86
+pandas==2.3.2
+python-dotenv==1.1.1
+scikit_learn==1.7.1
+starlette==0.47.3
+tensorflow==2.19.0
+tqdm==4.67.1
+uvicorn==0.35.0
+transformers
+torch

runtime.txt

@@ -0,0 +1 @@
+python-3.11.0

upload.py

@@ -0,0 +1,97 @@
+import os
+import sys
+import uvicorn
+from fastapi import FastAPI, UploadFile, File, Form
+from typing import Optional
+from starlette.responses import JSONResponse
+
+# --- Import AI Functions ---
+# Now that Detection.py is in the same directory, the import is straightforward.
+try:
+    from models.Detection import load_model_and_assets, process_image_and_predict, process_video_and_predict_realtime, MIN_CONFIDENCE_THRESHOLD
+    print("Successfully imported functions from Detection.py")
+except ImportError as e:
+    print(f"ERROR: Could not import from Detection.py. Please ensure the file is in the same repository as app.py: {e}")
+    # It's good practice to exit if a critical import fails
+    sys.exit(1)
+
+# --- FastAPI App Initialization ---
+app = FastAPI()
+
+# --- Load AI Model and Assets on Startup ---
+# This function, located in your Detection.py, should be updated to correctly
+# reference the files within the 'models' subfolder of your Hugging Face Space.
+print("\n--- Hugging Face Space starting: Loading AI model and assets... ---")
+load_model_and_assets()
+print("--- AI model and assets loaded. Ready to serve predictions. ---\n")
+
+# --- FastAPI Routes ---
+# The rest of the routes remain the same, as they now correctly call the functions
+# from your Detection.py script.
+
+@app.post("/process-image")
+async def process_image_api(
+    image: UploadFile = File(...),
+    min_confidence: Optional[float] = Form(MIN_CONFIDENCE_THRESHOLD)
+):
+    try:
+        contents = await image.read()
+        temp_filepath = f"/tmp/{image.filename}"
+        with open(temp_filepath, "wb") as f:
+            f.write(contents)
+        
+        action, confidence = process_image_and_predict(temp_filepath, min_confidence)
+        os.remove(temp_filepath)
+
+        response = {
+            "sign": action if action else "UNKNOWN",
+            "confidence": round(float(confidence), 2),
+            "success": True,
+            "filename": image.filename
+        }
+        return JSONResponse(content=response)
+    
+    except Exception as e:
+        return JSONResponse(
+            status_code=500,
+            content={
+                "error": "Error processing image with AI model",
+                "details": str(e),
+                "success": False
+            }
+        )
+
+@app.post("/process-video")
+async def process_video_api(
+    video: UploadFile = File(...),
+    min_confidence: Optional[float] = Form(MIN_CONFIDENCE_THRESHOLD)
+):
+    try:
+        contents = await video.read()
+        temp_filepath = f"/tmp/{video.filename}"
+        with open(temp_filepath, "wb") as f:
+            f.write(contents)
+            
+        action, confidence = process_video_and_predict_realtime(temp_filepath, min_confidence)
+        os.remove(temp_filepath)
+        
+        response = {
+            "phrase": action if action else "UNKNOWN",
+            "confidence": round(float(confidence), 2),
+            "success": True,
+            "filename": video.filename
+        }
+        return JSONResponse(content=response)
+    except Exception as e:
+        return JSONResponse(
+            status_code=500,
+            content={
+                "error": "Error processing video with AI model",
+                "details": str(e),
+                "success": False
+            }
+        )
+
+#just added
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=7860)

wordsController.py

@@ -0,0 +1,141 @@
+import cv2
+import numpy as np
+import pandas as pd
+from tensorflow.keras.models import load_model
+import mediapipe as mp
+
+MODEL_PATH = 'ai_model/words/saved_models/best_sign_classifier_model_125_words_seq90.keras'
+CSV_PATH = 'ai_model/words/wlasl_125_words_personal_final_processed_data_augmented_seq90.csv'
+SEQUENCE_LENGTH = 90
+EXPECTED_COORDS_PER_FRAME = 1662
+CONFIDENCE_THRESHOLD = 0.1
+
+model = load_model(MODEL_PATH)
+df = pd.read_csv(CSV_PATH)
+unique_glosses = df['gloss'].unique()
+id_to_gloss = {i: g for i, g in enumerate(unique_glosses)}
+
+mp_holistic = mp.solutions.holistic.Holistic(
+    static_image_mode=True,
+    model_complexity=1,
+    min_detection_confidence=0.2,
+    min_tracking_confidence=0.5
+)
+
+NUM_POSE_COORDS_SINGLE = 33*4
+NUM_HAND_COORDS_SINGLE = 21*3
+NUM_FACE_COORDS_SINGLE = 468*3
+
+def normalize_landmarks(landmarks_sequence):
+    if landmarks_sequence.ndim == 1:
+        landmarks_sequence = np.expand_dims(landmarks_sequence, axis=0)
+
+    normalized_sequences = []
+    for frame_landmarks in landmarks_sequence:
+        if np.all(frame_landmarks == 0):
+            normalized_sequences.append(np.zeros(EXPECTED_COORDS_PER_FRAME, dtype=np.float32))
+            continue
+
+        pose_coords_flat = frame_landmarks[0 : NUM_POSE_COORDS_SINGLE]
+        left_hand_coords_flat = frame_landmarks[NUM_POSE_COORDS_SINGLE : NUM_POSE_COORDS_SINGLE + NUM_HAND_COORDS_SINGLE]
+        right_hand_coords_flat = frame_landmarks[NUM_POSE_COORDS_SINGLE + NUM_HAND_COORDS_SINGLE : NUM_POSE_COORDS_SINGLE + NUM_HAND_COORDS_SINGLE*2]
+        face_coords_flat = frame_landmarks[NUM_POSE_COORDS_SINGLE + NUM_HAND_COORDS_SINGLE*2 : ]
+
+        all_parts_data = [
+            (pose_coords_flat, 4, [0.0]*NUM_POSE_COORDS_SINGLE),
+            (left_hand_coords_flat, 3, [0.0]*NUM_HAND_COORDS_SINGLE),
+            (right_hand_coords_flat, 3, [0.0]*NUM_HAND_COORDS_SINGLE),
+            (face_coords_flat, 3, [0.0]*NUM_FACE_COORDS_SINGLE)
+        ]
+
+        normalized_frame_parts = []
+        for flat_lms, coords_per_lm, template in all_parts_data:
+            if np.all(flat_lms==0):
+                normalized_frame_parts.append(np.array(template, dtype=np.float32))
+                continue
+
+            lms_array = flat_lms.reshape(-1, coords_per_lm)
+            coords_for_mean = lms_array[:, :3] if coords_per_lm==4 else lms_array
+            mean_coords = np.mean(coords_for_mean, axis=0)
+            translated_lms = lms_array.copy()
+            translated_lms[:, :3] -= mean_coords
+            scale_factor = np.max(np.linalg.norm(translated_lms[:, :3], axis=1))
+            if scale_factor > 1e-6:
+                translated_lms[:, :3] /= scale_factor
+            normalized_frame_parts.append(translated_lms.flatten())
+
+        combined_frame = np.concatenate(normalized_frame_parts).astype(np.float32)
+        if len(combined_frame) < EXPECTED_COORDS_PER_FRAME:
+            combined_frame = np.pad(combined_frame, (0, EXPECTED_COORDS_PER_FRAME - len(combined_frame)), 'constant')
+        elif len(combined_frame) > EXPECTED_COORDS_PER_FRAME:
+            combined_frame = combined_frame[:EXPECTED_COORDS_PER_FRAME]
+
+        normalized_sequences.append(combined_frame)
+
+    return np.array(normalized_sequences, dtype=np.float32)
+
+def pad_or_truncate_sequence(sequence, target_length, feature_dimension):
+    if sequence.shape[0] < target_length:
+        padding = np.zeros((target_length - sequence.shape[0], feature_dimension), dtype=np.float32)
+        return np.vstack((sequence, padding))
+    return sequence[:target_length, :]
+
+def detectWords(image_paths):
+    results_dict = {}
+    sequence = []
+
+    for idx, path in enumerate(image_paths):
+        img = cv2.imread(path)
+        if img is None:
+            print(f"Warning: Could not read image {path}")
+            continue
+
+        img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+        mp_results = mp_holistic.process(img_rgb)
+
+        frame_lms = np.zeros(EXPECTED_COORDS_PER_FRAME, dtype=np.float32)
+        current_idx = 0
+
+        if mp_results.pose_landmarks:
+            pose_flat = [coord for lm in mp_results.pose_landmarks.landmark for coord in [lm.x, lm.y, lm.z, lm.visibility]]
+            frame_lms[current_idx:current_idx+len(pose_flat)] = pose_flat
+        else: 
+            print(f"Warning: No pose landmarks detected in frame {idx}")
+        current_idx += NUM_POSE_COORDS_SINGLE
+
+        if mp_results.left_hand_landmarks:
+            lh_flat = [coord for lm in mp_results.left_hand_landmarks.landmark for coord in [lm.x, lm.y, lm.z]]
+            frame_lms[current_idx:current_idx+len(lh_flat)] = lh_flat
+        else: 
+            print(f"Warning: No left hand landmarks detected in frame {idx}")
+        current_idx += NUM_HAND_COORDS_SINGLE
+
+        if mp_results.right_hand_landmarks:
+            rh_flat = [coord for lm in mp_results.right_hand_landmarks.landmark for coord in [lm.x, lm.y, lm.z]]
+            frame_lms[current_idx:current_idx+len(rh_flat)] = rh_flat
+        else: 
+            print(f"Warning: No right hand landmarks detected in frame {idx}")
+        current_idx += NUM_HAND_COORDS_SINGLE
+
+        if mp_results.face_landmarks:
+            face_flat = [coord for lm in mp_results.face_landmarks.landmark for coord in [lm.x, lm.y, lm.z]]
+            frame_lms[current_idx:current_idx+len(face_flat)] = face_flat
+        else: 
+            print(f"Warning: No pose landmarks detected in frame {idx}")
+
+        sequence.append(frame_lms)
+
+    sequence = normalize_landmarks(np.array(sequence, dtype=np.float32))
+    sequence = pad_or_truncate_sequence(sequence, SEQUENCE_LENGTH, EXPECTED_COORDS_PER_FRAME)
+    sequence = np.expand_dims(sequence, axis=0) 
+
+    preds = model.predict(sequence, verbose=0)
+    predicted_id = int(np.argmax(preds))
+    confidence = float(np.max(preds))
+    predicted_word = id_to_gloss.get(predicted_id, "Unknown")
+
+    result = {"word": predicted_word if confidence >= CONFIDENCE_THRESHOLD else "",
+              "confidence": confidence}
+    
+    print(f"Prediction result: {result}")
+    return result