Update app.py

app.py

@@ -1,312 +1,196 @@
 import cv2
 import base64
 import numpy as np
-import io
-from flask import Flask, render_template, Response, request, jsonify
-from flask_socketio import SocketIO, emit
-from PIL import Image
-from time import time as unix_time
-import os
-import mediapipe as mp
-from mediapipe.tasks import python
-from mediapipe.tasks.python import vision
+from flask import Flask, render_template, request, jsonify, send_from_directory
 import time
-import argparse
+import mediapipe as mp
 from mediapipe.framework.formats import landmark_pb2
 from mediapipe import solutions
 from tflite_support.task import vision as vision2
-from tflite_support.task import core
-from tflite_support.task import processor
+from tflite_support.task import core, processor
 from numpy.linalg import norm
 
-#Image Annotation Utils
-char_list=[]
-global letter_result
-letter_result = 0
-global old_letter_result
-old_letter_result = 0
-MARGIN = 10  # pixels
-FONT_SIZE = 1
-FONT_THICKNESS = 1
-HANDEDNESS_TEXT_COLOR = (88, 205, 54) # vibrant green
-global test_x
-global test_y
-global result_to_show
-result_to_show=0
-global cresult_to_show
-cresult_to_show=0
-text_x = 0
-text_y = 0
-cwhich=0
-lastwidth = 400
-letterscore=0
-frame_time=0
-same_letter_time=0
-no_hand_flag=1
-# UTILS
-
-
-def brightness(img):
-    if len(img.shape) == 3:
-        # Colored RGB or BGR (*Do Not* use HSV images with this function)
-        # create brightness with euclidean norm
-        return np.average(norm(img, axis=2)) / np.sqrt(3)
-    else:
-        # Grayscale
-        return np.average(img)
-
-
-def draw_landmarks_on_image(rgb_image, detection_result):
-  hand_landmarks_list = detection_result.hand_landmarks
-  handedness_list = detection_result.handedness
-  annotated_image = np.copy(rgb_image)
-  crop = []
-  image_height, image_width, image_heightgray=annotated_image.shape
-
-
-  # Loop through the detected hands to visualize.
-  for idx in range(len(hand_landmarks_list)):
-    hand_landmarks = hand_landmarks_list[idx]
-    handedness = handedness_list[idx]
-
-    # Draw the hand landmarks.
-    hand_landmarks_proto = landmark_pb2.NormalizedLandmarkList()
-    hand_landmarks_proto.landmark.extend([
-      landmark_pb2.NormalizedLandmark(x=landmark.x, y=landmark.y, z=landmark.z) for landmark in hand_landmarks
-    ])
-    solutions.drawing_utils.draw_landmarks(
-      annotated_image,
-      hand_landmarks_proto,
-      solutions.hands.HAND_CONNECTIONS,
-      solutions.drawing_styles.get_default_hand_landmarks_style(),
-      solutions.drawing_styles.get_default_hand_connections_style())
-
-    # Get bounding box
-
-    height, width, _ = annotated_image.shape
-    
-    x_coordinates = [landmark.x for landmark in hand_landmarks]
-    y_coordinates = [landmark.y for landmark in hand_landmarks]
-    
-    min_x = int(min(x_coordinates) * width)  # Left
-    min_y = int(min(y_coordinates) * height)    # Top
-    max_x = int(max(x_coordinates) * width)  # Right
-    max_y = int(max(y_coordinates) * height) # Bottom
-
-    #Get dimensions of bounding box
-    sect_height = max_y-(min_y)
-    sect_width = max_x-(min_x)
-
-    #Get center of bounding box
-    center_x=(min_x+max_x)/2
-    center_y=(min_y+max_y)/2
-
-    sect_diameter=50
-    #Define dominant axis for aspect ratio
-
-    if(sect_height>sect_width):
-        sect_diameter = sect_height
-
-    if(sect_height<sect_width):
-        sect_diameter = sect_width
-
-    sect_diameter=sect_diameter+50 # Pad diameter
-    sect_radius=int(sect_diameter/2) # Find radius
-    
-    #Crop Image
-    crop_top=int(center_y-sect_radius) #Top boundry
-    crop_bottom=int(center_y+sect_radius) #Bottom boundry
-    crop_left=int(center_x-sect_radius) #Left boundry
-    crop_right=int(center_x+sect_radius) #Right boundry
-
-    #Account for out of canvas
-    if(crop_top<0): #Bounding box too high
-        crop_top=0
-
-    if(crop_left<0): #Bounding box too far left
-        crop_left=0
-
-    if(crop_right>image_width): #Bounding box too far right
-        crop_right=image_width
-
-    if(crop_bottom>image_height): #Bounding box too low
-        crop_bottom=image_height
-
-    # Trace bounding box
-    annotated_image = cv2.rectangle(annotated_image, (crop_left, crop_top), (crop_right, crop_bottom), (255,0,0), 6)
-
-    global text_x
-    global text_y
-
-    # For text, currently not used
-    text_x=crop_left
-    text_y=crop_top
-    
-    # Get cropped image
-    crop = annotated_image[crop_top:crop_bottom, crop_left:crop_right] 
-
-    # Scale cropped image
-    h, w = crop.shape[0:2]
-    neww = 150
-    newh = int(neww*(h/w))
-    crop = cv2.resize(crop, (neww, newh))
-    
-    #annotated_image[0:0+crop.shape[0], 0:0+crop.shape[1]] = crop # Used for superimposition
-
-    #annotated_image=crop # Used for replacement
-    
-  return [annotated_image, crop]
-
-#-------------------------------------------------------------
+# Flask app setup
+app = Flask(__name__)
 
-# Letter List
-letter_list=["A","B","C","D","E","F","G","H","I","J","K","L","M","N","O","P","Q","R","S","T","U","V","W","X","Y","Z","#"]
+# Global variables for letter detection results
+letter_result = 0
+result_to_show = 0
+cresult_to_show = 0
+letterscore = 0
+no_hand_flag = 1
 
-# Initialise MediaPipe hand landmark detction
-RESULT = None
+# Initialize MediaPipe hand landmark detection
 BaseOptions = mp.tasks.BaseOptions
 HandLandmarker = mp.tasks.vision.HandLandmarker
 HandLandmarkerOptions = mp.tasks.vision.HandLandmarkerOptions
 HandLandmarkerResult = mp.tasks.vision.HandLandmarkerResult
 VisionRunningMode = mp.tasks.vision.RunningMode
 
-cbase_options = core.BaseOptions(file_name="./better_exported/model.tflite") # New tflite
-ccbase_options = core.BaseOptions(file_name="./exported/model.tflite") # Old tflite
-
-# Initialise ASL tflite model
+# Load your TFLite models (adjust paths if needed)
+cbase_options = core.BaseOptions(file_name="./exported/model.tflite")  # New model
+ccbase_options = core.BaseOptions(file_name="./exported/word.tflite")  # Old model or word model
 
 cclassification_options = processor.ClassificationOptions(max_results=1)
 coptions = vision2.ImageClassifierOptions(base_options=cbase_options, classification_options=cclassification_options)
 ccoptions = vision2.ImageClassifierOptions(base_options=ccbase_options, classification_options=cclassification_options)
+
 cclassifier = vision2.ImageClassifier.create_from_options(coptions)
 ccclassifier = vision2.ImageClassifier.create_from_options(ccoptions)
 
-
+# Callback to store MediaPipe detection results asynchronously
+RESULT = None
 
 def print_result(result: HandLandmarkerResult, output_image: mp.Image, timestamp_ms: int):
-
     global RESULT
-    RESULT=result
-
+    RESULT = result
 
 options = HandLandmarkerOptions(
     base_options=BaseOptions(model_asset_path='hand_landmarker.task'),
     running_mode=VisionRunningMode.LIVE_STREAM,
     result_callback=print_result)
 
+detector = mp.tasks.vision.HandLandmarker.create_from_options(options)
 
-detector = vision.HandLandmarker.create_from_options(options)
-video_frames=[]
-
-
-
-app = Flask(__name__)
+# Utility functions for image processing
+def data_uri_to_image(data_uri):
+    header, encoded = data_uri.split(',', 1)
+    decoded_data = base64.b64decode(encoded)
+    nparr = np.frombuffer(decoded_data, np.uint8)
+    image = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
+    return image 
 
+def image_to_data_uri(image):
+    _, buffer = cv2.imencode('.jpg', image)
+    image_bytes = buffer.tobytes()
+    base64_encoded = base64.b64encode(image_bytes).decode('utf-8')
+    return f"data:image/jpeg;base64,{base64_encoded}"
 
+def draw_landmarks_on_image(rgb_image, detection_result):
+    hand_landmarks_list = detection_result.hand_landmarks
+    annotated_image = np.copy(rgb_image)
+    image_height, image_width, _ = annotated_image.shape
+
+    for idx in range(len(hand_landmarks_list)):
+        hand_landmarks = hand_landmarks_list[idx]
+        hand_landmarks_proto = landmark_pb2.NormalizedLandmarkList()
+        hand_landmarks_proto.landmark.extend([
+            landmark_pb2.NormalizedLandmark(x=landmark.x, y=landmark.y, z=landmark.z) for landmark in hand_landmarks
+        ])
+        solutions.drawing_utils.draw_landmarks(
+            annotated_image,
+            hand_landmarks_proto,
+            solutions.hands.HAND_CONNECTIONS,
+            solutions.drawing_styles.get_default_hand_landmarks_style(),
+            solutions.drawing_styles.get_default_hand_connections_style()
+        )
+    return annotated_image
+
+# Letter list - modify if needed
+letter_list = [chr(i) for i in range(65, 91)] + ['#']  # A-Z + #
+
+# Isẹ̀kiri dictionary (Example mapping, update with real words)
+isekiri_dict = {
+    'A': 'Àṣẹ',
+    'B': 'Bí',
+    'C': 'Ṣe',
+    'D': 'Dá',
+    'E': 'Ẹ̀',
+    'F': 'Fẹ́',
+    'G': 'Gba',
+    'H': 'Hàn',
+    'I': 'Ìyà',
+    'J': 'Jẹ',
+    'K': 'Kọ',
+    'L': 'Lá',
+    'M': 'Má',
+    'N': 'Ná',
+    'O': 'Ọ̀',
+    'P': 'Pẹ̀',
+    'Q': 'Kù',  # approximate since Q rarely used
+    'R': 'Rà',
+    'S': 'Ṣá',
+    'T': 'Tẹ',
+    'U': 'Ú',
+    'V': 'Vẹ',
+    'W': 'Wá',
+    'X': 'Ẹ́s',
+    'Y': 'Yá',
+    'Z': 'Zà',
+    '#': '#'
+}
+
+# Routes for web UI and models
 @app.route('/')
 def index():
     return render_template('index.html')
 
+@app.route('/exported/<path:filename>')
+def send_model(filename):
+    return send_from_directory('exported', filename)
 
+# Video frame processing API (ASL detection)
 @app.route('/api/data', methods=['POST'])
 def handle_video_frame():
-    frame = request.json.get('key')
-    #print(request.json)
-    response_frame = data_uri_to_image(frame)
-    decimg = response_frame
-
-#--------------------------------------------
-    mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=decimg) # Create MediaPipe image
-    #print(mp.Timestamp.from_seconds(time.time()).value)
-    
-    detection_result = detector.detect_async(mp_image, mp.Timestamp.from_seconds(time.time()).value) # detct
-    
-    # Try-Catch block, because detection is not done during model initialisation
-    global no_hand_flag, frame_time, same_letter_time, letter_result, old_letter_result, char_list, letterscore
+    global letter_result, result_to_show, cresult_to_show, letterscore, no_hand_flag
 
-    try:
-        result_images = draw_landmarks_on_image(mp_image.numpy_view(), RESULT) # Array of annotated and cropped images
-        annotated_image = result_images[0] 
-        cropped_image = result_images[1]
+    frame_data_uri = request.json.get('key')
+    if not frame_data_uri:
+        return jsonify({'error': 'No frame data received'}), 400
 
-        #Standardise and fit shape by resizing
+    frame = data_uri_to_image(frame_data_uri)
+    mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=frame)
 
-        h, w = annotated_image.shape[0:2]
-        neww = 500
-        newh = int(neww*(h/w))
-        resized_image = cv2.resize(annotated_image, (neww, newh))
-        final_image=resized_image
+    try:
+        detection_result = detector.detect_async(mp_image, mp.Timestamp.from_seconds(time.time()).value)
 
-        if(RESULT.handedness != []): # To chack if there is any result at all and then feed tflite model
-            no_hand_flag=0
+        global RESULT
+        if RESULT is None:
+            return jsonify({'result': '_', 'frame': frame_data_uri})
 
+        annotated_image = draw_landmarks_on_image(frame, RESULT)
+        
+        if RESULT.handedness:
+            no_hand_flag = 0
+            # If right hand detected, classify using models
             if RESULT.handedness[0][0].display_name == 'Right':
-                tf_image = vision2.TensorImage.create_from_array(cropped_image)
-                classification_result = cclassifier.classify(tf_image) # New
-                cclassification_result = ccclassifier.classify(tf_image) # Old
+                tf_image = vision2.TensorImage.create_from_array(frame)
+                classification_result = cclassifier.classify(tf_image)
+                cclassification_result = ccclassifier.classify(tf_image)
                 
-                result_to_show = classification_result.classifications[0].categories[0].category_name # New
-                cresult_to_show = cclassification_result.classifications[0].categories[0].category_name # Old
-
+                result_to_show = classification_result.classifications[0].categories[0].category_name
+                cresult_to_show = cclassification_result.classifications[0].categories[0].category_name
+                
+                # Simple decision logic between old and new models
                 if cclassification_result.classifications[0].categories[0].score > classification_result.classifications[0].categories[0].score:
-                    letter_result = cresult_to_show # To implement further UX with Text to Speech
-                    cwhich="Old"
-                    if result_to_show == "P" and cresult_to_show !="P":
-                        cwhich="New"
-                        letter_result = result_to_show
-                    
-                    
+                    letter_result = cresult_to_show
                 else:
-                    letter_result = result_to_show # To implement further UX with Text to Speech
-                    cwhich="New"
-                    if cresult_to_show == "M" and cresult_to_show !="M":
-                        cwhich="Old"
-                    
-                    if result_to_show != "R" and cresult_to_show =="R":
-                        cwhich="Old"
-                        letter_result = cresult_to_show
-
-                    if result_to_show != "T" and cresult_to_show =="T":
-                        cwhich="Old"
-                        letter_result = cresult_to_show
-                if cwhich=="Old" :
-                    letterscore = cclassification_result.classifications[0].categories[0].score
-
-                if cwhich=="New" :
-                    letterscore = classification_result.classifications[0].categories[0].score
+                    letter_result = result_to_show
+                letterscore = max(
+                    classification_result.classifications[0].categories[0].score,
+                    cclassification_result.classifications[0].categories[0].score
+                )
             else:
-                tf_image = vision2.TensorImage.create_from_array(cropped_image)
-                classification_result = cclassifier.classify(tf_image) # New
-                result_to_show = classification_result.classifications[0].categories[0].category_name # New
-
-                if result_to_show != "B":
-                    letter_result='_'
-                else:
-                    letter_result='>'
-    except Exception as e: 
-        # Ha! The catch err{throw err} scenario, it was actually quite useful in debugging though
-        print(e)
-    frame_data = image_to_data_uri(final_image)
-    #print(frame_data)
-    
-    return jsonify({"result": letter_result, "frame": frame_data}), 200
-
-def data_uri_to_image(data_uri):
-    header, encoded = data_uri.split(',', 1)
-    decoded_data = base64.b64decode(encoded)
-    nparr = np.frombuffer(decoded_data, np.uint8)
-    image = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
-    return image 
-
-def image_to_data_uri(image):
-    # Encode the image as a JPEG
-    _, buffer = cv2.imencode('.jpg', image)
-    # Convert the buffer to bytes
-    image_bytes = buffer.tobytes()
-    # Encode the bytes to Base64
-    base64_encoded = base64.b64encode(image_bytes).decode('utf-8')
-    # Create the Data URI
-    data_uri = f"data:image/jpeg;base64,{base64_encoded}"
-    return data_uri
-if (__name__ == '__main__'):
-    app.run( host='0.0.0.0', port=7860)
+                letter_result = '_'
+        else:
+            letter_result = '_'
+
+    except Exception as e:
+        print("Detection error:", e)
+        letter_result = '_'
+        annotated_image = frame
+
+    frame_out = image_to_data_uri(annotated_image)
+    return jsonify({"result": letter_result, "frame": frame_out})
+
+# Isẹ̀kiri translation API
+@app.route('/api/translate', methods=['POST'])
+def translate_to_isekiri():
+    data = request.json
+    text = data.get('text', '')
+    # Translate each letter to Isẹ̀kiri word or keep as is if unknown
+    translated = ' '.join(isekiri_dict.get(ch.upper(), ch) for ch in text if ch.strip())
+    return jsonify({'isekiri': translated})
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=7860)