initial commit

app.py

@@ -1,60 +1,390 @@
+import cv2
+import base64
+import numpy as np
+import io
+from flask import Flask, render_template, Response
+from flask_socketio import SocketIO, emit
+from PIL import Image
+from time import time as unix_time
 import os
-import requests
-import json
-from io import BytesIO
+import mediapipe as mp
+from mediapipe.tasks import python
+from mediapipe.tasks.python import vision
+import time
+import argparse
+from mediapipe.framework.formats import landmark_pb2
+from mediapipe import solutions
+from tflite_support.task import vision as vision2
+from tflite_support.task import core
+from tflite_support.task import processor
+from numpy.linalg import norm
 
-from flask import Flask, jsonify, render_template, request, send_file
+#Image Annotation Utils
+char_list=[]
+global letter_result
+letter_result = 0
+global old_letter_result
+old_letter_result = 0
+MARGIN = 10  # pixels
+FONT_SIZE = 1
+FONT_THICKNESS = 1
+HANDEDNESS_TEXT_COLOR = (88, 205, 54) # vibrant green
+global test_x
+global test_y
+global result_to_show
+result_to_show=0
+global cresult_to_show
+cresult_to_show=0
+text_x = 0
+text_y = 0
+cwhich=0
+lastwidth = 400
+letterscore=0
+frame_time=0
+same_letter_time=0
+no_hand_flag=1
+# UTILS
 
-from modules.inference import infer_t5
-from modules.dataset import query_emotion
 
-# https://huggingface.co/settings/tokens
-# https://huggingface.co/spaces/{username}/{space}/settings
-API_TOKEN = os.getenv("BIG_GAN_TOKEN")
+def brightness(img):
+    if len(img.shape) == 3:
+        # Colored RGB or BGR (*Do Not* use HSV images with this function)
+        # create brightness with euclidean norm
+        return np.average(norm(img, axis=2)) / np.sqrt(3)
+    else:
+        # Grayscale
+        return np.average(img)
 
-app = Flask(__name__)
+
+def draw_landmarks_on_image(rgb_image, detection_result):
+  hand_landmarks_list = detection_result.hand_landmarks
+  handedness_list = detection_result.handedness
+  annotated_image = np.copy(rgb_image)
+  crop = []
+  image_height, image_width, image_heightgray=annotated_image.shape
+
+
+  # Loop through the detected hands to visualize.
+  for idx in range(len(hand_landmarks_list)):
+    hand_landmarks = hand_landmarks_list[idx]
+    handedness = handedness_list[idx]
+
+    # Draw the hand landmarks.
+    hand_landmarks_proto = landmark_pb2.NormalizedLandmarkList()
+    hand_landmarks_proto.landmark.extend([
+      landmark_pb2.NormalizedLandmark(x=landmark.x, y=landmark.y, z=landmark.z) for landmark in hand_landmarks
+    ])
+    solutions.drawing_utils.draw_landmarks(
+      annotated_image,
+      hand_landmarks_proto,
+      solutions.hands.HAND_CONNECTIONS,
+      solutions.drawing_styles.get_default_hand_landmarks_style(),
+      solutions.drawing_styles.get_default_hand_connections_style())
+
+    # Get bounding box
+
+    height, width, _ = annotated_image.shape
+    
+    x_coordinates = [landmark.x for landmark in hand_landmarks]
+    y_coordinates = [landmark.y for landmark in hand_landmarks]
+    
+    min_x = int(min(x_coordinates) * width)  # Left
+    min_y = int(min(y_coordinates) * height)    # Top
+    max_x = int(max(x_coordinates) * width)  # Right
+    max_y = int(max(y_coordinates) * height) # Bottom
+
+    #Get dimensions of bounding box
+    sect_height = max_y-(min_y)
+    sect_width = max_x-(min_x)
+
+    #Get center of bounding box
+    center_x=(min_x+max_x)/2
+    center_y=(min_y+max_y)/2
+
+    sect_diameter=50
+    #Define dominant axis for aspect ratio
+
+    if(sect_height>sect_width):
+        sect_diameter = sect_height
+
+    if(sect_height<sect_width):
+        sect_diameter = sect_width
+
+    sect_diameter=sect_diameter+50 # Pad diameter
+    sect_radius=int(sect_diameter/2) # Find radius
+    
+    #Crop Image
+    crop_top=int(center_y-sect_radius) #Top boundry
+    crop_bottom=int(center_y+sect_radius) #Bottom boundry
+    crop_left=int(center_x-sect_radius) #Left boundry
+    crop_right=int(center_x+sect_radius) #Right boundry
+
+    #Account for out of canvas
+    if(crop_top<0): #Bounding box too high
+        crop_top=0
+
+    if(crop_left<0): #Bounding box too far left
+        crop_left=0
+
+    if(crop_right>image_width): #Bounding box too far right
+        crop_right=image_width
+
+    if(crop_bottom>image_height): #Bounding box too low
+        crop_bottom=image_height
+
+    # Trace bounding box
+    annotated_image = cv2.rectangle(annotated_image, (crop_left, crop_top), (crop_right, crop_bottom), (255,0,0), 6)
+
+    global text_x
+    global text_y
+
+    # For text, currently not used
+    text_x=crop_left
+    text_y=crop_top
+    
+    # Get cropped image
+    crop = annotated_image[crop_top:crop_bottom, crop_left:crop_right] 
+
+    # Scale cropped image
+    h, w = crop.shape[0:2]
+    neww = 150
+    newh = int(neww*(h/w))
+    crop = cv2.resize(crop, (neww, newh))
+    
+    #annotated_image[0:0+crop.shape[0], 0:0+crop.shape[1]] = crop # Used for superimposition
+
+    #annotated_image=crop # Used for replacement
+    
+  return [annotated_image, crop]
+
+#-------------------------------------------------------------
+
+# Letter List
+letter_list=["A","B","C","D","E","F","G","H","I","J","K","L","M","N","O","P","Q","R","S","T","U","V","W","X","Y","Z","#"]
+
+# Initialise MediaPipe hand landmark detction
+RESULT = None
+BaseOptions = mp.tasks.BaseOptions
+HandLandmarker = mp.tasks.vision.HandLandmarker
+HandLandmarkerOptions = mp.tasks.vision.HandLandmarkerOptions
+HandLandmarkerResult = mp.tasks.vision.HandLandmarkerResult
+VisionRunningMode = mp.tasks.vision.RunningMode
+
+cbase_options = core.BaseOptions(file_name="./better_exported/model.tflite") # New tflite
+ccbase_options = core.BaseOptions(file_name="./exported/model.tflite") # Old tflite
+
+# Initialise ASL tflite model
+
+cclassification_options = processor.ClassificationOptions(max_results=1)
+coptions = vision2.ImageClassifierOptions(base_options=cbase_options, classification_options=cclassification_options)
+ccoptions = vision2.ImageClassifierOptions(base_options=ccbase_options, classification_options=cclassification_options)
+cclassifier = vision2.ImageClassifier.create_from_options(coptions)
+ccclassifier = vision2.ImageClassifier.create_from_options(ccoptions)
 
 
-@app.route("/")
+
+def print_result(result: HandLandmarkerResult, output_image: mp.Image, timestamp_ms: int):
+
+    global RESULT
+    RESULT=result
+
+
+options = HandLandmarkerOptions(
+    base_options=BaseOptions(model_asset_path='hand_landmarker.task'),
+    running_mode=VisionRunningMode.LIVE_STREAM,
+    result_callback=print_result)
+
+
+detector = vision.HandLandmarker.create_from_options(options)
+video_frames=[]
+
+
+
+app = Flask(__name__)
+socketio = SocketIO(app)
+
+@app.route('/')
 def index():
-    return render_template("index.html")
+    return render_template('index.html')
+
+
+@socketio.on('video_frame')
+def handle_video_frame(frame):
+
+    response_frame = data_uri_to_image(frame)
+    decimg = response_frame
+
+#--------------------------------------------
+    mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=decimg) # Create MediaPipe image
+    #print(mp.Timestamp.from_seconds(time.time()).value)
+    
+    detection_result = detector.detect_async(mp_image, mp.Timestamp.from_seconds(time.time()).value) # detct
+    
+    # Try-Catch block, because detection is not done during model initialisation
+    global no_hand_flag, frame_time, same_letter_time, letter_result, old_letter_result, char_list, letterscore
+
+    try:
+        result_images = draw_landmarks_on_image(mp_image.numpy_view(), RESULT) # Array of annotated and cropped images
+        annotated_image = result_images[0] 
+        cropped_image = result_images[1]
+
+        #Standardise and fit shape by resizing
+
+        h, w = annotated_image.shape[0:2]
+        neww = 500
+        newh = int(neww*(h/w))
+        resized_image = cv2.resize(annotated_image, (neww, newh))
+        final_image=resized_image
+
+        if(RESULT.handedness != []): # To chack if there is any result at all and then feed tflite model
+            no_hand_flag=0
+
+            if RESULT.handedness[0][0].display_name == 'Right':
+                tf_image = vision2.TensorImage.create_from_array(cropped_image)
+                classification_result = cclassifier.classify(tf_image) # New
+                cclassification_result = ccclassifier.classify(tf_image) # Old
+                
+                result_to_show = classification_result.classifications[0].categories[0].category_name # New
+                cresult_to_show = cclassification_result.classifications[0].categories[0].category_name # Old
+
+                if cclassification_result.classifications[0].categories[0].score > classification_result.classifications[0].categories[0].score:
+                    letter_result = cresult_to_show # To implement further UX with Text to Speech
+                    cwhich="Old"
+                    if result_to_show == "P" and cresult_to_show !="P":
+                        cwhich="New"
+                        letter_result = result_to_show
+                    
+                    
+                else:
+                    letter_result = result_to_show # To implement further UX with Text to Speech
+                    cwhich="New"
+                    if cresult_to_show == "M" and cresult_to_show !="M":
+                        cwhich="Old"
+                    
+                    if result_to_show != "R" and cresult_to_show =="R":
+                        cwhich="Old"
+                        letter_result = cresult_to_show
+
+                    if result_to_show != "T" and cresult_to_show =="T":
+                        cwhich="Old"
+                        letter_result = cresult_to_show
+                if cwhich=="Old" :
+                    letterscore = cclassification_result.classifications[0].categories[0].score
+
+                if cwhich=="New" :
+                    letterscore = classification_result.classifications[0].categories[0].score
+            else:
+                tf_image = vision2.TensorImage.create_from_array(cropped_image)
+                classification_result = cclassifier.classify(tf_image) # New
+                result_to_show = classification_result.classifications[0].categories[0].category_name # New
+
+                if result_to_show != "B":
+                    letter_result='_'
+                else:
+                    letter_result='>'
+
+            same_letter_time = round((unix_time()) - frame_time, 2)
+            
+            #print(frame_time, same_letter_time)
+
+            if old_letter_result != letter_result:
+                frame_time = (unix_time())# Log Time
+                same_letter_time=0
+            
+            old_letter_result = letter_result
+            
+                
+        
+        
+
+                
 
+        else:
+            local_same_letter_time=0
+            if(no_hand_flag==0):
+                same_letter_time = round((unix_time()) - frame_time, 2)
 
-@app.route("/infer_biggan")
-def biggan():
-    input = request.args.get("input")
+            local_same_letter_time = round((unix_time()) - frame_time, 2)
 
-    output = requests.request(
-        "POST",
-        "https://api-inference.huggingface.co/models/osanseviero/BigGAN-deep-128",
-        headers={"Authorization": f"Bearer {API_TOKEN}"},
-        data=json.dumps(input),
-    )
+            letterscore = 0
+            #print(brightness(final_image))
 
-    return send_file(BytesIO(output.content), mimetype="image/png")
+            if local_same_letter_time>1.2 and brightness(final_image) < 40:
+                char_list.pop()
+                print (string.join(char_list))
+                frame_time = (unix_time())# Log Time
+                same_letter_time=0
+            
+            if same_letter_time>1.4 and brightness(final_image) > 40:
+                frame_time = (unix_time())# Log Time
+                same_letter_time=0
+                no_hand_flag=1
+           
+        same_letter_time_width = 10 + int(same_letter_time*100)
+        if same_letter_time_width > 190:
+            same_letter_time_width = 190
+        iheight, iwidth = final_image.shape[:2]
+        cv2.rectangle(final_image, (0, 0), (iwidth, 50), (255,255,255), -1)
 
+        cv2.rectangle(final_image, (0, 50), (200, 100), (255,255,255), -1)
+        cv2.rectangle(final_image, (8, 58), (192, 72), (255,100,100), -1)
+        cv2.rectangle(final_image, (10, 60), (190, 70), (255,200,200), -1)
+        cv2.rectangle(final_image, (10, 60), (same_letter_time_width, 70), (255,100,100), -1)
 
-@app.route("/infer_t5")
-def t5():
-    input = request.args.get("input")
+        letterscore_width = 10+ int(letterscore*100)
 
-    output = infer_t5(input)
+        
+        cv2.rectangle(final_image, (8, 78), (192, 92), (100,100,200), -1)
+        cv2.rectangle(final_image, (10, 80), (190, 90), (150,175,255), -1)
+        cv2.rectangle(final_image, (10, 80), (letterscore_width, 90), (100,100,200), -1)
+        
+        cv2.putText(final_image, f"{(letterscore_width)}", # Display result
+        (12, 70), cv2.FONT_HERSHEY_DUPLEX,
+        0.5, (0, 0, 0), 1, cv2.LINE_AA)
 
-    return jsonify({"output": output})
+        cv2.putText(final_image, f"[ {letter_result} ] {''.join(char_list)} |", # Display result
+            (20, 40), cv2.FONT_HERSHEY_DUPLEX,
+            0.7, (0, 0, 0), 2, cv2.LINE_AA)
+        
+        print(letter_result, same_letter_time)
+                
+    except Exception as e: 
+        # Ha! The catch err{throw err} scenario, it was actually quite useful in debugging though
+        print(e)
 
 
-@app.route("/query_emotion")
-def emotion():
-    start = request.args.get("start")
-    end = request.args.get("end")
+    if  same_letter_time > 0 and RESULT.handedness != []: # If 'a' key was pressed and a hand exists
+        
+        if same_letter_time > 1.7 and RESULT.handedness[0][0].display_name == 'Right': # Right hand
+            char_list.append(letter_result)
+            string = ""
+            print (string.join(char_list))
+            frame_time = (unix_time())# Log Time
+            same_letter_time=0
+            
 
-    print(start)
-    print(end)
+        if  same_letter_time > 0.9 and RESULT.handedness[0][0].display_name == 'Left':
+                if same_letter_time > 1.2 and letter_result=='_':
+                    char_list.append(" ")
+                    string = "" 
+                    print (string.join(char_list))
+                    frame_time = (unix_time())# Log Time
+                    same_letter_time=0
 
-    output = query_emotion(int(start), int(end))
+                if letter_result == ">":
+                    string=""
+                    if(string.join(char_list) != ''):
+                        os.system(f'echo {string.join(char_list)} | espeak -p 70 -s 140')
+                        char_list=[]
+                        print (string.join(char_list))             
 
-    return jsonify({"output": output})
 
+def data_uri_to_image(data_uri):
+    header, encoded = data_uri.split(',', 1)
+    decoded_data = base64.b64decode(encoded)
+    nparr = np.frombuffer(decoded_data, np.uint8)
+    image = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
+    return image 
 
-if __name__ == "__main__":
-    app.run(host="0.0.0.0", port=7860)
+if (__name__ == '__main__'):
+    app.run( host='0.0.0.0', port=7860)

requirements.txt

@@ -1,6 +1,6 @@
-datasets==2.*
-flask==3.0.0
-requests==2.31.*
-sentencepiece==0.1.*
-torch==2.*
-transformers==4.*
+opencv-python-headless == 4.10.0.84
+numpy == 1.23.3
+Flask == 3.0.3
+Flask-SocketIO == 5.4.1
+mediapipe == 0.10.11
+tensorflow == 2.8.0