Spaces:
Runtime error
Runtime error
| import cv2 | |
| import base64 | |
| import numpy as np | |
| import io | |
| from flask import Flask, render_template, Response | |
| from flask_socketio import SocketIO, emit | |
| from PIL import Image | |
| from time import time as unix_time | |
| import os | |
| import mediapipe as mp | |
| from mediapipe.tasks import python | |
| from mediapipe.tasks.python import vision | |
| import time | |
| import argparse | |
| from mediapipe.framework.formats import landmark_pb2 | |
| from mediapipe import solutions | |
| from tflite_support.task import vision as vision2 | |
| from tflite_support.task import core | |
| from tflite_support.task import processor | |
| from numpy.linalg import norm | |
| #Image Annotation Utils | |
| char_list=[] | |
| global letter_result | |
| letter_result = 0 | |
| global old_letter_result | |
| old_letter_result = 0 | |
| MARGIN = 10 # pixels | |
| FONT_SIZE = 1 | |
| FONT_THICKNESS = 1 | |
| HANDEDNESS_TEXT_COLOR = (88, 205, 54) # vibrant green | |
| global test_x | |
| global test_y | |
| global result_to_show | |
| result_to_show=0 | |
| global cresult_to_show | |
| cresult_to_show=0 | |
| text_x = 0 | |
| text_y = 0 | |
| cwhich=0 | |
| lastwidth = 400 | |
| letterscore=0 | |
| frame_time=0 | |
| same_letter_time=0 | |
| no_hand_flag=1 | |
| # UTILS | |
| def brightness(img): | |
| if len(img.shape) == 3: | |
| # Colored RGB or BGR (*Do Not* use HSV images with this function) | |
| # create brightness with euclidean norm | |
| return np.average(norm(img, axis=2)) / np.sqrt(3) | |
| else: | |
| # Grayscale | |
| return np.average(img) | |
| def draw_landmarks_on_image(rgb_image, detection_result): | |
| hand_landmarks_list = detection_result.hand_landmarks | |
| handedness_list = detection_result.handedness | |
| annotated_image = np.copy(rgb_image) | |
| crop = [] | |
| image_height, image_width, image_heightgray=annotated_image.shape | |
| # Loop through the detected hands to visualize. | |
| for idx in range(len(hand_landmarks_list)): | |
| hand_landmarks = hand_landmarks_list[idx] | |
| handedness = handedness_list[idx] | |
| # Draw the hand landmarks. | |
| hand_landmarks_proto = landmark_pb2.NormalizedLandmarkList() | |
| hand_landmarks_proto.landmark.extend([ | |
| landmark_pb2.NormalizedLandmark(x=landmark.x, y=landmark.y, z=landmark.z) for landmark in hand_landmarks | |
| ]) | |
| solutions.drawing_utils.draw_landmarks( | |
| annotated_image, | |
| hand_landmarks_proto, | |
| solutions.hands.HAND_CONNECTIONS, | |
| solutions.drawing_styles.get_default_hand_landmarks_style(), | |
| solutions.drawing_styles.get_default_hand_connections_style()) | |
| # Get bounding box | |
| height, width, _ = annotated_image.shape | |
| x_coordinates = [landmark.x for landmark in hand_landmarks] | |
| y_coordinates = [landmark.y for landmark in hand_landmarks] | |
| min_x = int(min(x_coordinates) * width) # Left | |
| min_y = int(min(y_coordinates) * height) # Top | |
| max_x = int(max(x_coordinates) * width) # Right | |
| max_y = int(max(y_coordinates) * height) # Bottom | |
| #Get dimensions of bounding box | |
| sect_height = max_y-(min_y) | |
| sect_width = max_x-(min_x) | |
| #Get center of bounding box | |
| center_x=(min_x+max_x)/2 | |
| center_y=(min_y+max_y)/2 | |
| sect_diameter=50 | |
| #Define dominant axis for aspect ratio | |
| if(sect_height>sect_width): | |
| sect_diameter = sect_height | |
| if(sect_height<sect_width): | |
| sect_diameter = sect_width | |
| sect_diameter=sect_diameter+50 # Pad diameter | |
| sect_radius=int(sect_diameter/2) # Find radius | |
| #Crop Image | |
| crop_top=int(center_y-sect_radius) #Top boundry | |
| crop_bottom=int(center_y+sect_radius) #Bottom boundry | |
| crop_left=int(center_x-sect_radius) #Left boundry | |
| crop_right=int(center_x+sect_radius) #Right boundry | |
| #Account for out of canvas | |
| if(crop_top<0): #Bounding box too high | |
| crop_top=0 | |
| if(crop_left<0): #Bounding box too far left | |
| crop_left=0 | |
| if(crop_right>image_width): #Bounding box too far right | |
| crop_right=image_width | |
| if(crop_bottom>image_height): #Bounding box too low | |
| crop_bottom=image_height | |
| # Trace bounding box | |
| annotated_image = cv2.rectangle(annotated_image, (crop_left, crop_top), (crop_right, crop_bottom), (255,0,0), 6) | |
| global text_x | |
| global text_y | |
| # For text, currently not used | |
| text_x=crop_left | |
| text_y=crop_top | |
| # Get cropped image | |
| crop = annotated_image[crop_top:crop_bottom, crop_left:crop_right] | |
| # Scale cropped image | |
| h, w = crop.shape[0:2] | |
| neww = 150 | |
| newh = int(neww*(h/w)) | |
| crop = cv2.resize(crop, (neww, newh)) | |
| #annotated_image[0:0+crop.shape[0], 0:0+crop.shape[1]] = crop # Used for superimposition | |
| #annotated_image=crop # Used for replacement | |
| return [annotated_image, crop] | |
| #------------------------------------------------------------- | |
| # Letter List | |
| letter_list=["A","B","C","D","E","F","G","H","I","J","K","L","M","N","O","P","Q","R","S","T","U","V","W","X","Y","Z","#"] | |
| # Initialise MediaPipe hand landmark detction | |
| RESULT = None | |
| BaseOptions = mp.tasks.BaseOptions | |
| HandLandmarker = mp.tasks.vision.HandLandmarker | |
| HandLandmarkerOptions = mp.tasks.vision.HandLandmarkerOptions | |
| HandLandmarkerResult = mp.tasks.vision.HandLandmarkerResult | |
| VisionRunningMode = mp.tasks.vision.RunningMode | |
| cbase_options = core.BaseOptions(file_name="./better_exported/model.tflite") # New tflite | |
| ccbase_options = core.BaseOptions(file_name="./exported/model.tflite") # Old tflite | |
| # Initialise ASL tflite model | |
| cclassification_options = processor.ClassificationOptions(max_results=1) | |
| coptions = vision2.ImageClassifierOptions(base_options=cbase_options, classification_options=cclassification_options) | |
| ccoptions = vision2.ImageClassifierOptions(base_options=ccbase_options, classification_options=cclassification_options) | |
| cclassifier = vision2.ImageClassifier.create_from_options(coptions) | |
| ccclassifier = vision2.ImageClassifier.create_from_options(ccoptions) | |
| def print_result(result: HandLandmarkerResult, output_image: mp.Image, timestamp_ms: int): | |
| global RESULT | |
| RESULT=result | |
| options = HandLandmarkerOptions( | |
| base_options=BaseOptions(model_asset_path='hand_landmarker.task'), | |
| running_mode=VisionRunningMode.LIVE_STREAM, | |
| result_callback=print_result) | |
| detector = vision.HandLandmarker.create_from_options(options) | |
| video_frames=[] | |
| app = Flask(__name__) | |
| socketio = SocketIO(app) | |
| def index(): | |
| return render_template('index.html') | |
| def handle_video_frame(frame): | |
| response_frame = data_uri_to_image(frame) | |
| decimg = response_frame | |
| #-------------------------------------------- | |
| mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=decimg) # Create MediaPipe image | |
| #print(mp.Timestamp.from_seconds(time.time()).value) | |
| detection_result = detector.detect_async(mp_image, mp.Timestamp.from_seconds(time.time()).value) # detct | |
| # Try-Catch block, because detection is not done during model initialisation | |
| global no_hand_flag, frame_time, same_letter_time, letter_result, old_letter_result, char_list, letterscore | |
| try: | |
| result_images = draw_landmarks_on_image(mp_image.numpy_view(), RESULT) # Array of annotated and cropped images | |
| annotated_image = result_images[0] | |
| cropped_image = result_images[1] | |
| #Standardise and fit shape by resizing | |
| h, w = annotated_image.shape[0:2] | |
| neww = 500 | |
| newh = int(neww*(h/w)) | |
| resized_image = cv2.resize(annotated_image, (neww, newh)) | |
| final_image=resized_image | |
| if(RESULT.handedness != []): # To chack if there is any result at all and then feed tflite model | |
| no_hand_flag=0 | |
| if RESULT.handedness[0][0].display_name == 'Right': | |
| tf_image = vision2.TensorImage.create_from_array(cropped_image) | |
| classification_result = cclassifier.classify(tf_image) # New | |
| cclassification_result = ccclassifier.classify(tf_image) # Old | |
| result_to_show = classification_result.classifications[0].categories[0].category_name # New | |
| cresult_to_show = cclassification_result.classifications[0].categories[0].category_name # Old | |
| if cclassification_result.classifications[0].categories[0].score > classification_result.classifications[0].categories[0].score: | |
| letter_result = cresult_to_show # To implement further UX with Text to Speech | |
| cwhich="Old" | |
| if result_to_show == "P" and cresult_to_show !="P": | |
| cwhich="New" | |
| letter_result = result_to_show | |
| else: | |
| letter_result = result_to_show # To implement further UX with Text to Speech | |
| cwhich="New" | |
| if cresult_to_show == "M" and cresult_to_show !="M": | |
| cwhich="Old" | |
| if result_to_show != "R" and cresult_to_show =="R": | |
| cwhich="Old" | |
| letter_result = cresult_to_show | |
| if result_to_show != "T" and cresult_to_show =="T": | |
| cwhich="Old" | |
| letter_result = cresult_to_show | |
| if cwhich=="Old" : | |
| letterscore = cclassification_result.classifications[0].categories[0].score | |
| if cwhich=="New" : | |
| letterscore = classification_result.classifications[0].categories[0].score | |
| else: | |
| tf_image = vision2.TensorImage.create_from_array(cropped_image) | |
| classification_result = cclassifier.classify(tf_image) # New | |
| result_to_show = classification_result.classifications[0].categories[0].category_name # New | |
| if result_to_show != "B": | |
| letter_result='_' | |
| else: | |
| letter_result='>' | |
| same_letter_time = round((unix_time()) - frame_time, 2) | |
| #print(frame_time, same_letter_time) | |
| if old_letter_result != letter_result: | |
| frame_time = (unix_time())# Log Time | |
| same_letter_time=0 | |
| old_letter_result = letter_result | |
| else: | |
| local_same_letter_time=0 | |
| if(no_hand_flag==0): | |
| same_letter_time = round((unix_time()) - frame_time, 2) | |
| local_same_letter_time = round((unix_time()) - frame_time, 2) | |
| letterscore = 0 | |
| #print(brightness(final_image)) | |
| if local_same_letter_time>1.2 and brightness(final_image) < 40: | |
| char_list.pop() | |
| print (string.join(char_list)) | |
| frame_time = (unix_time())# Log Time | |
| same_letter_time=0 | |
| if same_letter_time>1.4 and brightness(final_image) > 40: | |
| frame_time = (unix_time())# Log Time | |
| same_letter_time=0 | |
| no_hand_flag=1 | |
| same_letter_time_width = 10 + int(same_letter_time*100) | |
| if same_letter_time_width > 190: | |
| same_letter_time_width = 190 | |
| iheight, iwidth = final_image.shape[:2] | |
| cv2.rectangle(final_image, (0, 0), (iwidth, 50), (255,255,255), -1) | |
| cv2.rectangle(final_image, (0, 50), (200, 100), (255,255,255), -1) | |
| cv2.rectangle(final_image, (8, 58), (192, 72), (255,100,100), -1) | |
| cv2.rectangle(final_image, (10, 60), (190, 70), (255,200,200), -1) | |
| cv2.rectangle(final_image, (10, 60), (same_letter_time_width, 70), (255,100,100), -1) | |
| letterscore_width = 10+ int(letterscore*100) | |
| cv2.rectangle(final_image, (8, 78), (192, 92), (100,100,200), -1) | |
| cv2.rectangle(final_image, (10, 80), (190, 90), (150,175,255), -1) | |
| cv2.rectangle(final_image, (10, 80), (letterscore_width, 90), (100,100,200), -1) | |
| cv2.putText(final_image, f"{(letterscore_width)}", # Display result | |
| (12, 70), cv2.FONT_HERSHEY_DUPLEX, | |
| 0.5, (0, 0, 0), 1, cv2.LINE_AA) | |
| cv2.putText(final_image, f"[ {letter_result} ] {''.join(char_list)} |", # Display result | |
| (20, 40), cv2.FONT_HERSHEY_DUPLEX, | |
| 0.7, (0, 0, 0), 2, cv2.LINE_AA) | |
| print(letter_result, same_letter_time) | |
| except Exception as e: | |
| # Ha! The catch err{throw err} scenario, it was actually quite useful in debugging though | |
| print(e) | |
| if same_letter_time > 0 and RESULT.handedness != []: # If 'a' key was pressed and a hand exists | |
| if same_letter_time > 1.7 and RESULT.handedness[0][0].display_name == 'Right': # Right hand | |
| char_list.append(letter_result) | |
| string = "" | |
| print (string.join(char_list)) | |
| frame_time = (unix_time())# Log Time | |
| same_letter_time=0 | |
| if same_letter_time > 0.9 and RESULT.handedness[0][0].display_name == 'Left': | |
| if same_letter_time > 1.2 and letter_result=='_': | |
| char_list.append(" ") | |
| string = "" | |
| print (string.join(char_list)) | |
| frame_time = (unix_time())# Log Time | |
| same_letter_time=0 | |
| if letter_result == ">": | |
| string="" | |
| if(string.join(char_list) != ''): | |
| os.system(f'echo {string.join(char_list)} | espeak -p 70 -s 140') | |
| char_list=[] | |
| print (string.join(char_list)) | |
| def data_uri_to_image(data_uri): | |
| header, encoded = data_uri.split(',', 1) | |
| decoded_data = base64.b64decode(encoded) | |
| nparr = np.frombuffer(decoded_data, np.uint8) | |
| image = cv2.imdecode(nparr, cv2.IMREAD_COLOR) | |
| return image | |
| if (__name__ == '__main__'): | |
| app.run( host='0.0.0.0', port=7860) |