import cv2 import base64 import numpy as np import io from flask import Flask, render_template, Response, request, jsonify from flask_socketio import SocketIO, emit from PIL import Image from time import time as unix_time import os import mediapipe as mp from mediapipe.tasks import python from mediapipe.tasks.python import vision import time import argparse from mediapipe.framework.formats import landmark_pb2 from mediapipe import solutions from tflite_support.task import vision as vision2 from tflite_support.task import core from tflite_support.task import processor from numpy.linalg import norm #Image Annotation Utils char_list=[] global letter_result letter_result = 0 global old_letter_result old_letter_result = 0 MARGIN = 10 # pixels FONT_SIZE = 1 FONT_THICKNESS = 1 HANDEDNESS_TEXT_COLOR = (88, 205, 54) # vibrant green global test_x global test_y global result_to_show result_to_show=0 global cresult_to_show cresult_to_show=0 text_x = 0 text_y = 0 cwhich=0 lastwidth = 400 letterscore=0 frame_time=0 same_letter_time=0 no_hand_flag=1 # UTILS def brightness(img): if len(img.shape) == 3: # Colored RGB or BGR (*Do Not* use HSV images with this function) # create brightness with euclidean norm return np.average(norm(img, axis=2)) / np.sqrt(3) else: # Grayscale return np.average(img) def draw_landmarks_on_image(rgb_image, detection_result): hand_landmarks_list = detection_result.hand_landmarks handedness_list = detection_result.handedness annotated_image = np.copy(rgb_image) crop = [] image_height, image_width, image_heightgray=annotated_image.shape # Loop through the detected hands to visualize. for idx in range(len(hand_landmarks_list)): hand_landmarks = hand_landmarks_list[idx] handedness = handedness_list[idx] # Draw the hand landmarks. hand_landmarks_proto = landmark_pb2.NormalizedLandmarkList() hand_landmarks_proto.landmark.extend([ landmark_pb2.NormalizedLandmark(x=landmark.x, y=landmark.y, z=landmark.z) for landmark in hand_landmarks ]) solutions.drawing_utils.draw_landmarks( annotated_image, hand_landmarks_proto, solutions.hands.HAND_CONNECTIONS, solutions.drawing_styles.get_default_hand_landmarks_style(), solutions.drawing_styles.get_default_hand_connections_style()) # Get bounding box height, width, _ = annotated_image.shape x_coordinates = [landmark.x for landmark in hand_landmarks] y_coordinates = [landmark.y for landmark in hand_landmarks] min_x = int(min(x_coordinates) * width) # Left min_y = int(min(y_coordinates) * height) # Top max_x = int(max(x_coordinates) * width) # Right max_y = int(max(y_coordinates) * height) # Bottom #Get dimensions of bounding box sect_height = max_y-(min_y) sect_width = max_x-(min_x) #Get center of bounding box center_x=(min_x+max_x)/2 center_y=(min_y+max_y)/2 sect_diameter=50 #Define dominant axis for aspect ratio if(sect_height>sect_width): sect_diameter = sect_height if(sect_heightimage_width): #Bounding box too far right crop_right=image_width if(crop_bottom>image_height): #Bounding box too low crop_bottom=image_height # Trace bounding box annotated_image = cv2.rectangle(annotated_image, (crop_left, crop_top), (crop_right, crop_bottom), (255,0,0), 6) global text_x global text_y # For text, currently not used text_x=crop_left text_y=crop_top # Get cropped image crop = annotated_image[crop_top:crop_bottom, crop_left:crop_right] # Scale cropped image h, w = crop.shape[0:2] neww = 150 newh = int(neww*(h/w)) crop = cv2.resize(crop, (neww, newh)) #annotated_image[0:0+crop.shape[0], 0:0+crop.shape[1]] = crop # Used for superimposition #annotated_image=crop # Used for replacement return [annotated_image, crop] #------------------------------------------------------------- # Letter List letter_list=["A","B","C","D","E","F","G","H","I","J","K","L","M","N","O","P","Q","R","S","T","U","V","W","X","Y","Z","#"] # Initialise MediaPipe hand landmark detction RESULT = None BaseOptions = mp.tasks.BaseOptions HandLandmarker = mp.tasks.vision.HandLandmarker HandLandmarkerOptions = mp.tasks.vision.HandLandmarkerOptions HandLandmarkerResult = mp.tasks.vision.HandLandmarkerResult VisionRunningMode = mp.tasks.vision.RunningMode cbase_options = core.BaseOptions(file_name="./better_exported/model.tflite") # New tflite ccbase_options = core.BaseOptions(file_name="./exported/model.tflite") # Old tflite # Initialise ASL tflite model cclassification_options = processor.ClassificationOptions(max_results=1) coptions = vision2.ImageClassifierOptions(base_options=cbase_options, classification_options=cclassification_options) ccoptions = vision2.ImageClassifierOptions(base_options=ccbase_options, classification_options=cclassification_options) cclassifier = vision2.ImageClassifier.create_from_options(coptions) ccclassifier = vision2.ImageClassifier.create_from_options(ccoptions) def print_result(result: HandLandmarkerResult, output_image: mp.Image, timestamp_ms: int): global RESULT RESULT=result options = HandLandmarkerOptions( base_options=BaseOptions(model_asset_path='hand_landmarker.task'), running_mode=VisionRunningMode.LIVE_STREAM, result_callback=print_result) detector = vision.HandLandmarker.create_from_options(options) video_frames=[] app = Flask(__name__) @app.route('/') def index(): return render_template('index.html') @app.route('/api/data', methods=['POST']) def handle_video_frame(): frame = request.json.get('key') #print(request.json) response_frame = data_uri_to_image(frame) decimg = response_frame #-------------------------------------------- mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=decimg) # Create MediaPipe image #print(mp.Timestamp.from_seconds(time.time()).value) detection_result = detector.detect_async(mp_image, mp.Timestamp.from_seconds(time.time()).value) # detct # Try-Catch block, because detection is not done during model initialisation global no_hand_flag, frame_time, same_letter_time, letter_result, old_letter_result, char_list, letterscore try: result_images = draw_landmarks_on_image(mp_image.numpy_view(), RESULT) # Array of annotated and cropped images annotated_image = result_images[0] cropped_image = result_images[1] #Standardise and fit shape by resizing h, w = annotated_image.shape[0:2] neww = 500 newh = int(neww*(h/w)) resized_image = cv2.resize(annotated_image, (neww, newh)) final_image=resized_image if(RESULT.handedness != []): # To chack if there is any result at all and then feed tflite model no_hand_flag=0 if RESULT.handedness[0][0].display_name == 'Right': tf_image = vision2.TensorImage.create_from_array(cropped_image) classification_result = cclassifier.classify(tf_image) # New cclassification_result = ccclassifier.classify(tf_image) # Old result_to_show = classification_result.classifications[0].categories[0].category_name # New cresult_to_show = cclassification_result.classifications[0].categories[0].category_name # Old if cclassification_result.classifications[0].categories[0].score > classification_result.classifications[0].categories[0].score: letter_result = cresult_to_show # To implement further UX with Text to Speech cwhich="Old" if result_to_show == "P" and cresult_to_show !="P": cwhich="New" letter_result = result_to_show else: letter_result = result_to_show # To implement further UX with Text to Speech cwhich="New" if cresult_to_show == "M" and cresult_to_show !="M": cwhich="Old" if result_to_show != "R" and cresult_to_show =="R": cwhich="Old" letter_result = cresult_to_show if result_to_show != "T" and cresult_to_show =="T": cwhich="Old" letter_result = cresult_to_show if cwhich=="Old" : letterscore = cclassification_result.classifications[0].categories[0].score if cwhich=="New" : letterscore = classification_result.classifications[0].categories[0].score else: tf_image = vision2.TensorImage.create_from_array(cropped_image) classification_result = cclassifier.classify(tf_image) # New result_to_show = classification_result.classifications[0].categories[0].category_name # New if result_to_show != "B": letter_result='_' else: letter_result='>' except Exception as e: # Ha! The catch err{throw err} scenario, it was actually quite useful in debugging though print(e) frame_data = image_to_data_uri(final_image) #print(frame_data) return jsonify({"result": letter_result, "frame": frame_data}), 200 def data_uri_to_image(data_uri): header, encoded = data_uri.split(',', 1) decoded_data = base64.b64decode(encoded) nparr = np.frombuffer(decoded_data, np.uint8) image = cv2.imdecode(nparr, cv2.IMREAD_COLOR) return image def image_to_data_uri(image): # Encode the image as a JPEG _, buffer = cv2.imencode('.jpg', image) # Convert the buffer to bytes image_bytes = buffer.tobytes() # Encode the bytes to Base64 base64_encoded = base64.b64encode(image_bytes).decode('utf-8') # Create the Data URI data_uri = f"data:image/jpeg;base64,{base64_encoded}" return data_uri if (__name__ == '__main__'): app.run( host='0.0.0.0', port=7860)