import numpy as np import cv2 import math import traceback from keras.models import load_model from cvzone.HandTrackingModule import HandDetector from string import ascii_uppercase import enchant import gradio as gr from PIL import Image import tempfile import os # Initialize components ddd = enchant.Dict("en-US") hd = HandDetector(maxHands=1) hd2 = HandDetector(maxHands=1) offset = 29 # Load model model = load_model('cnn8grps_rad1_model.h5') print("Model loaded successfully") # Create white canvas if not exists if not os.path.exists("white.jpg"): white_canvas = np.ones((400, 400, 3), dtype=np.uint8) * 255 cv2.imwrite("white.jpg", white_canvas) class SignLanguageRecognizer: def __init__(self): self.ct = {} self.ct['blank'] = 0 self.blank_flag = 0 self.space_flag = False self.next_flag = True self.prev_char = "" self.count = -1 self.ten_prev_char = [" "] * 10 self.str = " " self.word = " " self.current_symbol = "C" self.word1 = " " self.word2 = " " self.word3 = " " self.word4 = " " for i in ascii_uppercase: self.ct[i] = 0 def distance(self, x, y): return math.sqrt(((x[0] - y[0]) ** 2) + ((x[1] - y[1]) ** 2)) def predict(self, test_image, pts): white = test_image white = white.reshape(1, 400, 400, 3) prob = np.array(model.predict(white, verbose=0)[0], dtype='float32') ch1 = np.argmax(prob, axis=0) prob[ch1] = 0 ch2 = np.argmax(prob, axis=0) prob[ch2] = 0 ch3 = np.argmax(prob, axis=0) pl = [ch1, ch2] # condition for [Aemnst] l = [[5, 2], [5, 3], [3, 5], [3, 6], [3, 0], [3, 2], [6, 4], [6, 1], [6, 2], [6, 6], [6, 7], [6, 0], [6, 5], [4, 1], [1, 0], [1, 1], [6, 3], [1, 6], [5, 6], [5, 1], [4, 5], [1, 4], [1, 5], [2, 0], [2, 6], [4, 6], [1, 0], [5, 7], [1, 6], [6, 1], [7, 6], [2, 5], [7, 1], [5, 4], [7, 0], [7, 5], [7, 2]] if pl in l: if (self.pts[6][1] < self.pts[8][1] and self.pts[10][1] < self.pts[12][1] and self.pts[14][1] < self.pts[16][1] and self.pts[18][1] < self.pts[20][ 1]): ch1 = 0 # print("00000") # condition for [o][s] l = [[2, 2], [2, 1]] if pl in l: if (self.pts[5][0] < self.pts[4][0]): ch1 = 0 print("++++++++++++++++++") # print("00000") # condition for [c0][aemnst] l = [[0, 0], [0, 6], [0, 2], [0, 5], [0, 1], [0, 7], [5, 2], [7, 6], [7, 1]] pl = [ch1, ch2] if pl in l: if (self.pts[0][0] > self.pts[8][0] and self.pts[0][0] > self.pts[4][0] and self.pts[0][0] > self.pts[12][0] and self.pts[0][0] > self.pts[16][ 0] and self.pts[0][0] > self.pts[20][0]) and self.pts[5][0] > self.pts[4][0]: ch1 = 2 # print("22222") # condition for [c0][aemnst] l = [[6, 0], [6, 6], [6, 2]] pl = [ch1, ch2] if pl in l: if self.distance(self.pts[8], self.pts[16]) < 52: ch1 = 2 # print("22222") # condition for [gh][bdfikruvw] l = [[1, 4], [1, 5], [1, 6], [1, 3], [1, 0]] pl = [ch1, ch2] if pl in l: if self.pts[6][1] > self.pts[8][1] and self.pts[14][1] < self.pts[16][1] and self.pts[18][1] < self.pts[20][1] and self.pts[0][0] < self.pts[8][ 0] and self.pts[0][0] < self.pts[12][0] and self.pts[0][0] < self.pts[16][0] and self.pts[0][0] < self.pts[20][0]: ch1 = 3 print("33333c") # con for [gh][l] l = [[4, 6], [4, 1], [4, 5], [4, 3], [4, 7]] pl = [ch1, ch2] if pl in l: if self.pts[4][0] > self.pts[0][0]: ch1 = 3 print("33333b") # con for [gh][pqz] l = [[5, 3], [5, 0], [5, 7], [5, 4], [5, 2], [5, 1], [5, 5]] pl = [ch1, ch2] if pl in l: if self.pts[2][1] + 15 < self.pts[16][1]: ch1 = 3 print("33333a") # con for [l][x] l = [[6, 4], [6, 1], [6, 2]] pl = [ch1, ch2] if pl in l: if self.distance(self.pts[4], self.pts[11]) > 55: ch1 = 4 # print("44444") # con for [l][d] l = [[1, 4], [1, 6], [1, 1]] pl = [ch1, ch2] if pl in l: if (self.distance(self.pts[4], self.pts[11]) > 50) and ( self.pts[6][1] > self.pts[8][1] and self.pts[10][1] < self.pts[12][1] and self.pts[14][1] < self.pts[16][1] and self.pts[18][1] < self.pts[20][1]): ch1 = 4 # print("44444") # con for [l][gh] l = [[3, 6], [3, 4]] pl = [ch1, ch2] if pl in l: if (self.pts[4][0] < self.pts[0][0]): ch1 = 4 # print("44444") # con for [l][c0] l = [[2, 2], [2, 5], [2, 4]] pl = [ch1, ch2] if pl in l: if (self.pts[1][0] < self.pts[12][0]): ch1 = 4 # print("44444") # con for [l][c0] l = [[2, 2], [2, 5], [2, 4]] pl = [ch1, ch2] if pl in l: if (self.pts[1][0] < self.pts[12][0]): ch1 = 4 # print("44444") # con for [gh][z] l = [[3, 6], [3, 5], [3, 4]] pl = [ch1, ch2] if pl in l: if (self.pts[6][1] > self.pts[8][1] and self.pts[10][1] < self.pts[12][1] and self.pts[14][1] < self.pts[16][1] and self.pts[18][1] < self.pts[20][ 1]) and self.pts[4][1] > self.pts[10][1]: ch1 = 5 print("55555b") # con for [gh][pq] l = [[3, 2], [3, 1], [3, 6]] pl = [ch1, ch2] if pl in l: if self.pts[4][1] + 17 > self.pts[8][1] and self.pts[4][1] + 17 > self.pts[12][1] and self.pts[4][1] + 17 > self.pts[16][1] and self.pts[4][ 1] + 17 > self.pts[20][1]: ch1 = 5 print("55555a") # con for [l][pqz] l = [[4, 4], [4, 5], [4, 2], [7, 5], [7, 6], [7, 0]] pl = [ch1, ch2] if pl in l: if self.pts[4][0] > self.pts[0][0]: ch1 = 5 # print("55555") # con for [pqz][aemnst] l = [[0, 2], [0, 6], [0, 1], [0, 5], [0, 0], [0, 7], [0, 4], [0, 3], [2, 7]] pl = [ch1, ch2] if pl in l: if self.pts[0][0] < self.pts[8][0] and self.pts[0][0] < self.pts[12][0] and self.pts[0][0] < self.pts[16][0] and self.pts[0][0] < self.pts[20][0]: ch1 = 5 # print("55555") # con for [pqz][yj] l = [[5, 7], [5, 2], [5, 6]] pl = [ch1, ch2] if pl in l: if self.pts[3][0] < self.pts[0][0]: ch1 = 7 # print("77777") # con for [l][yj] l = [[4, 6], [4, 2], [4, 4], [4, 1], [4, 5], [4, 7]] pl = [ch1, ch2] if pl in l: if self.pts[6][1] < self.pts[8][1]: ch1 = 7 # print("77777") # con for [x][yj] l = [[6, 7], [0, 7], [0, 1], [0, 0], [6, 4], [6, 6], [6, 5], [6, 1]] pl = [ch1, ch2] if pl in l: if self.pts[18][1] > self.pts[20][1]: ch1 = 7 # print("77777") # condition for [x][aemnst] l = [[0, 4], [0, 2], [0, 3], [0, 1], [0, 6]] pl = [ch1, ch2] if pl in l: if self.pts[5][0] > self.pts[16][0]: ch1 = 6 print("666661") # condition for [yj][x] print("2222 ch1=+++++++++++++++++", ch1, ",", ch2) l = [[7, 2]] pl = [ch1, ch2] if pl in l: if self.pts[18][1] < self.pts[20][1] and self.pts[8][1] < self.pts[10][1]: ch1 = 6 print("666662") # condition for [c0][x] l = [[2, 1], [2, 2], [2, 6], [2, 7], [2, 0]] pl = [ch1, ch2] if pl in l: if self.distance(self.pts[8], self.pts[16]) > 50: ch1 = 6 print("666663") # con for [l][x] l = [[4, 6], [4, 2], [4, 1], [4, 4]] pl = [ch1, ch2] if pl in l: if self.distance(self.pts[4], self.pts[11]) < 60: ch1 = 6 print("666664") # con for [x][d] l = [[1, 4], [1, 6], [1, 0], [1, 2]] pl = [ch1, ch2] if pl in l: if self.pts[5][0] - self.pts[4][0] - 15 > 0: ch1 = 6 print("666665") # con for [b][pqz] l = [[5, 0], [5, 1], [5, 4], [5, 5], [5, 6], [6, 1], [7, 6], [0, 2], [7, 1], [7, 4], [6, 6], [7, 2], [5, 0], [6, 3], [6, 4], [7, 5], [7, 2]] pl = [ch1, ch2] if pl in l: if (self.pts[6][1] > self.pts[8][1] and self.pts[10][1] > self.pts[12][1] and self.pts[14][1] > self.pts[16][1] and self.pts[18][1] > self.pts[20][ 1]): ch1 = 1 print("111111") # con for [f][pqz] l = [[6, 1], [6, 0], [0, 3], [6, 4], [2, 2], [0, 6], [6, 2], [7, 6], [4, 6], [4, 1], [4, 2], [0, 2], [7, 1], [7, 4], [6, 6], [7, 2], [7, 5], [7, 2]] pl = [ch1, ch2] if pl in l: if (self.pts[6][1] < self.pts[8][1] and self.pts[10][1] > self.pts[12][1] and self.pts[14][1] > self.pts[16][1] and self.pts[18][1] > self.pts[20][1]): ch1 = 1 print("111112") l = [[6, 1], [6, 0], [4, 2], [4, 1], [4, 6], [4, 4]] pl = [ch1, ch2] if pl in l: if (self.pts[10][1] > self.pts[12][1] and self.pts[14][1] > self.pts[16][1] and self.pts[18][1] > self.pts[20][1]): ch1 = 1 print("111112") # con for [d][pqz] fg = 19 # print("_________________ch1=",ch1," ch2=",ch2) l = [[5, 0], [3, 4], [3, 0], [3, 1], [3, 5], [5, 5], [5, 4], [5, 1], [7, 6]] pl = [ch1, ch2] if pl in l: if ((self.pts[6][1] > self.pts[8][1] and self.pts[10][1] < self.pts[12][1] and self.pts[14][1] < self.pts[16][1] and self.pts[18][1] < self.pts[20][1]) and (self.pts[2][0] < self.pts[0][0]) and self.pts[4][1] > self.pts[14][1]): ch1 = 1 print("111113") l = [[4, 1], [4, 2], [4, 4]] pl = [ch1, ch2] if pl in l: if (self.distance(self.pts[4], self.pts[11]) < 50) and ( self.pts[6][1] > self.pts[8][1] and self.pts[10][1] < self.pts[12][1] and self.pts[14][1] < self.pts[16][1] and self.pts[18][1] < self.pts[20][1]): ch1 = 1 print("1111993") l = [[3, 4], [3, 0], [3, 1], [3, 5], [3, 6]] pl = [ch1, ch2] if pl in l: if ((self.pts[6][1] > self.pts[8][1] and self.pts[10][1] < self.pts[12][1] and self.pts[14][1] < self.pts[16][1] and self.pts[18][1] < self.pts[20][1]) and (self.pts[2][0] < self.pts[0][0]) and self.pts[14][1] < self.pts[4][1]): ch1 = 1 print("1111mmm3") l = [[6, 6], [6, 4], [6, 1], [6, 2]] pl = [ch1, ch2] if pl in l: if self.pts[5][0] - self.pts[4][0] - 15 < 0: ch1 = 1 print("1111140") # con for [i][pqz] l = [[5, 4], [5, 5], [5, 1], [0, 3], [0, 7], [5, 0], [0, 2], [6, 2], [7, 5], [7, 1], [7, 6], [7, 7]] pl = [ch1, ch2] if pl in l: if ((self.pts[6][1] < self.pts[8][1] and self.pts[10][1] < self.pts[12][1] and self.pts[14][1] < self.pts[16][1] and self.pts[18][1] > self.pts[20][1])): ch1 = 1 print("111114") # con for [yj][bfdi] l = [[1, 5], [1, 7], [1, 1], [1, 6], [1, 3], [1, 0]] pl = [ch1, ch2] if pl in l: if (self.pts[4][0] < self.pts[5][0] + 15) and ( (self.pts[6][1] < self.pts[8][1] and self.pts[10][1] < self.pts[12][1] and self.pts[14][1] < self.pts[16][1] and self.pts[18][1] > self.pts[20][1])): ch1 = 7 print("111114lll;;p") # con for [uvr] l = [[5, 5], [5, 0], [5, 4], [5, 1], [4, 6], [4, 1], [7, 6], [3, 0], [3, 5]] pl = [ch1, ch2] if pl in l: if ((self.pts[6][1] > self.pts[8][1] and self.pts[10][1] > self.pts[12][1] and self.pts[14][1] < self.pts[16][1] and self.pts[18][1] < self.pts[20][1])) and self.pts[4][1] > self.pts[14][1]: ch1 = 1 print("111115") # con for [w] fg = 13 l = [[3, 5], [3, 0], [3, 6], [5, 1], [4, 1], [2, 0], [5, 0], [5, 5]] pl = [ch1, ch2] if pl in l: if not (self.pts[0][0] + fg < self.pts[8][0] and self.pts[0][0] + fg < self.pts[12][0] and self.pts[0][0] + fg < self.pts[16][0] and self.pts[0][0] + fg < self.pts[20][0]) and not ( self.pts[0][0] > self.pts[8][0] and self.pts[0][0] > self.pts[12][0] and self.pts[0][0] > self.pts[16][0] and self.pts[0][0] > self.pts[20][ 0]) and self.distance(self.pts[4], self.pts[11]) < 50: ch1 = 1 print("111116") # con for [w] l = [[5, 0], [5, 5], [0, 1]] pl = [ch1, ch2] if pl in l: if self.pts[6][1] > self.pts[8][1] and self.pts[10][1] > self.pts[12][1] and self.pts[14][1] > self.pts[16][1]: ch1 = 1 print("1117") # -------------------------condn for 8 groups ends # -------------------------condn for subgroups starts # if ch1 == 0: ch1 = 'A' elif ch1 == 1: ch1 = 'B' elif ch1 == 2: ch1 = 'C' # ... continue with your mapping return ch1 def process_frame(self, image): try: # Convert PIL image to numpy array if isinstance(image, np.ndarray): frame = image else: frame = np.array(image) # Flip image horizontally frame = cv2.flip(frame, 1) frame_copy = frame.copy() # Detect hands hands = hd.findHands(frame, draw=False, flipType=True) if hands: hand = hands[0] x, y, w, h = hand['bbox'] # Extract hand region y_start = max(0, y - offset) y_end = min(frame.shape[0], y + h + offset) x_start = max(0, x - offset) x_end = min(frame.shape[1], x + w + offset) hand_image = frame_copy[y_start:y_end, x_start:x_end] if hand_image.size > 0: # Load white background white = cv2.imread("white.jpg") if white is None: white = np.ones((400, 400, 3), dtype=np.uint8) * 255 # Detect hand landmarks handz = hd2.findHands(hand_image, draw=False, flipType=True) if handz: hand = handz[0] pts = hand['lmList'] if len(pts) >= 21: # Calculate offset os = ((400 - w) // 2) - 15 os1 = ((400 - h) // 2) - 15 # Draw landmarks on white background for t in range(0, 4, 1): cv2.line(white, (pts[t][0] + os, pts[t][1] + os1), (pts[t + 1][0] + os, pts[t + 1][1] + os1), (0, 255, 0), 3) # ... add all your drawing code here for i in range(21): cv2.circle(white, (pts[i][0] + os, pts[i][1] + os1), 2, (0, 0, 255), 1) # Get prediction character = self.predict(white, pts) # Update sentence logic if character == "next" and self.prev_char != "next": if len(self.ten_prev_char) > 0: prev_char = self.ten_prev_char[(self.count-2) % 10] if prev_char != "next" and prev_char != "Backspace": self.str += prev_char self.current_symbol = character self.prev_char = character self.count += 1 self.ten_prev_char[self.count % 10] = character # Show result return white, character, self.str else: return frame, "No hand detected", self.str else: return frame, "No hand landmarks", self.str else: return frame, "Hand region invalid", self.str else: return frame, "No hand detected", self.str except Exception as e: print(f"Error: {e}") traceback.print_exc() return frame, "Error processing", self.str # Initialize recognizer recognizer = SignLanguageRecognizer() # Create Gradio interface def recognize_sign(image): if image is None: return None, "No image", " " processed_image, character, sentence = recognizer.process_frame(image) return processed_image, character, sentence # Create interface iface = gr.Interface( fn=recognize_sign, inputs=gr.Image(type="pil", label="Upload or Capture Hand Sign"), outputs=[ gr.Image(type="numpy", label="Processed Hand Landmarks"), gr.Textbox(label="Recognized Character", lines=1), gr.Textbox(label="Sentence", lines=3) ], title="Sign Language Recognition System", description="Convert hand signs to text using CNN and MediaPipe. Show your hand sign to the camera and get real-time recognition!", examples=[], cache_examples=False, live=True # Enable real-time processing ) # Launch the app if __name__ == "__main__": iface.launch(share=True)