#Modified by Augmented Startups 2021 #Face Landmark User Interface with StreamLit #Watch Computer Vision Tutorials at www.augmentedstartups.info/YouTube import streamlit as st from transformer_comparison import render_transformer_comparison app_mode = st.sidebar.selectbox( "Choose the App mode", ["Transformer comparison", "Run on Test Videos", "About App"], ) if app_mode == "Transformer comparison": render_transformer_comparison() st.stop() import os os.environ["KERAS_BACKEND"] = "torch" import keras import cv2 import numpy as np import tempfile import time from PIL import Image from keras.models import Sequential import os from keras.models import Sequential import pickle import keras from keras.models import Sequential import os from keras.layers import LSTM, Dense, Bidirectional, Dropout,Input,BatchNormalization from model import handpose_model, bodypose_25_model from expression_mapping import expression_mapping from ISL_Model_parameter import ISLSignPosTranslator import pandas as pd import numpy as np import ffmpeg import subprocess from typing import NamedTuple import json import util from huggingface_hub import hf_hub_download import shutil, platform, subprocess import uuid st.write("Python:", platform.python_version()) st.write("FFmpeg exists:", shutil.which("ffmpeg"), "FFprobe:", shutil.which("ffprobe")) try: import cv2 st.write("OpenCV version:", cv2.__version__) except Exception as e: st.error(f"OpenCV import failed: {e}") try: import torch, keras st.write("Torch:", torch.__version__, "Keras:", keras.__version__) except Exception as e: st.error(f"Torch/Keras import failed: {e}") class FFProbeResult(NamedTuple): return_code: int json: str error: str def ffprobe(file_path) -> FFProbeResult: command_array = ["ffprobe", "-v", "quiet", "-print_format", "json", "-show_format", "-show_streams", file_path] result = subprocess.run(command_array, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) return FFProbeResult(return_code=result.returncode, json=result.stdout, error=result.stderr) X_body_test = [f'bodypeaks_x_{i}' for i in range(15)] + [f'bodypeaks_y_{i}' for i in range(15)] X_hand0_test = [f'hand0peaks_x_{i}' for i in range(21)] + [f'hand0peaks_y_{i}' for i in range(21)] + [f'hand0peaks_peaktxt{i}' for i in range(21)] X_hand1_test = [f'hand1peaks_x_{i}' for i in range(21)] + [f'hand1peaks_y_{i}' for i in range(21)] + [f'hand1peaks_peaktxt{i}' for i in range(21)] feature_columns_new = X_body_test + X_hand0_test + X_hand1_test label_columns = ['Expression_encoded'] @st.cache_resource def create_timeseries_data(isl_data,feature_columns,label_columns, window_size=20): """ Creates timeseries data from a DataFrame with a specified window size and padding at the end. Args: df (pandas.DataFrame): The input DataFrame. window_size (int, optional): The window size for creating timeseries data. Defaults to 20. pad_value (any, optional): The value to use for padding at the end. Defaults to None. Returns: list: A list of lists, where each inner list represents a window of timeseries data. """ # Handle empty DataFrame if isl_data.empty: return [],[] X=[] y=[] i=0 for group, file_df in isl_data.groupby(['Type','Expression_encoded','FileName']): expr_types,exprs,filepaths=group # print('expr_types,exprs,filepaths',(expr_types,exprs,filepaths)) # print(type(name)) # Get the rolling window iterator with padding first_frame=np.zeros((1,156)) for idx,x in enumerate([file_df[i:i+window_size] for i in range(0,file_df.shape[0],1)]):#enumerate(file_df.rolling(window=20, step=20,min_periods=1)): # print(f'records processed {idx} of {file_df.shape[0]}') # print(f"{filepaths}-Frame#{x['Frame'].values}/{file_df['Frame'].max()}") if x.shape[0]4: # break # i=i+1 # if i>4: # break return X,y translation_model=None @st.cache_resource def get_translator_model(): translation_model = Sequential() translation_model.add(Input(shape=((20, 156)))) translation_model.add(keras.layers.Masking(mask_value=0.)) translation_model.add(BatchNormalization()) translation_model.add(Bidirectional(LSTM(32, recurrent_dropout=0.2, return_sequences=True))) translation_model.add(Dropout(0.2)) translation_model.add(Bidirectional(LSTM(32, recurrent_dropout=0.2))) translation_model.add(keras.layers.Activation('elu')) translation_model.add(Dense(32, use_bias=False, kernel_initializer='he_normal')) translation_model.add(BatchNormalization()) translation_model.add(Dropout(0.2)) translation_model.add(keras.layers.Activation('elu')) translation_model.add(Dense(32, kernel_initializer='he_normal',use_bias=False)) translation_model.add(BatchNormalization()) translation_model.add(keras.layers.Activation('elu')) translation_model.add(Dropout(0.2)) translation_model.add(Dense(len(list(expression_mapping.keys())), activation='softmax')) model_file = hf_hub_download( repo_id="sunilsarolkar/isl-translation-model", filename="isl_model_final.keras" ) translation_model.load_weights(model_file) translation_model.load_weights(model_file) return translation_model testing_cleaned = hf_hub_download( repo_id="sunilsarolkar/isl-test-data", filename="testing_cleaned.csv", repo_type="dataset" ) testing_df=pd.read_csv(testing_cleaned) test_files = hf_hub_download( repo_id="sunilsarolkar/isl-test-data", filename="test_files.csv", repo_type="dataset" ) # test_statistic_df=pd.read_csv('test_statistic.csv') test_files_df=pd.read_csv(test_files) # mp_drawing = mp.solutions.drawing_utils # mp_face_mesh = mp.solutions.face_mesh class Writer(): def __init__(self, output_file, input_fps, input_framesize, input_pix_fmt, input_vcodec): # if os.path.exists(output_file): # os.remove(output_file) self.ff_proc = ( ffmpeg .input('pipe:', format='rawvideo', pix_fmt="bgr24", s='%sx%s'%(input_framesize[1],input_framesize[0]), r=input_fps) .output(output_file, pix_fmt=input_pix_fmt, vcodec=input_vcodec) .overwrite_output() .run_async(pipe_stdin=True) ) def __call__(self, frame): self.ff_proc.stdin.write(frame.tobytes()) def close(self): self.ff_proc.stdin.close() self.ff_proc.wait() st.title("SignLanguageTranslate") st.header("Indian Sign Language · legacy LSTM") st.markdown( """ """, unsafe_allow_html=True, ) st.sidebar.title('ISL Sign Language Translation using Openpose') st.sidebar.subheader('Parameters') frame_wise_outputs={} def weighted_average(nums, weights): if sum(weights)==0: return 0 return sum(x * y for x, y in zip(nums, weights)) / sum(weights) @st.cache_data def image_resize(image, width=None, height=None, inter=cv2.INTER_AREA): # initialize the dimensions of the image to be resized and # grab the image size dim = None (h, w) = image.shape[:2] # if both the width and height are None, then return the # original image if width is None and height is None: return image # check to see if the width is None if width is None: # calculate the ratio of the height and construct the # dimensions r = height / float(h) dim = (int(w * r), height) # otherwise, the height is None else: # calculate the ratio of the width and construct the # dimensions r = width / float(w) dim = (width, int(h * r)) # resize the image resized = cv2.resize(image, dim, interpolation=inter) # return the resized image return resized if app_mode =='About App': st.markdown('In this application we are demonstrating model developed for translating the Indian Sign Language(ISL) using LSTM') st.markdown( """ """, unsafe_allow_html=True, ) # st.video('https://www.youtube.com/watch?v=FMaNNXgB_5c&ab_channel=AugmentedStartups') tab1, tab2, tab3, tab4 = st.tabs(["Dataset Overview", "Data Pipeline", "Model Architecture", "Training Details"]) with tab1: st.markdown(''' # Dataset Used \n This model is trained using [INCLUDE](https://zenodo.org/records/4010759) dataset. \n ### Key Statistics for the dataset is as follows- +-----------------------+-----------------+ | Characteristics | INCLUDE-DATASET | +-----------------------+-----------------+ | Categories | 15 | | Words | 263 | | Videos | 4257 | | Avg Videos per class | 16.3 | | Avg Video Length | 2.57s | | Min Video Length | 1.28s | | Max Video Length | 6.16s | | Frame Rate | 25fps | | Resolution | 1920x1080 | +-----------------------+-----------------+ #### Size of each category +--------------------+-------------------+------------------+ | Category | Number of Classes | Number of Videos | +--------------------+-------------------+------------------+ | Adjectives | 59 | 791 | | Animals | 8 | 166 | | Clothes | 10 | 198 | | Colours | 11 | 222 | | Days and Time | 22 | 306 | | Electronics | 10 | 140 | | Greetings | 9 | 185 | | Means of Transport | 9 | 186 | | Objects at Home | 27 | 379 | | Occupations | 16 | 225 | | People | 26 | 513 | | Places | 19 | 399 | | Pronouns | 8 | 168 | | Seasons | 6 | 85 | | Society | 23 | 324 | | | Categories# 263 | Total Videos-4287| +--------------------+-------------------+------------------+ ''') st.info("💡 **Note:** The dataset used for training contains **1986 processed videos out of 4287**. We processed a limited set of records due to time and compute constraints.") with tab2: st.markdown(''' ### Data Pipeline The pipeline processes video frames to extract pose and hand landmarks using an OpenPose-like approach. For each frame, the feature extraction process produces a vector of **156 features**, consisting of: - **Body Pose**: X and Y coordinates for 15 body keypoints, along with edge lengths and angles. - **Hand Pose**: X and Y coordinates for 21 keypoints on each hand (left and right). This structured tabular data is then grouped sequentially into sliding windows to capture the temporal motion of the signs. ''') with tab3: st.markdown(''' ### Model Structure The translation model utilizes a sequence-to-sequence architecture based on Bidirectional LSTMs. ```python translation_model = Sequential() translation_model.add(Input(shape=((20, 156)))) translation_model.add(keras.layers.Masking(mask_value=0.)) translation_model.add(BatchNormalization()) translation_model.add(Bidirectional(LSTM(32, recurrent_dropout=0.2, return_sequences=True))) translation_model.add(Dropout(0.2)) translation_model.add(Bidirectional(LSTM(32, recurrent_dropout=0.2))) translation_model.add(keras.layers.Activation('elu')) translation_model.add(Dense(32, use_bias=False, kernel_initializer='he_normal')) translation_model.add(BatchNormalization()) translation_model.add(Dropout(0.2)) translation_model.add(keras.layers.Activation('elu')) translation_model.add(Dense(32, kernel_initializer='he_normal',use_bias=False)) translation_model.add(BatchNormalization()) translation_model.add(keras.layers.Activation('elu')) translation_model.add(Dropout(0.2)) translation_model.add(Dense(len(list(expression_mapping.keys())), activation='softmax')) isl_translator=ISLSignPosTranslator(bodypose_25_model(),handpose_model(), translation_model) ``` **Parameters:** - Total params: 82,679 (322.96 KB) - Trainable params: 82,239 (321.25 KB) - Non-trainable params: 440 (1.72 KB) ''') with tab4: st.markdown(''' ### Training Details The model was trained using the **Keras 3 API with a PyTorch backend**. **Dataset Size:** - **123,743 individual frames** were extracted from the processed videos and used as the training set. **Sequence Windowing:** - The temporal data is structured into windows of **20 frames** per sequence (`shape=(20, 156)`), allowing the Bidirectional LSTMs to learn the motion context of the signs. **Metrics & Logs:** - You can view the detailed training progression on [Tensorboard](https://huggingface.co/cdsteameight/ISL-SignLanguageTranslation/tensorboard). ''') elif app_mode =='Run on Test Videos': st.markdown(''' ## How to use the Test Videos Follow these steps to run a prediction on a test video: 1. Open the **sidebar** on the left. 2. Select a **Category** from the dropdown. 3. Select a specific **Class** (the sign language word you want to test). 4. Select a specific **File** from the available test videos. 5. Click the **"Start"** button at the bottom of the sidebar to run the prediction model on the video. The app will load the frames, process them, and display the predicted translation! --- ''') # placeholder = st.empty() category = st.sidebar.selectbox('Choose Category', np.sort(test_files_df['Category'].unique(), axis=-1, kind='mergesort')) # print(category) mask = (test_files_df['Category']==category) test_files_df_category=test_files_df[mask] cls = st.sidebar.selectbox('Choose Class', np.sort(test_files_df_category['Class'].unique(), axis=-1, kind='mergesort') ) mask = (test_files_df['Class']==cls) filename = st.sidebar.selectbox('Choose File', np.sort(test_files_df_category[mask]['Filename'].unique(), axis=-1, kind='mergesort') ) # print(f'test/{category}/{cls}/{filename}') # mask = (include_df['Filepath'].str.contains(key[0])) & (include_df['type']==key[2]) & (include_df['expression']==key[1]) # stframe = st.empty() if st.sidebar.button("Start", type="primary"): mask = (testing_df['FileName'] == filename) & (testing_df['Type']==category)& (testing_df['Expression']==cls) # filtered_df = current_test_df.sort_ window_size=20 current_test_df=testing_df[mask] if current_test_df.empty: st.error(f"⚠️ No matching rows in testing_df for this selection.{filename} {category} {cls}") st.stop() else: st.success(f"✅ Loaded {current_test_df.shape[0]} frames for this selection.") X_test_filtered,y_test_filtered = create_timeseries_data(current_test_df,feature_columns_new,label_columns,window_size=window_size) # y_filtered_encoded=to_categorical(y_test_filtered, num_classes=len(df['Expression_encoded'].unique())) X_test_filtered=np.array(X_test_filtered) # encoded_translation=model(frame.reshape(1,frame.shape[0],frame.shape[1])) # use_webcam = st.sidebar.button('Use Webcam') # record = st.sidebar.checkbox("Record Video") # if record: # st.checkbox("Recording", value=True) st.sidebar.markdown('---') st.markdown( """ """, unsafe_allow_html=True, ) st.sidebar.markdown('---') st.markdown(' ## Output') runtime_progress = st.empty() with runtime_progress.container(): df1 = pd.DataFrame([['--','--']], columns=['Frames Processed','Detected Class']) my_table = st.table(df1) # kpi1, kpi2 = st.columns(2) # with kpi1: # st.markdown("**Frames Processed**") # kpi1_text = st.markdown(f'0/{current_test_df.shape[0]}') # with kpi2: # st.markdown("**Detected Class**") # kpi2_text = st.markdown("--") view = st.empty() st.markdown("
", unsafe_allow_html=True) stframes = st.empty()#[st.empty() for _ in range(20)] # video_file_buffer = st.sidebar.file_uploader("Upload a video", type=[ "mp4", "mov",'avi','asf', 'm4v' ]) # tfflie = tempfile.NamedTemporaryFile(delete=False) vid_file = hf_hub_download( repo_id="sunilsarolkar/isl-test-data", filename=f'test/{category}/{cls}/{filename}', repo_type="dataset" ) if not os.path.exists(vid_file): st.error(f"⚠️ Video file not found: {f'test/{category}/{cls}/{filename}'}") st.stop() vid = cv2.VideoCapture(vid_file) ffprobe_result = ffprobe(vid_file) info = json.loads(ffprobe_result.json) videoinfo = [i for i in info["streams"] if i["codec_type"] == "video"][0] input_fps = videoinfo["avg_frame_rate"] # input_fps = float(input_fps[0])/float(input_fps[1]) input_pix_fmt = videoinfo["pix_fmt"] input_vcodec = videoinfo["codec_name"] postfix = info["format"]["format_name"].split(",")[0] # print(f'input_vcodec-{input_vcodec}') width = int(vid.get(cv2.CAP_PROP_FRAME_WIDTH)) height = int(vid.get(cv2.CAP_PROP_FRAME_HEIGHT)) fps_input = int(vid.get(cv2.CAP_PROP_FPS)) #codec = cv2.VideoWriter_fourcc(*FLAGS.output_format) # codec = cv2.VideoWriter_fourcc('V','P','0','9') # out = cv2.VideoWriter('output1.mp4', codec, fps_input, (width, height)) # st.sidebar.text('Input Video') # st.sidebar.video(tfflie.name) fps = 0 i = 0 # cap = cv2.VideoCapture(video_file,) totalFrames=int(vid.get(cv2.CAP_PROP_FRAME_COUNT)) window_size=20 # print('current_test_df',current_test_df) # print('totalFrames',totalFrames) window=[] prevTime = 0 postfix = info["format"]["format_name"].split(",")[0] output_file = f"/tmp/output_{uuid.uuid4().hex}.{postfix}" # height = int(vid.get(cv2.CAP_PROP_FRAME_HEIGHT)) fps_input = int(vid.get(cv2.CAP_PROP_FPS)) #codec = cv2.VideoWriter_fourcc(*FLAGS.output_format) # codec = cv2.VideoWriter_fourcc('m','p','4','v') out = None writer=None weighted_avg_dict={} idx=0 try: for _, row in current_test_df.iterrows():#enumerate(file_df.rolling(window=20, step=20,min_periods=1)): # print(f'captured frame#{idx}') if not vid.isOpened(): st.error(f"Could not open video: {vid_file}") else: if(vid.isOpened()): ret, frame = vid.read() if len(window)']], columns=['Frames Processed','Detected Class']) my_table = st.table(df1) window.append(frame) # kpi1_text.write(f"

{idx+1}/{current_test_df.shape[0]}

", unsafe_allow_html=True) # kpi2_text.write(f"

--

", unsafe_allow_html=True) view.image(cv2.imencode('.jpg', canvas_with_plot)[1].tobytes(), use_column_width=True) else: window[:-1] = window[1:] window[-1]=frame translation_model=get_translator_model() # testing_df[] encoded_translation = translation_model(X_test_filtered[idx-20].reshape(1,X_test_filtered[idx-20].shape[0],X_test_filtered[idx-20].shape[1])) encoded_translation=encoded_translation[0].cpu().detach().numpy() sorted_index=np.argsort(encoded_translation)[::-1] maxindex=np.argmax(encoded_translation) top_3_probs = encoded_translation.argsort()[-3:][::-1] # Get indices of top 3 probabilities (descending order) top_3_categories = [expression_mapping[i] for i in top_3_probs] # Convert indices to category names (assuming class_names list exists) top_3_values = encoded_translation[top_3_probs] # Get corresponding probabilities # print(f'{idx} {encoded_translation[maxindex]:0.4f} {maxindex}-{expression_mapping[maxindex]} ')#{[(pi,encoded_translation[pi],expression_mapping[pi]) for pi in sorted_index]} for category, prob in zip(top_3_categories, top_3_values): if category not in frame_wise_outputs: frame_wise_outputs[category]=[] frame_wise_outputs[category].append(prob) current_prob={} for category, prob in zip(top_3_categories, top_3_values): current_prob[category]=prob for key in frame_wise_outputs: weighted_avg_dict[key]=weighted_average(frame_wise_outputs[key],[len(frame_wise_outputs[key]) for i in range(len(frame_wise_outputs[key]))]) sorted_dict = dict(sorted(weighted_avg_dict.items(), key=lambda item: item[1], reverse=True)) canvas=util.drawStickmodel(frame,eval(row['bodypose_circles']),eval(row['bodypose_sticks']),eval(row['handpose_edges']),eval(row['handpose_peaks'])) canvas_with_plot=util.draw_bar_plot_below_image(canvas,current_prob, f'Prediction at frame window({idx-20+1}-{idx+1})',canvas) canvas_with_plot=util.draw_bar_plot_below_image(canvas_with_plot,weighted_avg_dict, f'Weighted avg till window {idx+1}',canvas) canvas_with_plot=util.add_padding_to_bottom(canvas_with_plot,(255,255,255),100) writer(canvas_with_plot) currTime = time.time() fps = 1 / (currTime - prevTime) prevTime = currTime # out.write(frame) # if record: # #st.checkbox("Recording", value=True) # out.write(frame) #Dashboard max_prob = float('-inf') # Initialize with negative infinity max_key = None for exp, prob in weighted_avg_dict.items(): if prob > max_prob: max_prob = prob max_key = exp with runtime_progress.container(): df1 = pd.DataFrame([[f'{idx+1}/{current_test_df.shape[0]}',f'{max_key} ({max_prob*100:.2f}%)']], columns=['Frames Processed','Detected Class']) my_table = st.table(df1) # kpi1_text.write(f"

{idx+1}/{current_test_df.shape[0]}

", unsafe_allow_html=True) # kpi2_text.write(f"

{max_key} ({max_prob*100:.2f}%)

", unsafe_allow_html=True) # with placeholder.container(): # # st.write(weighted_avg_dict) # # data = { # # "I": 0.7350964583456516, # # "Hello": 0.1078806109726429, # # "you": 0.11776176246348768, # # "you (plural)": 0.12685142129916568 # # } # # Convert the dictionary to a Pandas DataFrame for easier plotting # df = pd.DataFrame.from_dict(weighted_avg_dict, orient='index', columns=['Values']) # # Create a bar chart with Streamlit # st.bar_chart(df) # frame = cv2.resize(frame,(0,0),fx = 0.8 , fy = 0.8) # frame = image_resize(image = frame, width = 640) view.image(cv2.imencode('.jpg', canvas_with_plot)[1].tobytes(), use_column_width=True) idx=idx+1 # st.text('Video Processed') with view.container(): if writer is not None: # ✅ safeguard writer.close() output_video = open(output_file,'rb') out_bytes = output_video.read() st.video(out_bytes) print(f'Output file - {output_file}') else: st.warning("No video was processed, writer is empty.") # out.release() print(f'Output file - {output_file}') finally: vid.release() if writer is not None: writer.close() cv2.destroyAllWindows()