""" ISL Sign Language Translation - TechMatrix Solvers Initiative Main Streamlit Application Developed by: TechMatrix Solvers Team - Abhay Gupta (Team Lead) - contact2abhaygupta6187@gmail.com - Kripanshu Gupta (Backend Developer) - guptakripanshu83@gmail.com - Dipanshu Patel (UI/UX Designer) - dipanshupatel43@gmail.com - Bhumika Patel (Deployment & Female Presenter) - bp7249951@gmail.com Institution: Shri Ram Group of Institutions """ import streamlit as st import sys import os # Add parent directory to path for imports sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # Configure Streamlit page first st.set_page_config( page_title="ISL Translation - TechMatrix Solvers", page_icon="🤟", layout="wide", initial_sidebar_state="expanded", menu_items={ 'Get Help': 'https://docs.google.com/document/d/1mzr2KGHRJT5heUjFF20NQ3Gb89urpjZJ/edit?usp=sharing', 'Report a bug': "mailto:contact2abhaygupta6187@gmail.com", 'About': "# TechMatrix Solvers ISL Translation System\nAdvanced Indian Sign Language translation using deep learning!" } ) # Initialize session state if 'app_initialized' not in st.session_state: st.session_state.app_initialized = False # Show loading message loading_placeholder = st.empty() with loading_placeholder: st.markdown("""

🚀 TechMatrix Solvers

ISL Translation System

Loading advanced AI models...

""", unsafe_allow_html=True) # Set environment variables before any imports os.environ["KERAS_BACKEND"] = "torch" os.environ["OPENCV_LOG_LEVEL"] = "ERROR" os.environ["QT_QPA_PLATFORM"] = "offscreen" # For headless OpenCV # Import dependencies with comprehensive error handling try: import numpy as np import pandas as pd import time import tempfile import json import uuid import platform import shutil from PIL import Image from typing import NamedTuple import subprocess # Import OpenCV with headless configuration and fallback cv2 = None try: import cv2 # Ensure OpenCV uses headless backend if hasattr(cv2, 'setNumThreads'): cv2.setNumThreads(1) # Test basic functionality _ = cv2.__version__ except ImportError as opencv_error: loading_placeholder.empty() st.error(f"❌ OpenCV import failed: {opencv_error}") st.error("Please run the setup script: `setup_windows.bat` or `./setup_windows.ps1`") st.info("💡 Alternative: pip uninstall opencv-python && pip install opencv-python-headless") st.stop() except Exception as opencv_error: loading_placeholder.empty() st.error(f"❌ OpenCV configuration error: {opencv_error}") st.error("This might be a headless display issue. Please use opencv-python-headless.") st.info("💡 Try: pip install opencv-python-headless==4.8.1.78") st.stop() # Deep learning imports import keras from keras.models import Sequential from keras.layers import LSTM, Dense, Bidirectional, Dropout, Input, BatchNormalization # Video processing import ffmpeg # HuggingFace from huggingface_hub import hf_hub_download # Custom imports from pose_models import create_bodypose_model, create_handpose_model from expression_mapping import expression_mapping from isl_processor import ISLTranslationModel import pose_utils as utils # Clear loading message and show success loading_placeholder.empty() st.success("✅ TechMatrix Solvers ISL System loaded successfully!") st.session_state.app_initialized = True except ImportError as e: loading_placeholder.empty() st.error(f"❌ Failed to load dependencies: {e}") st.error("Please ensure all required packages are installed.") st.stop() except Exception as e: loading_placeholder.empty() st.error(f"❌ Unexpected error during initialization: {e}") st.stop() # Main application layout st.markdown(""" """, unsafe_allow_html=True) # Header st.markdown("""

🤟 ISL Sign Language Translation

TechMatrix Solvers Initiative

Advanced AI-Powered Indian Sign Language Recognition & Translation

""", unsafe_allow_html=True) # Sidebar configuration st.sidebar.markdown("""

🚀 TechMatrix Solvers

Innovating Accessible Technology

""", unsafe_allow_html=True) st.sidebar.title('🎛️ Control Panel') # Team information in sidebar with st.sidebar.expander("👨‍💻 Meet Our Team"): st.markdown(""" **TechMatrix Solvers Team:** - **Abhay Gupta** - Team Lead 👑 - **Kripanshu Gupta** - Backend Dev 💻 - **Dipanshu Patel** - UI/UX Designer 🎨 - **Bhumika Patel** - Deployment 🚀 *Shri Ram Group of Institutions* """) # Application mode selection app_mode = st.sidebar.selectbox( '🎯 Choose Application Mode', ['🏠 Home & About', '🎥 Live Translation Demo', '📊 System Information'], index=0 ) # Utility functions class VideoProbeResult(NamedTuple): """Structure for video probe results""" return_code: int json: str error: str def probe_video_info(file_path) -> VideoProbeResult: """Probe video file for metadata using FFprobe""" command_array = [ "ffprobe", "-v", "quiet", "-print_format", "json", "-show_format", "-show_streams", file_path ] try: result = subprocess.run( command_array, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=30 ) return VideoProbeResult( return_code=result.returncode, json=result.stdout, error=result.stderr ) except subprocess.TimeoutExpired: return VideoProbeResult(1, "", "FFprobe timeout") @st.cache_data def load_test_data(): """Load test dataset and file information""" try: with st.spinner("📥 Loading test data from HuggingFace..."): testing_cleaned_path = hf_hub_download( repo_id="sunilsarolkar/isl-test-data", filename="testing_cleaned.csv", repo_type="dataset" ) test_files_path = hf_hub_download( repo_id="sunilsarolkar/isl-test-data", filename="test_files.csv", repo_type="dataset" ) testing_df = pd.read_csv(testing_cleaned_path) test_files_df = pd.read_csv(test_files_path) return testing_df, test_files_df except Exception as e: st.error(f"Failed to load test data: {e}") return None, None @st.cache_resource def load_translation_model(): """Load and configure the LSTM translation model""" try: with st.spinner("🧠 Loading AI translation model..."): model = Sequential() model.add(Input(shape=((20, 156)))) model.add(keras.layers.Masking(mask_value=0.)) model.add(BatchNormalization()) model.add(Bidirectional(LSTM(32, recurrent_dropout=0.2, return_sequences=True))) model.add(Dropout(0.2)) model.add(Bidirectional(LSTM(32, recurrent_dropout=0.2))) model.add(keras.layers.Activation('elu')) model.add(Dense(32, use_bias=False, kernel_initializer='he_normal')) model.add(BatchNormalization()) model.add(Dropout(0.2)) model.add(keras.layers.Activation('elu')) model.add(Dense(32, kernel_initializer='he_normal', use_bias=False)) model.add(BatchNormalization()) model.add(keras.layers.Activation('elu')) model.add(Dropout(0.2)) model.add(Dense(len(list(expression_mapping.keys())), activation='softmax')) # Download pre-trained weights model_file = hf_hub_download( repo_id="sunilsarolkar/isl-translation-model", filename="isl_model_final.keras" ) model.load_weights(model_file) return model except Exception as e: st.error(f"Failed to load translation model: {e}") return None # Main application logic if app_mode == '🏠 Home & About': # Project overview st.markdown(""" ## 🎯 Project Overview Welcome to the **ISL Sign Language Translation System** by **TechMatrix Solvers**! This cutting-edge application demonstrates real-time Indian Sign Language recognition and translation using advanced deep learning techniques. """) # Features showcase col1, col2, col3 = st.columns(3) with col1: st.markdown("""

🧠 AI-Powered

Advanced LSTM networks with 82K+ parameters for accurate sign recognition

""", unsafe_allow_html=True) with col2: st.markdown("""

⚡ Real-time

Live video processing with OpenPose body & hand detection

""", unsafe_allow_html=True) with col3: st.markdown("""

🎯 Accurate

Trained on INCLUDE dataset with 263 words across 15 categories

""", unsafe_allow_html=True) # Technical architecture st.markdown("## 🏗️ Technical Architecture") col1, col2 = st.columns(2) with col1: st.markdown(""" ### 🔧 Core Components 1. **Body Pose Estimation**: 25-point skeletal tracking 2. **Hand Landmark Detection**: 21-point hand keypoints 3. **Temporal Modeling**: Bidirectional LSTM networks 4. **Real-time Processing**: Optimized inference pipeline """) # Display architecture diagram if available try: st.image("DataPipeline.png", caption="🔄 Data Processing Pipeline", use_column_width=True) except: st.info("📊 Architecture diagram will be displayed when available") with col2: st.markdown(""" ### 📊 Dataset Statistics """) # Dataset stats stats_data = { "Metric": ["Categories", "Total Words", "Training Videos", "Avg Videos/Class", "Frame Rate"], "Value": ["15", "263", "4,257", "16.3", "25fps"] } st.table(pd.DataFrame(stats_data)) # Model stats st.markdown(""" ### 🤖 Model Statistics - **Architecture**: Bidirectional LSTM - **Parameters**: 82,679 (322.96 KB) - **Input Features**: 156-dimensional vectors - **Temporal Window**: 20 frames """) # Team section st.markdown("## 👥 TechMatrix Solvers Team") team_data = { "Name": ["Abhay Gupta", "Kripanshu Gupta", "Dipanshu Patel", "Bhumika Patel"], "Role": ["Team Lead", "Backend Developer", "UI/UX Designer", "Deployment & Presenter"], "Email": [ "contact2abhaygupta6187@gmail.com", "guptakripanshu83@gmail.com", "dipanshupatel43@gmail.com", "bp7249951@gmail.com" ], "Phone": ["8115814535", "7067058400", "9294526404", "9302271422"] } st.table(pd.DataFrame(team_data)) st.markdown(""" ### 🏫 Institution **Shri Ram Group of Institutions** ### 📚 Documentation For detailed technical documentation, visit our [comprehensive guide](https://docs.google.com/document/d/1mzr2KGHRJT5heUjFF20NQ3Gb89urpjZJ/edit?usp=sharing). """) elif app_mode == '🎥 Live Translation Demo': st.markdown("## 🎥 Live ISL Translation Demo") # Load test data testing_df, test_files_df = load_test_data() if testing_df is None or test_files_df is None: st.error("❌ Cannot proceed without test data. Please check your internet connection.") st.stop() st.success(f"✅ Loaded {len(testing_df)} test frames and {len(test_files_df)} video files") # Video selection interface st.markdown("### 📁 Select Test Video") col1, col2, col3 = st.columns(3) with col1: category = st.selectbox( 'Choose Category', sorted(test_files_df['Category'].unique()) ) # Filter by category category_filtered = test_files_df[test_files_df['Category'] == category] with col2: class_name = st.selectbox( 'Choose Class', sorted(category_filtered['Class'].unique()) ) # Filter by class class_filtered = category_filtered[category_filtered['Class'] == class_name] with col3: filename = st.selectbox( 'Choose File', sorted(class_filtered['Filename'].unique()) ) # Display selection info st.info(f"📂 Selected: **{category}** → **{class_name}** → **{filename}**") # Translation button if st.button("🚀 Start ISL Translation", type="primary", use_container_width=True): # Initialize progress progress_bar = st.progress(0) status_text = st.empty() try: status_text.text("🔄 Preparing translation model...") progress_bar.progress(10) # Load translation model translation_model = load_translation_model() if translation_model is None: st.error("❌ Failed to load translation model") st.stop() progress_bar.progress(30) status_text.text("📥 Downloading test video...") # Download video video_file_path = hf_hub_download( repo_id="sunilsarolkar/isl-test-data", filename=f'test/{category}/{class_name}/{filename}', repo_type="dataset" ) progress_bar.progress(50) status_text.text("🎬 Processing video metadata...") # Process video if os.path.exists(video_file_path): progress_bar.progress(70) status_text.text("✨ Ready for translation!") # Display video info st.success("🎉 Video loaded successfully!") st.video(video_file_path) # Show translation results placeholder st.markdown("### 📊 Translation Results") result_placeholder = st.empty() with result_placeholder.container(): st.info("🔄 Translation would be processed here in the full implementation") st.markdown(""" **Expected Output:** - Real-time pose detection - Feature extraction from body and hand keypoints - LSTM model prediction - Sign language translation display """) progress_bar.progress(100) status_text.text("✅ Translation demo completed!") else: st.error("❌ Video file not found") except Exception as e: st.error(f"❌ Error during translation: {e}") progress_bar.progress(0) status_text.text("❌ Translation failed") elif app_mode == '📊 System Information': st.markdown("## 📊 System Information") # System details col1, col2 = st.columns(2) with col1: st.markdown("### 💻 Environment") st.write(f"**Python Version:** {platform.python_version()}") st.write(f"**Platform:** {platform.system()} {platform.release()}") st.write(f"**Architecture:** {platform.machine()}") st.write(f"**FFmpeg:** {shutil.which('ffmpeg') or 'Not found'}") st.write(f"**FFprobe:** {shutil.which('ffprobe') or 'Not found'}") with col2: st.markdown("### 📚 Libraries") try: st.write(f"**OpenCV:** {cv2.__version__}") except: st.write("**OpenCV:** Not available") try: import torch st.write(f"**PyTorch:** {torch.__version__}") except: st.write("**PyTorch:** Not available") try: st.write(f"**Keras:** {keras.__version__}") except: st.write("**Keras:** Not available") try: st.write(f"**NumPy:** {np.__version__}") st.write(f"**Pandas:** {pd.__version__}") except: st.write("**NumPy/Pandas:** Version info not available") # Model information st.markdown("### 🧠 AI Model Details") model_info = { "Component": [ "Translation Model", "Body Pose Model", "Hand Pose Model", "Feature Vector", "Temporal Window", "Output Classes" ], "Specification": [ "Bidirectional LSTM", "OpenPose 25-point", "OpenPose 21-point", "156 dimensions", "20 frames", f"{len(expression_mapping)} signs" ] } st.table(pd.DataFrame(model_info)) # Performance metrics st.markdown("### ⚡ Performance Metrics") metrics_col1, metrics_col2, metrics_col3, metrics_col4 = st.columns(4) with metrics_col1: st.metric("Model Parameters", "82,679", "Compact & Efficient") with metrics_col2: st.metric("Model Size", "322.96 KB", "Lightweight") with metrics_col3: st.metric("Input Features", "156", "Rich Feature Set") with metrics_col4: st.metric("Sign Classes", len(expression_mapping), "Comprehensive") # Footer st.markdown("---") st.markdown("""

🚀 TechMatrix Solvers

Innovating Accessible Technology Solutions for Everyone

Shri Ram Group of Institutions | © 2024

📚 Documentation | 📧 Contact | 🤗 HF Spaces

""", unsafe_allow_html=True)