import streamlit as st import numpy as np import cv2 import tensorflow as tf from tensorflow.keras.models import load_model import pandas as pd from io import BytesIO import base64 import matplotlib.pyplot as plt from reportlab.lib.pagesizes import letter from reportlab.pdfgen import canvas from concurrent.futures import ThreadPoolExecutor import urllib.parse import json import random import os # Set page configuration st.set_page_config(page_title="Indian Sign Language Classifier", page_icon="🤟", layout="wide") # Define paths MODEL_PATH = "C:/Users/Cherukuri Gowtham/OneDrive/project/model.keras" # Update with your actual model path DATASET_PATH = "C:\\Users\\Cherukuri Gowtham\\OneDrive\\project\\isl dataset\\Indian" # Verify this path exists # Load the trained model try: model = load_model(MODEL_PATH) except Exception as e: st.error(f"Error loading model: {e}") st.stop() # Define class labels class_labels = ['1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'] # Translations for prediction output and learning mode translations = { 'en': { 'prediction_text': "The predicted sign is 🤟: {sign}", 'confidence_text': "Confidence: {confidence:.2%}", 'description_text': "Description: Sign {sign} represents the {type} {sign} in Indian Sign Language.", 'top_3_text': "Top 3 Suggestions:", 'top_3_item': "- {sign}: {confidence:.2%}", 'learning_text': "Practice Sign: {sign}", 'learning_description': "Sign {sign} is the {type} {sign} in Indian Sign Language.", }, 'hi': { 'prediction_text': "अनुमानित संकेत है 🤟: {sign}", 'confidence_text': "आत्मविश्वास: {confidence:.2%}", 'description_text': "विवरण: संकेत {sign} भारतीय सांकेतिक भाषा में {type} {sign} को दर्शाता है।", 'top_3_text': "शीर्ष 3 सुझाव:", 'top_3_item': "- {sign}: {confidence:.2%}", 'learning_text': "अभ्यास संकेत: {sign}", 'learning_description': "संकेत {sign} भारतीय सांकेतिक भाषा में {type} {sign} है।", }, 'ta': { 'prediction_text': "கணிக்கப்பட்ட குறியீடு 🤟: {sign}", 'confidence_text': "நம்பிக்கை: {confidence:.2%}", 'description_text': "விளக்கம்: குறியீடு {sign} இந்திய சைகை மொழியில் {type} {sign} ஐ பிரதிநிதித்துவப்படுத்துகிறது。", 'top_3_text': "முதல் 3 பரிந்துரைகள்:", 'top_3_item': "- {sign}: {confidence:.2%}", 'learning_text': "பயிற்சி குறியீடு: {sign}", 'learning_description': "குறியீடு {sign} இந்திய சைகை மொழியில் {type} {sign} ஆகும்。", }, 'te': { 'prediction_text': "అంచనా వేసిన సంజ్ఞ 🤟: {sign}", 'confidence_text': "విశ్వాసం: {confidence:.2%}", 'description_text': "వివరణ: సంజ్ఞ {sign} భారతీయ సంజ్ఞా భాషలో {type} {sign} ని సూచిస్తుంది。", 'top_3_text': "టాప్ 3 సూచనలు:", 'top_3_item': "- {sign}: {confidence:.2%}", 'learning_text': "అభ్యాస సంజ్ఞ: {sign}", 'learning_description': "సంజ్ఞ {sign} భారతీయ సంజ్ఞా భాషలో {type} {sign} గా ఉంది।", }, 'bn': { 'prediction_text': "পূর্বাভাসিত সংকেত 🤟: {sign}", 'confidence_text': "আত্মবিশ্বাস: {confidence:.2%}", 'description_text': "বর্ণনা: সংকেত {sign} ভারতীয় সংকেত ভাষায় {type} {sign} প্রতিনিধিত্ব করে।", 'top_3_text': "শীর্ষ 3 পরামর্শ:", 'top_3_item': "- {sign}: {confidence:.2%}", 'learning_text': "অভ্যাস সংকেত: {sign}", 'learning_description': "সংকেত {sign} ভারতীয় সংকেত ভাষায় {type} {sign} হিসেবে প্রতিনিধিত্ব করে।", }, 'mr': { 'prediction_text': "अंदाजित संकेत आहे 🤟: {sign}", 'confidence_text': "आत्मविश्वास: {confidence:.2%}", 'description_text': "वर्णन: संकेत {sign} भारतीय संकेत भाषेत {type} {sign} दर्शवितो।", 'top_3_text': "शीर्ष 3 सूचना:", 'top_3_item': "- {sign}: {confidence:.2%}", 'learning_text': "सराव संकेत: {sign}", 'learning_description': "संकेत {sign} भारतीय संकेत भाषेत {type} {sign} आहे।", }, 'gu': { 'prediction_text': "આગાહી કરેલ સંકેત છે 🤟: {sign}", 'confidence_text': "આત્મવિશ્વાસ: {confidence:.2%}", 'description_text': "વર્ણન: સંકેત {sign} ભારતીય સંકેત ભાષામાં {type} {sign} નું પ્રતિનિધિત્વ કરે છે।", 'top_3_text': "ટોચના 3 સૂચનો:", 'top_3_item': "- {sign}: {confidence:.2%}", 'learning_text': "અભ્યાસ સંકેત: {sign}", 'learning_description': "સંકેત {sign} ભારતીય સંકેત ભાષામાં {type} {sign} છે।", }, 'kn': { 'prediction_text': "ಊಹಿಸಲಾದ ಸಂಕೇತ 🤟: {sign}", 'confidence_text': "ವಿಶ್ವಾಸ: {confidence:.2%}", 'description_text': "ವಿವರಣೆ: ಸಂಕೇತ {sign} ಭಾರತೀಯ ಸಂಕೇತ ಭಾಷೆಯಲ್ಲಿ {type} {sign} ಗೆ ಪ್ರತಿನಿಧಿಯಾಗಿರುತ್ತದೆ।", 'top_3_text': "ಟಾಪ್ 3 ಸಲಹೆಗಳು:", 'top_3_item': "- {sign}: {confidence:.2%}", 'learning_text': "ಅಭ್ಯಾಸ ಸಂಕೇತ: {sign}", 'learning_description': "ಸಂಕೇತ {sign} ಭಾರತೀಯ ಸಂಕೇತ ಭಾಷೆಯಲ್ಲಿ {type} {sign} ಆಗಿದೆ।", }, 'ml': { 'prediction_text': "പ്രവചിച്ച ആംഗ്യം 🤟: {sign}", 'confidence_text': "ആത്മവിശ്വാസം: {confidence:.2%}", 'description_text': "വിവരണം: ആംഗ്യം {sign} ഇന്ത്യൻ ആംഗ്യഭാഷയിൽ {type} {sign} നെ പ്രതിനിധീകരിക്കുന്നു।", 'top_3_text': "മികച്ച 3 നിർദ്ദേശങ്ങൾ:", 'top_3_item': "- {sign}: {confidence:.2%}", 'learning_text': "പരിശീലന ആംഗ്യം: {sign}", 'learning_description': "ആംഗ്യം {sign} ഇന്ത്യൻ ആംഗ്യഭাষയിൽ {type} {sign} ആണ്。", }, 'pa': { 'prediction_text': "ਅੰਦਾਜ਼ਾ ਲਗਾਇਆ ਸੰਕੇਤ 🤟: {sign}", 'confidence_text': "ਵਿਸ਼ਵਾਸ: {confidence:.2%}", 'description_text': "ਵੇਰਵਾ: ਸੰਕੇਤ {sign} ਭਾਰਤੀ ਸੰਕੇਤ ਭਾਸ਼ਾ ਵਿੱਚ {type} {sign} ਨੂੰ ਦਰਸਾਉਂਦਾ ਹੈ।", 'top_3_text': "ਸਿਖਰ ਦੇ 3 ਸੁਝਾਅ:", 'top_3_item': "- {sign}: {confidence:.2%}", 'learning_text': "ਅਭਿਆਸ ਸੰਕੇਤ: {sign}", 'learning_description': "ਸੰਕੇਤ {sign} ਭਾਰਤੀ ਸੰਕੇਤ ਭਾਸ਼ਾ ਵਿੱਚ {type} {sign} ਹੈ।", }, 'or': { 'prediction_text': "ପୂର୍ବାନୁମାନିତ ଚିହ୍ନ 🤟: {sign}", 'confidence_text': "ଆତ୍ମବିଶ୍ୱାସ: {confidence:.2%}", 'description_text': "ବିବରଣୀ: ଚିହ୍ନ {sign} ଭାରତୀୟ ସଙ୍କେତ ଭାଷାରେ {type} {sign} କୁ ପ୍ରତିନିଧିତ୍ୱ କରେ।", 'top_3_text': "ଶୀର୍ଷ 3 ପରାମର୍ଶ:", 'top_3_item': "- {sign}: {confidence:.2%}", 'learning_text': "ଅଭ୍ୟାସ ଚିହ୍ନ: {sign}", 'learning_description': "ଚିହ୍ନ {sign} ଭାରତୀୟ ସଙ୍କେତ ଭାଷାରେ {type} {sign} ଅଟେ।", }, 'as': { 'prediction_text': "পূৰ্বাভাস কৰা সংকেত 🤟: {sign}", 'confidence_text': "আত্মবিশ্বাস: {confidence:.2%}", 'description_text': "বিৱৰণ: সংকেত {sign} ভাৰতীয় সংকেত ভাষাত {type} {sign} ক প্ৰতিনিধিত্ব কৰে।", 'top_3_text': "শীৰ্ষ ৩ পৰামৰ্শ:", 'top_3_item': "- {sign}: {confidence:.2%}", 'learning_text': "অভ্যাস সংকেত: {sign}", 'learning_description': "সংকেত {sign} ভাৰতীয় সংকেত ভাষাত {type} {sign} হয়।", } } # Preprocess image with caching @st.cache_data def preprocess_image(image, target_size=(64, 62)): image_resized = cv2.resize(image, target_size) image_preprocessed = tf.keras.preprocessing.image.img_to_array(image_resized) / 255.0 return image_resized, image_preprocessed # Load sign image with caching @st.cache_data def load_sign_image(sign): # Debugging: Show attempted paths st.write(f"Attempting to load image for sign '{sign}' from {DATASET_PATH}") # Try subfolder structure (e.g., DATASET_PATH/A/image.png) subfolder_path = os.path.join(DATASET_PATH, sign) if os.path.isdir(subfolder_path): st.write(f"Found subfolder: {subfolder_path}") for ext in ['png', 'jpg', 'jpeg']: images = [f for f in os.listdir(subfolder_path) if f.lower().endswith(f'.{ext}')] if images: image_path = os.path.join(subfolder_path, images[0]) st.write(f"Selected image: {image_path}") return image_path st.write(f"No images found in {subfolder_path}") # Try alternative subfolder names (e.g., letter_A, 0 for numbers) alt_subfolder = f"letter_{sign}" if sign.isalpha() else str(int(sign) - 1) if sign.isdigit() else sign alt_subfolder_path = os.path.join(DATASET_PATH, alt_subfolder) if os.path.isdir(alt_subfolder_path): st.write(f"Found alternative subfolder: {alt_subfolder_path}") for ext in ['png', 'jpg', 'jpeg']: images = [f for f in os.listdir(alt_subfolder_path) if f.lower().endswith(f'.{ext}')] if images: image_path = os.path.join(alt_subfolder_path, images[0]) st.write(f"Selected image: {image_path}") return image_path st.write(f"No images found in {alt_subfolder_path}") # Try single image (e.g., DATASET_PATH/A.png) for ext in ['png', 'jpg', 'jpeg']: image_path = os.path.join(DATASET_PATH, f"{sign}.{ext}") if os.path.exists(image_path): st.write(f"Found single image: {image_path}") return image_path # Try case-insensitive subfolder for folder in os.listdir(DATASET_PATH): if folder.lower() == sign.lower() and os.path.isdir(os.path.join(DATASET_PATH, folder)): subfolder_path = os.path.join(DATASET_PATH, folder) st.write(f"Found case-insensitive subfolder: {subfolder_path}") for ext in ['png', 'jpg', 'jpeg']: images = [f for f in os.listdir(subfolder_path) if f.lower().endswith(f'.{ext}')] if images: image_path = os.path.join(subfolder_path, images[0]) st.write(f"Selected image: {image_path}") return image_path st.write(f"No image found for sign '{sign}'") return None # Convert image to base64 def image_to_base64(image): image_bgr = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) _, buffer = cv2.imencode('.png', image_bgr) return base64.b64encode(buffer).decode('utf-8') # Generate PDF report (English for simplicity) def generate_pdf_report(df): buffer = BytesIO() c = canvas.Canvas(buffer, pagesize=letter) c.setFont("Helvetica", 12) c.drawString(100, 750, "Indian Sign Language Prediction Report") y = 700 for _, row in df.iterrows(): c.drawString(100, y, f"Image: {row['Image']}") c.drawString(100, y-20, f"Predicted Sign: {row['Predicted Sign']}") c.drawString(100, y-40, f"Confidence: {row['Confidence']:.2%}") y -= 60 c.save() buffer.seek(0) return buffer # Visualization function def generate_visualization(df): if df.empty: st.warning("No predictions to visualize. Please upload images first.") return chart_type = st.selectbox("Select Chart Type", ["Bar Chart", "Pie Chart", "Confidence Trend"], key="chart_type") if chart_type == "Bar Chart": sign_counts = df["Predicted Sign"].value_counts() fig, ax = plt.subplots() ax.bar(sign_counts.index, sign_counts.values) ax.set_title("Prediction Distribution (Bar Chart)") ax.set_xlabel("Signs") ax.set_ylabel("Count") plt.xticks(rotation=45) st.pyplot(fig) elif chart_type == "Pie Chart": sign_counts = df["Predicted Sign"].value_counts() fig, ax = plt.subplots() ax.pie(sign_counts.values, labels=sign_counts.index, autopct='%1.1f%%', startangle=90) ax.axis('equal') ax.set_title("Prediction Distribution (Pie Chart)") st.pyplot(fig) else: # Confidence Trend selected_sign = st.selectbox("Select Sign for Confidence Trend", options=class_labels, key="trend_sign") trend_df = df[df["Predicted Sign"] == selected_sign][["Confidence"]].reset_index(drop=True) if trend_df.empty: st.warning(f"No predictions for sign {selected_sign}.") return fig, ax = plt.subplots() ax.plot(trend_df.index, trend_df["Confidence"], marker='o') ax.set_title(f"Confidence Trend for Sign {selected_sign}") ax.set_xlabel("Prediction Instance") ax.set_ylabel("Confidence") ax.grid(True) st.pyplot(fig) # CSS for accessible, visual-first styling st.markdown(""" """, unsafe_allow_html=True) # Header st.markdown("

Indian Sign Language Classifier

", unsafe_allow_html=True) # Sidebar st.sidebar.header("Settings & Instructions") st.sidebar.markdown(""" 1. Select the language for prediction output and learning content (English by default). 2. Use the Image Upload tab to classify signs. 3. Adjust settings for image processing. 4. Set confidence threshold for predictions. 5. Search prediction history by sign, image name, or confidence in the Image Upload tab. 6. Use the Visualization tab for prediction distribution or confidence trends. 7. Export results as CSV or PDF in the Image Upload tab. 8. Practice signs with images from the dataset in the Learning tab. 9. Provide feedback in the Feedback tab. """) language_options = { 'en': 'English', 'hi': 'Hindi', 'ta': 'Tamil', 'te': 'Telugu', 'bn': 'Bengali', 'mr': 'Marathi', 'gu': 'Gujarati', 'kn': 'Kannada', 'ml': 'Malayalam', 'pa': 'Punjabi', 'or': 'Odia', 'as': 'Assamese' } if 'selected_language' not in st.session_state: st.session_state.selected_language = 'en' selected_language = st.sidebar.selectbox( "Prediction Language", options=list(language_options.values()), index=list(language_options.keys()).index(st.session_state.selected_language), help="Choose the language for prediction output and learning content" ) st.session_state.selected_language = list(language_options.keys())[list(language_options.values()).index(selected_language)] theme = st.sidebar.selectbox("Theme", ["Light", "Dark", "High Contrast"], help="Choose a theme for better visibility") if theme == "Dark": st.markdown("", unsafe_allow_html=True) elif theme == "High Contrast": st.markdown("", unsafe_allow_html=True) target_size = st.sidebar.slider("Target Image Height (px)", 16, 64, 64, step=8, help="Set image height (width fixed at 62)") target_size = (target_size, 62) confidence_threshold = st.sidebar.slider("Minimum Confidence Threshold", 0.0, 1.0, 0.5, 0.05, help="Filter low-confidence predictions") # Initialize session state if 'predictions_df' not in st.session_state: st.session_state.predictions_df = pd.DataFrame(columns=["Image", "Predicted Sign", "Confidence", "Image Base64"]) if 'current_sign' not in st.session_state: st.session_state.current_sign = random.choice(class_labels) # Process single image def process_single_image(uploaded_file, target_size, confidence_threshold): try: file_bytes = np.asarray(bytearray(uploaded_file.read()), dtype=np.uint8) image = cv2.imdecode(file_bytes, cv2.IMREAD_COLOR) image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) image_resized, image_preprocessed = preprocess_image(image, target_size) prediction = model.predict(np.expand_dims(image_preprocessed, axis=0), verbose=0) predicted_class = class_labels[np.argmax(prediction)] confidence = np.max(prediction) top_3 = np.argsort(prediction[0])[-3:][::-1] top_3_signs = [(class_labels[i], prediction[0][i]) for i in top_3] if confidence >= confidence_threshold: image_base64 = image_to_base64(image_resized) return { "Image": uploaded_file.name, "Predicted Sign": predicted_class, "Confidence": confidence, "Image Base64": image_base64, "Top 3 Signs": top_3_signs } else: return {"error": f"Prediction for {uploaded_file.name} below confidence threshold ({confidence:.2%} < {confidence_threshold:.2%})"} except Exception as e: return {"error": f"Error processing {uploaded_file.name}: {e}"} # Tabs tab1, tab2, tab3, tab4 = st.tabs(["Image Upload", "Visualization", "Feedback", "Learning"]) # Tab 1: Image Upload with tab1: st.subheader("Classify Signs") uploaded_files = st.file_uploader("Upload image(s)", type=["jpg", "png", "jpeg"], accept_multiple_files=True, help="Upload images of signs") if st.button("Reset History", help="Clear all predictions"): st.session_state.predictions_df = pd.DataFrame(columns=["Image", "Predicted Sign", "Confidence", "Image Base64"]) st.success("Prediction history reset.") st.rerun() if uploaded_files: st.subheader("Prediction Results") progress_bar = st.progress(0) total_files = len(uploaded_files) with ThreadPoolExecutor() as executor: results = list(executor.map(lambda f: process_single_image(f, target_size, confidence_threshold), uploaded_files)) for i, result in enumerate(results): if "error" not in result: new_row = pd.DataFrame([{k: v for k, v in result.items() if k != "Top 3 Signs"}]) st.session_state.predictions_df = pd.concat([st.session_state.predictions_df, new_row], ignore_index=True) # Display prediction in the selected language lang = st.session_state.selected_language sign_type = "number" if result['Predicted Sign'].isdigit() else "letter" st.markdown(f"""

{translations[lang]['prediction_text'].format(sign=result['Predicted Sign'])}

{translations[lang]['confidence_text'].format(confidence=result['Confidence'])}

""", unsafe_allow_html=True) st.markdown(translations[lang]['description_text'].format(sign=result['Predicted Sign'], type=sign_type)) st.markdown(translations[lang]['top_3_text']) for s, c in result["Top 3 Signs"]: st.markdown(translations[lang]['top_3_item'].format(sign=s, confidence=c)) else: st.error(result["error"]) progress_bar.progress((i + 1) / total_files) if not st.session_state.predictions_df.empty: st.subheader("Prediction Summary") st.markdown("**Search Prediction History**") search_query = st.text_input("Search by sign, image name, or confidence (e.g., 'A', 'image1.jpg', '0.9')", "", help="Enter a sign, image name, or confidence value") filter_sign = st.multiselect("Filter by Predicted Sign", options=class_labels, default=[], help="Filter predictions by sign") filtered_df = st.session_state.predictions_df if filter_sign: filtered_df = filtered_df[filtered_df["Predicted Sign"].isin(filter_sign)] if search_query: try: confidence_search = float(search_query) if search_query.replace('.', '', 1).isdigit() else None filtered_df = filtered_df[ (filtered_df["Predicted Sign"].str.contains(search_query, case=False)) | (filtered_df["Image"].str.contains(search_query, case=False)) | (filtered_df["Confidence"].apply(lambda x: abs(x - confidence_search) < 0.05) if confidence_search is not None else False) ] except ValueError: filtered_df = filtered_df[ (filtered_df["Predicted Sign"].str.contains(search_query, case=False)) | (filtered_df["Image"].str.contains(search_query, case=False)) ] selected_row = st.dataframe( filtered_df[["Image", "Predicted Sign", "Confidence"]].style.format({"Confidence": "{:.2%}"}), on_select="rerun", selection_mode="single-row", use_container_width=True ) if selected_row["selection"]["rows"]: idx = selected_row["selection"]["rows"][0] row = filtered_df.iloc[idx] st.image(base64.b64decode(row["Image Base64"]), caption="Processed Image", width=200) st.markdown(f"**Sign**: {row['Predicted Sign']}") st.markdown(f"**Confidence**: {row['Confidence']:.2%}") st.markdown(f"**Description**: Sign {row['Predicted Sign']} represents the {'number' if row['Predicted Sign'].isdigit() else 'letter'} {row['Predicted Sign']} in Indian Sign Language.") st.subheader("Export Results") col1, col2, col3 = st.columns(3) with col1: st.download_button( label="Download Predictions as CSV", data=st.session_state.predictions_df[["Image", "Predicted Sign", "Confidence"]].to_csv(index=False).encode('utf-8'), file_name="predictions.csv", mime="text/csv", help="Download predictions as CSV" ) with col2: st.download_button( label="Download PDF Report", data=generate_pdf_report(st.session_state.predictions_df), file_name="isl_report.pdf", mime="application/pdf", help="Download predictions as PDF" ) with col3: if st.button("Share Prediction", help="Share the latest prediction"): latest_prediction = st.session_state.predictions_df.iloc[-1].to_dict() prediction_json = json.dumps(latest_prediction) encoded = urllib.parse.quote(prediction_json) share_url = f"{st.get_option('server.baseUrlPath')}?prediction={encoded}" st.markdown(f"Share this prediction: [Link]({share_url})") # Tab 2: Visualization with tab2: st.subheader("Prediction History Visualization") generate_visualization(st.session_state.predictions_df) # Tab 3: Feedback with tab3: st.subheader("Feedback") with st.form("feedback_form"): st.markdown("Help us improve the app!") rating = st.slider("Rate this app (1-5) ⭐", 1, 5, help="Rate your experience") comments = st.text_area("Comments 💬", help="Share your thoughts (processed in English)") submitted = st.form_submit_button("Submit Feedback") if submitted: st.success("Thank you for your feedback!") with open("feedback.txt", "a") as f: f.write(f"Rating: {rating}, Comments: {comments}\n") # Tab 4: Learning with tab4: st.subheader("Sign Learning Mode") st.markdown("Practice Indian Sign Language signs by viewing images and descriptions from the dataset.") lang = st.session_state.selected_language sign = st.session_state.current_sign sign_type = "number" if sign.isdigit() else "letter" # Load and display sign image image_path = load_sign_image(sign) if image_path: st.image( image_path, caption=f"Indian Sign Language Sign: {sign}", width=200, use_column_width=False, output_format="auto", clamp=True, channels="RGB" ) else: st.warning(f"Image for sign {sign} not found in {DATASET_PATH}. Ensure a subfolder '{sign}' or file '{sign}.png/jpg' exists.") # Display text description st.markdown(f"""

{translations[lang]['learning_text'].format(sign=sign)}

{translations[lang]['learning_description'].format(sign=sign, type=sign_type)}

""", unsafe_allow_html=True) if st.button("Show New Sign", help="Display a new random sign"): st.session_state.current_sign = random.choice(class_labels) st.rerun() # Footer st.markdown("
Powered by Streamlit, TensorFlow, OpenCV.
", unsafe_allow_html=True)