Spaces:
Sleeping
Sleeping
| import os # For file operations and path handling | |
| # Retrieve API keys directly from Hugging Face Secrets | |
| groq_api_key = os.getenv('GROQ_API_KEY') | |
| elevenlabs_api_key = os.getenv('ELEVENLABS_API_KEY') | |
| # Import necessary modules | |
| import os # For file operations and path handling | |
| import gradio as gr # Gradio for web UI | |
| import tensorflow as tf # TensorFlow for model loading and inference | |
| from tensorflow import keras # Keras API for model operations | |
| import numpy as np # Numerical operations on arrays | |
| from PIL import Image # Image processing | |
| from deep_translator import GoogleTranslator # For translating doctor responses | |
| from fpdf import FPDF # For generating PDF reports | |
| import datetime # For timestamping PDF filenames | |
| import threading # For delayed shutdown in exit tab | |
| import matplotlib.pyplot as plt # For plotting prediction bar charts | |
| # Custom modules for AI processing | |
| from brain_of_the_doctor import encode_image, analyze_image_with_query # Functions for LLM-based insights | |
| from voice_of_the_patient import transcribe_with_groq # For speech-to-text conversion | |
| from voice_of_the_doctor import text_to_speech_with_gtts # For text-to-speech responses | |
| # Load the trained models | |
| def load_skin_disease_model(): | |
| print("Loading Skin Disease Model...") | |
| model = keras.models.load_model("skin_disease_model.keras") | |
| print("Skin Disease Model loaded successfully!") | |
| return model | |
| def load_resnet_model(): | |
| print("Loading ResNet Model...") | |
| model = keras.models.load_model("ResNet_model.keras") | |
| print("ResNet Model loaded successfully!") | |
| return model | |
| def load_densenet_model(): | |
| print("Loading DenseNet121 Model...") | |
| model = keras.models.load_model("DenseNet121_skin_disease_model.keras") | |
| print("DenseNet121 Model loaded successfully!") | |
| return model | |
| # Instantiate models at startup | |
| skin_disease_model = load_skin_disease_model() | |
| resnet_model = load_resnet_model() | |
| densenet_model = load_densenet_model() | |
| # Define class names for predictions | |
| class_names = ["Acne", "Eczema", "Melanoma", "Unknown"] | |
| # Mapping of supported languages to codes and font files | |
| LANGUAGE_CODES = { | |
| "English": ("en", "NotoSans-Regular.ttf"), | |
| "Hindi": ("hi", "NotoSansDevanagari-Regular.ttf"), | |
| "Bengali": ("bn", "NotoSansBengali-Regular.ttf"), | |
| "Tamil": ("ta", "NotoSansTamil-Regular.ttf"), | |
| "Telugu": ("te", "NotoSansTelugu-Regular.ttf"), | |
| "Marathi": ("mr", "NotoSansDevanagari-Regular.ttf") | |
| } | |
| # --- UPDATE: Explanation-only prompt replaces classification prompt --- | |
| explanation_prompt = """ | |
| You are a dermatologist AI. Your task is to classify the given skin image as *Acne, Eczema, Melanoma, or Unknown*. | |
| 🔹 *Rules for your response:* | |
| 1️⃣ Strictly return the classification using *only one* of these options: | |
| - Acne | |
| - Eczema | |
| - Melanoma | |
| - Unknown (if condition does not match any of the above) | |
| 2️⃣ Follow this exact response format: | |
| --- | |
| Possible Skin Condition: (Acne, Eczema, Melanoma, or Unknown) | |
| Reason: (Briefly explain the visible symptoms in the image.) | |
| Medical Explanation: (Describe the causes and contributing factors of the condition.) | |
| Causes: (Describe the causes) | |
| Symptoms: (Comma-or-bullet list of common symptoms) | |
| Recommendation: (Simple advice; do not mention any product names, or next steps.) | |
| --- | |
| """ | |
| # Image preprocessing function | |
| def preprocess_image(image_path): | |
| print(f"Processing image: {image_path}") | |
| image = Image.open(image_path).convert("RGB").resize((224, 224)) | |
| arr = np.array(image, dtype=np.float32) / 255.0 | |
| image_tensor = np.expand_dims(arr, axis=0) | |
| print("Image preprocessed successfully!") | |
| return image_tensor | |
| # Grad-CAM helper function | |
| def get_gradcam_heatmap(model, img_array, last_conv_layer_name=None): | |
| if last_conv_layer_name is None: | |
| for layer in reversed(model.layers): | |
| if isinstance(layer, tf.keras.layers.Conv2D): | |
| last_conv_layer_name = layer.name | |
| break | |
| if last_conv_layer_name is None: | |
| raise ValueError("No convolutional layer found in the model.") | |
| print(f"Using last conv layer for Grad-CAM: {last_conv_layer_name}") | |
| grad_model = tf.keras.models.Model( | |
| inputs=model.inputs, | |
| outputs=[model.get_layer(last_conv_layer_name).output, model.output] | |
| ) | |
| with tf.GradientTape() as tape: | |
| conv_outputs, predictions = grad_model(img_array) | |
| loss = predictions[:, tf.argmax(predictions[0])] | |
| grads = tape.gradient(loss, conv_outputs) | |
| pooled_grads = tf.reduce_mean(grads, axis=(0, 1, 2)) | |
| conv_outputs = conv_outputs[0] | |
| heatmap = conv_outputs @ pooled_grads[..., tf.newaxis] | |
| heatmap = tf.squeeze(heatmap) | |
| heatmap = tf.maximum(heatmap, 0) / (tf.math.reduce_max(heatmap) + 1e-8) | |
| return heatmap.numpy() | |
| # Generate PDF report with Unicode support | |
| def generate_pdf(doctor_response, skin_disease_prediction, language): | |
| lang_code, font_file = LANGUAGE_CODES.get(language, ("en", "NotoSans-Regular.ttf")) | |
| font_path = os.path.join(".", font_file) | |
| pdf = FPDF() | |
| pdf.add_page() | |
| pdf.set_auto_page_break(auto=True, margin=15) | |
| if not os.path.exists(font_path): | |
| print(f"Font file missing: {font_path}") | |
| return None | |
| pdf.add_font('LangFont', '', font_path, uni=True) | |
| pdf.set_font('LangFont', '', 16) | |
| pdf.cell(200, 10, 'MediMind AI - Diagnosis Report', ln=True, align='C') | |
| pdf.ln(10) | |
| pdf.set_font('LangFont', '', 12) | |
| pdf.cell(200, 10, 'Skin Disease Prediction:', ln=True) | |
| pdf.multi_cell(0, 10, skin_disease_prediction) | |
| pdf.ln(5) | |
| pdf.cell(200, 10, "Doctor's Response:", ln=True) | |
| pdf.multi_cell(0, 10, doctor_response) | |
| pdf.ln(5) | |
| timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") | |
| pdf_output_path = f"skin_disease_report_{timestamp}.pdf" | |
| pdf.output(pdf_output_path) | |
| return pdf_output_path | |
| # Main processing function for Gradio interface | |
| def process_inputs(image_filepath, language, model_choice, prediction_mode): | |
| doctor_response = "" | |
| skin_disease_prediction = "" | |
| pdf_path = "" | |
| predicted_class = "Unknown" | |
| confidence = 0.0 | |
| if image_filepath: | |
| try: | |
| img_tensor = preprocess_image(image_filepath) | |
| # Choose model based on user selection | |
| if prediction_mode == "Single Model Prediction": | |
| if model_choice == "MobileNetV3": | |
| preds = skin_disease_model.predict(img_tensor)[0] | |
| model = skin_disease_model | |
| elif model_choice == "ResNet Model": | |
| preds = resnet_model.predict(img_tensor)[0] | |
| model = resnet_model | |
| else: | |
| preds = densenet_model.predict(img_tensor)[0] | |
| model = densenet_model | |
| confidence = float(np.max(preds) * 100) | |
| predicted_class = class_names[int(np.argmax(preds))] | |
| if confidence < 60: | |
| predicted_class = "Unknown" | |
| skin_disease_prediction = ( | |
| "I cannot confidently identify this condition. Please consult a dermatologist." | |
| if predicted_class == "Unknown" | |
| else f"Disease: {predicted_class} (Confidence: {confidence:.2f}%)" | |
| ) | |
| else: | |
| results = [] | |
| for name, mdl in [("MobileNetV3", skin_disease_model), ("ResNet Model", resnet_model), ("DenseNet Model", densenet_model)]: | |
| p = mdl.predict(img_tensor)[0] | |
| c = float(np.max(p) * 100) | |
| cl = class_names[int(np.argmax(p))] | |
| if c < 60: | |
| cl = "Unknown" | |
| results.append((name, cl, f"{c:.2f}%")) | |
| table = ( | |
| "| Model | Predicted Class | Confidence |\n" | |
| "|-------|------------------|------------|\n" | |
| ) | |
| for r in results: | |
| table += f"| {r[0]} | {r[1]} | {r[2]} |\n" | |
| skin_disease_prediction = table | |
| model = skin_disease_model | |
| preds = model.predict(img_tensor)[0] | |
| predicted_class = class_names[int(np.argmax(preds))] | |
| except Exception as e: | |
| skin_disease_prediction = f"Error in classification: {str(e)}" | |
| # --- Grad-CAM and bar chart --- | |
| heatmap = get_gradcam_heatmap(model, img_tensor) | |
| heatmap_img = Image.fromarray((heatmap * 255).astype('uint8')).resize((224, 224)).convert('RGB') | |
| orig = Image.open(image_filepath).resize((224, 224)) | |
| overlay = Image.blend(orig, heatmap_img, alpha=0.4) | |
| fig, ax = plt.subplots() | |
| ax.bar(class_names, preds * 100) | |
| ax.set_ylabel('Confidence (%)') | |
| fig.tight_layout() | |
| # --- UPDATE: Construct explanation-only query --- | |
| if predicted_class == "Unknown": | |
| doctor_query = "This condition does not match any known skin diseases in my database. Please consult a dermatologist." | |
| else: | |
| doctor_query = f"{explanation_prompt}\nCondition: {predicted_class}. Please provide a medical explanation and recommendation for this condition." | |
| # Get LLM explanation (no classification) | |
| doctor_response = analyze_image_with_query( | |
| query=doctor_query, | |
| encoded_image=encode_image(image_filepath), | |
| model="meta-llama/llama-4-scout-17b-16e-instruct" | |
| ) | |
| # Translate doctor response if needed | |
| if language != "English": | |
| lang_codes = {"Hindi":"hi","Bengali":"bn","Telugu":"te","Marathi":"mr","Tamil":"ta"} | |
| target_lang = lang_codes.get(language, "en") | |
| doctor_response = GoogleTranslator(source='en', target=target_lang).translate(doctor_response) | |
| # Convert text response to speech | |
| voice_of_doctor = text_to_speech_with_gtts(input_text=doctor_response) | |
| # Generate downloadable PDF report | |
| pdf_path = generate_pdf(doctor_response, skin_disease_prediction, language) | |
| return ( | |
| doctor_response, | |
| voice_of_doctor, | |
| skin_disease_prediction, | |
| pdf_path, | |
| fig, | |
| overlay, | |
| round(confidence, 2) | |
| ) | |
| # Define Gradio interfaces | |
| main_app = gr.Interface( | |
| fn=process_inputs, | |
| inputs=[ | |
| gr.Image(type="filepath", label="Upload Skin Image"), | |
| gr.Radio(list(LANGUAGE_CODES.keys()), value="English", label="Select Language"), | |
| gr.Radio(["MobileNetV3", "ResNet Model", "DenseNet Model"], value="MobileNetV3", label="Select Model"), | |
| gr.Radio(["Single Model Prediction", "Compare All Models"], value="Single Model Prediction", label="Prediction Mode") | |
| ], | |
| outputs=[ | |
| gr.Textbox(label="MediMind AI Response"), | |
| gr.Audio(label="MediMind AI Voice"), | |
| gr.Textbox(label="MediMind AI Skin Disease Prediction"), | |
| gr.File(label="Download PDF Report"), | |
| gr.Plot(label="Prediction Probabilities"), | |
| gr.Image(label="Grad-CAM Heatmap Overlay"), | |
| gr.Slider(0, 100, step=0.1, label="Confidence Meter (%)") | |
| ], | |
| title="MediMind AI" | |
| ) | |
| about_text = """ | |
| # About MediMind AI | |
| **MediMind AI** is an intelligent dermatology assistant designed to help users identify potential skin conditions through advanced deep learning models. | |
| It combines powerful AI models, explainable Grad-CAM visualizations and multilingual support in an easy-to-use interface. | |
| ### Key Features: | |
| - **Skin Disease Prediction:** Detects Acne, Eczema, Melanoma, or Unknown conditions using MobileNetV3, ResNet, and DenseNet121 models. | |
| - **Explainable AI:** Grad-CAM heatmaps to visually explain model predictions. | |
| - **AI Doctor Consultation:** Generates AI-based diagnosis explanations and treatment recommendations. | |
| - **Multilingual Support:** Provides results in English, Hindi, Bengali, Tamil, Telugu, and Marathi. | |
| - **Voice Interaction:** Text-to-speech responses to enhance user accessibility. | |
| - **Downloadable Report:** PDF diagnosis reports with detailed findings and advice. | |
| --- | |
| **Developed with ❤️ using AI and Deep Learning.** | |
| """ | |
| about_app = gr.Interface( | |
| fn=lambda: about_text, | |
| inputs=[], | |
| outputs=[gr.Markdown()], | |
| title="About MediMind AI" | |
| ) | |
| # Help Tab | |
| help_text = """ | |
| # How to Use MediMind AI | |
| 1. Upload a clear skin image... | |
| 2. Select language... | |
| 3. Choose model and mode... | |
| 4. Click Submit and view results. | |
| """ | |
| help_app = gr.Interface(fn=lambda: help_text, inputs=[], outputs=[gr.Markdown()], title="Help") | |
| # Feedback Tab | |
| import pandas as pd | |
| import re | |
| # Directory & file setup | |
| feedback_dir = "feedbacks" | |
| feedback_file = os.path.join(feedback_dir, "feedback.csv") | |
| os.makedirs(feedback_dir, exist_ok=True) | |
| # Basic spam/bad word filtering | |
| BAD_WORDS = ["spam", "abuse", "hack", "fraud", "scam", "attack", "hate"] | |
| def is_spam(message): | |
| message_lower = message.lower() | |
| for word in BAD_WORDS: | |
| if word in message_lower: | |
| return True | |
| if len(message.strip()) < 10: # too short | |
| return True | |
| return False | |
| # Main feedback submit function | |
| def submit_feedback(name, email, message): | |
| if is_spam(message): | |
| return "⚠️ Your feedback seems suspicious or too short. Please provide valid feedback." | |
| ts = datetime.datetime.now().isoformat() | |
| # Create or append feedback | |
| feedback_entry = { | |
| "timestamp": ts, | |
| "name": name if name else "Anonymous", | |
| "email": email if email else "Not Provided", | |
| "message": message.strip() | |
| } | |
| try: | |
| if not os.path.exists(feedback_file): | |
| # Create new file | |
| df = pd.DataFrame([feedback_entry]) | |
| else: | |
| # Append to existing | |
| df_existing = pd.read_csv(feedback_file) | |
| df = pd.concat([df_existing, pd.DataFrame([feedback_entry])], ignore_index=True) | |
| df.to_csv(feedback_file, index=False) | |
| return "✅ Thank you for your feedback! We appreciate your time." | |
| except Exception as e: | |
| return f"❌ Error saving feedback: {str(e)}" | |
| # Gradio Interface | |
| feedback_app = gr.Interface( | |
| fn=submit_feedback, | |
| inputs=[ | |
| gr.Textbox(label="Name (optional)"), | |
| gr.Textbox(label="Email (optional)"), | |
| gr.Textbox(label="Your Feedback", lines=5, placeholder="Write your feedback in detail...") | |
| ], | |
| outputs=gr.Textbox(label="Response"), | |
| title="📝 Submit Your Feedback", | |
| theme="default", | |
| css="body {background-color: #f7f7f7;}" | |
| ) | |
| # Exit Tab (Client-side window close) | |
| def exit_app_fn(): | |
| # Delay shutdown slightly to allow response | |
| threading.Timer(0.5, lambda: os._exit(0)).start() | |
| return "Exiting application..." | |
| exit_app = gr.Interface( | |
| fn=exit_app_fn, | |
| inputs=[], | |
| outputs=[gr.Textbox(label="Status")], | |
| title="Exit" | |
| ) | |
| # Combine in Tabs | |
| app = gr.TabbedInterface( | |
| [main_app, about_app, help_app, feedback_app, exit_app], | |
| ["MediMind AI","About","Help","Feedback","Exit"] | |
| ) | |
| if __name__ == "__main__": | |
| app.launch(debug=True) # Launch the Gradio app in debug mode | |