Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -1,480 +1,318 @@
|
|
| 1 |
import gradio as gr
|
| 2 |
import librosa
|
| 3 |
import numpy as np
|
|
|
|
|
|
|
|
|
|
| 4 |
import os
|
| 5 |
-
import hashlib
|
| 6 |
from datetime import datetime
|
| 7 |
-
from transformers import pipeline
|
| 8 |
-
import soundfile
|
| 9 |
-
import torch
|
| 10 |
-
from tenacity import retry, stop_after_attempt, wait_fixed
|
| 11 |
import logging
|
|
|
|
|
|
|
|
|
|
| 12 |
import tempfile
|
| 13 |
-
import shutil
|
| 14 |
-
from simple_salesforce import Salesforce
|
| 15 |
-
from dotenv import load_dotenv
|
| 16 |
-
import pyttsx3
|
| 17 |
-
from cryptography.fernet import Fernet
|
| 18 |
-
import asyncio
|
| 19 |
-
import base64
|
| 20 |
-
import traceback
|
| 21 |
|
| 22 |
-
# Set up logging
|
| 23 |
-
logging.basicConfig(
|
| 24 |
-
level=logging.DEBUG,
|
| 25 |
-
format="%(asctime)s - %(levelname)s - %(message)s",
|
| 26 |
-
handlers=[logging.FileHandler("voice_analyzer.log"), logging.StreamHandler()]
|
| 27 |
-
)
|
| 28 |
logger = logging.getLogger(__name__)
|
|
|
|
| 29 |
|
| 30 |
-
#
|
| 31 |
-
load_dotenv()
|
| 32 |
-
|
| 33 |
-
# Salesforce configuration
|
| 34 |
SF_USERNAME = os.getenv("SF_USERNAME")
|
| 35 |
SF_PASSWORD = os.getenv("SF_PASSWORD")
|
| 36 |
SF_SECURITY_TOKEN = os.getenv("SF_SECURITY_TOKEN")
|
| 37 |
-
|
|
|
|
|
|
|
|
|
|
| 38 |
sf = None
|
| 39 |
-
|
| 40 |
-
|
| 41 |
sf = Salesforce(
|
| 42 |
username=SF_USERNAME,
|
| 43 |
password=SF_PASSWORD,
|
| 44 |
-
security_token=SF_SECURITY_TOKEN
|
| 45 |
-
|
| 46 |
-
logger.info("Salesforce connection established")
|
| 47 |
-
except Exception as e:
|
| 48 |
-
logger.error(f"Salesforce connection failed: {str(e)}")
|
| 49 |
-
SF_ENABLED = False
|
| 50 |
-
|
| 51 |
-
# Encryption setup (AES-256)
|
| 52 |
-
ENCRYPTION_KEY = os.getenv("ENCRYPTION_KEY") or Fernet.generate_key()
|
| 53 |
-
fernet = Fernet(ENCRYPTION_KEY)
|
| 54 |
-
|
| 55 |
-
# Initialize text-to-speech with fallback
|
| 56 |
-
tts_engine = None
|
| 57 |
-
try:
|
| 58 |
-
tts_engine = pyttsx3.init()
|
| 59 |
-
tts_engine.setProperty("rate", 150)
|
| 60 |
-
logger.info("pyttsx3 initialized successfully")
|
| 61 |
-
except Exception as e:
|
| 62 |
-
logger.warning(f"Failed to initialize pyttsx3: {str(e)}. Text-to-speech disabled.")
|
| 63 |
-
|
| 64 |
-
# Initialize local models
|
| 65 |
-
@retry(stop=stop_after_attempt(3), wait=wait_fixed(2))
|
| 66 |
-
def load_whisper_model():
|
| 67 |
-
try:
|
| 68 |
-
model = pipeline(
|
| 69 |
-
"automatic-speech-recognition",
|
| 70 |
-
model="openai/whisper-large-v3",
|
| 71 |
-
device=-1, # CPU; use device=0 for GPU
|
| 72 |
-
model_kwargs={"use_safetensors": True}
|
| 73 |
-
)
|
| 74 |
-
logger.info("Whisper-large-v3 model loaded successfully")
|
| 75 |
-
return model
|
| 76 |
-
except Exception as e:
|
| 77 |
-
logger.error(f"Failed to load Whisper model: {str(e)}")
|
| 78 |
-
raise
|
| 79 |
-
|
| 80 |
-
@retry(stop=stop_after_attempt(3), wait=wait_fixed(2))
|
| 81 |
-
def load_symptom_model():
|
| 82 |
-
try:
|
| 83 |
-
model = pipeline(
|
| 84 |
-
"text-classification",
|
| 85 |
-
model="abhirajeshbhai/symptom-2-disease-net",
|
| 86 |
-
device=-1,
|
| 87 |
-
model_kwargs={"use_safetensors": True},
|
| 88 |
-
return_all_scores=False
|
| 89 |
)
|
| 90 |
-
logger.info("
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
logger.error(f"Failed to load Symptom-2-Disease model: {str(e)}")
|
| 94 |
-
# Disable fallback for now to isolate issue
|
| 95 |
-
raise
|
| 96 |
-
|
| 97 |
-
whisper = None
|
| 98 |
-
symptom_classifier = None
|
| 99 |
-
is_fallback_model = False
|
| 100 |
-
|
| 101 |
-
try:
|
| 102 |
-
whisper = load_whisper_model()
|
| 103 |
except Exception as e:
|
| 104 |
-
logger.error(f"
|
| 105 |
|
|
|
|
| 106 |
try:
|
| 107 |
-
|
|
|
|
|
|
|
|
|
|
| 108 |
except Exception as e:
|
| 109 |
-
logger.error(f"
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
try:
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 152 |
except Exception as e:
|
| 153 |
-
logger.error(f"
|
| 154 |
-
|
| 155 |
|
| 156 |
-
|
| 157 |
-
"""Transcribe audio using Whisper model."""
|
| 158 |
-
if not whisper:
|
| 159 |
-
logger.error("Whisper model not loaded")
|
| 160 |
-
return "Error: Whisper model not loaded"
|
| 161 |
try:
|
| 162 |
-
|
| 163 |
-
if not isinstance(audio_file, (str, bytes, os.PathLike)) or not os.path.exists(audio_file):
|
| 164 |
-
logger.error(f"Invalid or missing audio file: {audio_file}")
|
| 165 |
-
return "Error: Invalid or missing audio file"
|
| 166 |
-
audio, sr = librosa.load(audio_file, sr=16000)
|
| 167 |
-
if len(audio) < 1600:
|
| 168 |
-
logger.error("Audio too short")
|
| 169 |
-
return "Error: Audio too short (<0.1s)"
|
| 170 |
-
if np.max(np.abs(audio)) < 1e-4:
|
| 171 |
-
logger.error("Audio too quiet")
|
| 172 |
-
return "Error: Audio too quiet"
|
| 173 |
-
|
| 174 |
-
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_wav:
|
| 175 |
-
temp_path = temp_wav.name
|
| 176 |
-
soundfile.write(audio, sr, temp_path)
|
| 177 |
-
logger.debug(f"Saved temp WAV: {temp_path}")
|
| 178 |
-
|
| 179 |
with torch.no_grad():
|
| 180 |
-
|
| 181 |
-
transcription =
|
| 182 |
logger.info(f"Transcription: {transcription}")
|
| 183 |
-
|
| 184 |
-
try:
|
| 185 |
-
os.remove(temp_path)
|
| 186 |
-
logger.debug(f"Deleted temp WAV: {temp_path}")
|
| 187 |
-
except Exception as e:
|
| 188 |
-
logger.error(f"Failed to delete temp WAV: {str(e)}")
|
| 189 |
-
|
| 190 |
-
if not transcription:
|
| 191 |
-
logger.error("Transcription empty")
|
| 192 |
-
return "Error: Transcription empty"
|
| 193 |
-
words = transcription.split()
|
| 194 |
-
if len(words) > 5 and len(set(words)) < len(words) / 2:
|
| 195 |
-
logger.error("Transcription repetitive")
|
| 196 |
-
return "Error: Transcription repetitive"
|
| 197 |
return transcription
|
| 198 |
except Exception as e:
|
| 199 |
logger.error(f"Transcription failed: {str(e)}")
|
| 200 |
-
return
|
| 201 |
-
|
| 202 |
-
def analyze_symptoms(text):
|
| 203 |
-
"""Analyze symptoms using Symptom-2-Disease model."""
|
| 204 |
-
if not symptom_classifier:
|
| 205 |
-
logger.error("Symptom model not loaded")
|
| 206 |
-
return "Error: Symptom model not loaded", 0.0
|
| 207 |
-
try:
|
| 208 |
-
if not text or not isinstance(text, str) or "Error" in text:
|
| 209 |
-
logger.error(f"Invalid text input: {text}")
|
| 210 |
-
return "Error: No valid transcription", 0.0
|
| 211 |
-
|
| 212 |
-
with torch.no_grad():
|
| 213 |
-
result = symptom_classifier(text)
|
| 214 |
-
logger.debug(f"Raw model output: type={type(result)}, value={result}")
|
| 215 |
-
|
| 216 |
-
# Initialize default values
|
| 217 |
-
prediction = "No health condition detected"
|
| 218 |
-
score = 0.0
|
| 219 |
-
|
| 220 |
-
# Handle expected output: list of dictionaries
|
| 221 |
-
if isinstance(result, list) and result:
|
| 222 |
-
valid_items = [
|
| 223 |
-
item for item in result
|
| 224 |
-
if isinstance(item, dict) and
|
| 225 |
-
"label" in item and isinstance(item["label"], str) and
|
| 226 |
-
"score" in item and isinstance(item["score"], (int, float)) and 0 <= item["score"] <= 1
|
| 227 |
-
]
|
| 228 |
-
if valid_items:
|
| 229 |
-
sorted_items = sorted(valid_items, key=lambda x: x["score"], reverse=True)
|
| 230 |
-
prediction = sorted_items[0]["label"]
|
| 231 |
-
score = sorted_items[0]["score"]
|
| 232 |
-
else:
|
| 233 |
-
logger.warning(f"Invalid items in result list: {result}")
|
| 234 |
-
elif isinstance(result, dict):
|
| 235 |
-
if "label" in result and "score" in result and isinstance(result["label"], str) and \
|
| 236 |
-
isinstance(result["score"], (int, float)) and 0 <= result["score"] <= 1:
|
| 237 |
-
prediction = result["label"]
|
| 238 |
-
score = result["score"]
|
| 239 |
-
else:
|
| 240 |
-
logger.warning(f"Invalid dictionary content: {result}")
|
| 241 |
-
elif isinstance(result, tuple):
|
| 242 |
-
logger.warning(f"Received tuple output: {result}")
|
| 243 |
-
if len(result) == 0:
|
| 244 |
-
logger.warning("Empty tuple received")
|
| 245 |
-
elif len(result) > 0:
|
| 246 |
-
if isinstance(result[0], dict):
|
| 247 |
-
if "label" in result[0] and "score" in result[0] and \
|
| 248 |
-
isinstance(result[0]["label"], str) and \
|
| 249 |
-
isinstance(result[0]["score"], (int, float)) and 0 <= result[0]["score"] <= 1:
|
| 250 |
-
prediction = result[0]["label"]
|
| 251 |
-
score = result[0]["score"]
|
| 252 |
-
else:
|
| 253 |
-
logger.warning(f"Invalid dictionary in tuple: {result[0]}")
|
| 254 |
-
else:
|
| 255 |
-
logger.warning(f"First tuple element is not a dict: {result[0]}")
|
| 256 |
-
else:
|
| 257 |
-
logger.warning(f"Invalid tuple content: {result}")
|
| 258 |
-
else:
|
| 259 |
-
logger.warning(f"Unexpected model output type: {type(result)}, value: {result}")
|
| 260 |
-
|
| 261 |
-
# Final validation
|
| 262 |
-
if not isinstance(prediction, str):
|
| 263 |
-
logger.warning(f"Invalid label type: {type(prediction)}, value: {prediction}")
|
| 264 |
-
prediction = "No health condition detected"
|
| 265 |
-
if not isinstance(score, (int, float)) or score < 0 or score > 1:
|
| 266 |
-
logger.warning(f"Invalid score: {score}")
|
| 267 |
-
score = 0.0
|
| 268 |
-
|
| 269 |
-
logger.info(f"Prediction: {prediction}, Score: {score:.4f}")
|
| 270 |
-
return prediction, score
|
| 271 |
-
except Exception as e:
|
| 272 |
-
logger.error(f"Symptom analysis failed: {str(e)}")
|
| 273 |
-
logger.error(f"Stack trace: {traceback.format_exc()}")
|
| 274 |
-
return "Error: Symptom analysis failed", 0.0
|
| 275 |
|
| 276 |
-
def
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
return
|
| 281 |
try:
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
"Prediction__c": prediction[:255],
|
| 289 |
-
"Confidence_Score__c": float(score),
|
| 290 |
-
"Feedback__c": encrypted_feedback[:255],
|
| 291 |
-
"Analysis_Date__c": datetime.utcnow().strftime("%Y-%m-%d")
|
| 292 |
-
})
|
| 293 |
-
logger.info("Saved analysis to Salesforce")
|
| 294 |
except Exception as e:
|
| 295 |
-
logger.error(f"
|
|
|
|
| 296 |
|
| 297 |
-
def
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
logger.error(f"Failed to generate report: {str(e)}")
|
| 313 |
-
return f"Error: {str(e)}"
|
| 314 |
-
|
| 315 |
-
async def speak_response(text):
|
| 316 |
-
"""Convert text to speech."""
|
| 317 |
-
if not tts_engine:
|
| 318 |
-
logger.warning("Text-to-speech unavailable; skipping")
|
| 319 |
-
return None
|
| 320 |
-
try:
|
| 321 |
-
def sync_speak():
|
| 322 |
-
tts_engine.say(text)
|
| 323 |
-
tts_engine.runAndWait()
|
| 324 |
-
loop = asyncio.get_event_loop()
|
| 325 |
-
await loop.run_in_executor(None, sync_speak)
|
| 326 |
-
logger.debug("Speech response generated")
|
| 327 |
-
except Exception as e:
|
| 328 |
-
logger.error(f"Text-to-speech failed: {str(e)}")
|
| 329 |
|
| 330 |
-
async def analyze_voice(audio_file, language="en", user_id="anonymous", consent_granted=True):
|
| 331 |
-
"""Analyze voice for health indicators."""
|
| 332 |
try:
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
|
| 342 |
-
|
| 343 |
-
|
| 344 |
-
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
| 352 |
-
f"
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
|
| 361 |
-
|
| 362 |
-
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
|
| 366 |
-
|
| 367 |
-
|
| 368 |
-
|
| 369 |
-
|
| 370 |
-
|
| 371 |
-
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
|
| 375 |
-
|
| 376 |
-
|
| 377 |
-
|
| 378 |
-
|
| 379 |
-
|
| 380 |
-
|
| 381 |
-
|
| 382 |
-
return prediction
|
| 383 |
-
|
| 384 |
-
feedback = (
|
| 385 |
-
"No health condition detected, consult a doctor if symptoms persist. This is not a medical diagnosis."
|
| 386 |
-
if prediction == "No health condition detected"
|
| 387 |
-
else f"Possible {prediction.lower()} detected based on symptoms like '{transcription.lower()}', consult a doctor. This is not a medical diagnosis."
|
| 388 |
-
)
|
| 389 |
-
logger.info(f"Feedback: {feedback}, Transcription: {transcription}, Prediction: {prediction}, Score: {score:.4f}")
|
| 390 |
-
|
| 391 |
-
# Save to Salesforce
|
| 392 |
-
save_to_salesforce(user_id, transcription, prediction, score, feedback, consent_granted)
|
| 393 |
-
|
| 394 |
-
try:
|
| 395 |
-
os.remove(audio_file)
|
| 396 |
-
logger.debug(f"Deleted audio file: {audio_file}")
|
| 397 |
-
except Exception as e:
|
| 398 |
-
logger.error(f"Failed to delete audio file: {str(e)}")
|
| 399 |
-
|
| 400 |
-
# Speak response
|
| 401 |
-
await speak_response(feedback)
|
| 402 |
-
|
| 403 |
-
return feedback
|
| 404 |
except Exception as e:
|
| 405 |
-
logger.error(f"
|
| 406 |
return f"Error: {str(e)}"
|
| 407 |
|
| 408 |
-
|
| 409 |
-
"""Test with synthetic audio."""
|
| 410 |
-
temp_dir = os.path.join(tempfile.gettempdir(), "audio_samples")
|
| 411 |
-
if not ensure_writable_dir(temp_dir):
|
| 412 |
-
fallback_dir = os.path.join(os.getcwd(), "temp_audio_samples")
|
| 413 |
-
if not ensure_writable_dir(fallback_dir):
|
| 414 |
-
logger.error(f"Temp directories {temp_dir} and {fallback_dir} not writable")
|
| 415 |
-
return f"Error: Temp directories not writable"
|
| 416 |
-
temp_dir = fallback_dir
|
| 417 |
-
|
| 418 |
-
sample_audio_path = os.path.join(temp_dir, "dummy_test.wav")
|
| 419 |
-
logger.info(f"Generating synthetic audio at: {sample_audio_path}")
|
| 420 |
-
sr = 16000
|
| 421 |
-
t = np.linspace(0, 2, 2 * sr)
|
| 422 |
-
freq_mod = 440 + 10 * np.sin(2 * np.pi * 0.5 * t)
|
| 423 |
-
amplitude_mod = 0.5 + 0.1 * np.sin(2 * np.pi * 0.3 * t)
|
| 424 |
-
noise = 0.01 * np.random.normal(0, 1, len(t))
|
| 425 |
-
dummy_audio = amplitude_mod * np.sin(2 * np.pi * freq_mod * t) + noise
|
| 426 |
try:
|
| 427 |
-
|
| 428 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 429 |
except Exception as e:
|
| 430 |
-
logger.error(f"
|
| 431 |
-
|
| 432 |
-
|
| 433 |
-
|
| 434 |
-
|
| 435 |
-
|
| 436 |
-
|
| 437 |
-
|
| 438 |
-
|
| 439 |
-
|
| 440 |
-
|
| 441 |
-
|
| 442 |
-
|
| 443 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 444 |
)
|
| 445 |
-
logger.info(f"Test feedback: {feedback}, Prediction: {prediction}, Score: {score:.4f}")
|
| 446 |
-
|
| 447 |
-
# Save to Salesforce
|
| 448 |
-
save_to_salesforce(user_id, mock_transcription, prediction, score, feedback, consent_granted)
|
| 449 |
-
|
| 450 |
-
try:
|
| 451 |
-
os.remove(sample_audio_path)
|
| 452 |
-
logger.debug(f"Deleted test audio: {sample_audio_path}")
|
| 453 |
-
except Exception:
|
| 454 |
-
pass
|
| 455 |
-
return feedback
|
| 456 |
-
|
| 457 |
-
async def voicebot_interface(audio_file, language="en", user_id="anonymous", consent_granted=True):
|
| 458 |
-
"""Gradio interface wrapper."""
|
| 459 |
-
return await analyze_voice(audio_file, language, user_id, consent_granted)
|
| 460 |
|
| 461 |
-
#
|
| 462 |
-
|
| 463 |
-
|
| 464 |
-
|
| 465 |
-
|
| 466 |
-
|
| 467 |
-
|
| 468 |
-
gr.
|
| 469 |
-
|
| 470 |
-
|
| 471 |
-
|
| 472 |
-
description="Record or upload a voice sample describing symptoms (e.g., 'I have a cough') for preliminary health assessment. Supports English, Spanish, Hindi, Mandarin. Not a diagnostic tool. Data is encrypted and stored with consent. Complies with HIPAA/GDPR."
|
| 473 |
-
)
|
| 474 |
|
| 475 |
if __name__ == "__main__":
|
| 476 |
-
logger.info("Starting
|
| 477 |
-
|
| 478 |
-
loop = asyncio.get_event_loop()
|
| 479 |
-
print(loop.run_until_complete(test_with_sample_audio()))
|
| 480 |
-
iface.launch(server_name="0.0.0.0", server_port=7860)
|
|
|
|
| 1 |
import gradio as gr
|
| 2 |
import librosa
|
| 3 |
import numpy as np
|
| 4 |
+
import torch
|
| 5 |
+
from transformers import WhisperProcessor, WhisperForConditionalGeneration
|
| 6 |
+
from simple_salesforce import Salesforce
|
| 7 |
import os
|
|
|
|
| 8 |
from datetime import datetime
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
import logging
|
| 10 |
+
import webrtcvad
|
| 11 |
+
import google.generativeai as genai
|
| 12 |
+
from gtts import gTTS
|
| 13 |
import tempfile
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
|
| 15 |
+
# Set up logging for usage metrics and debugging
|
| 16 |
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
logger = logging.getLogger(__name__)
|
| 18 |
+
usage_metrics = {"total_assessments": 0}
|
| 19 |
|
| 20 |
+
# Environment variables for secure credentials
|
|
|
|
|
|
|
|
|
|
| 21 |
SF_USERNAME = os.getenv("SF_USERNAME")
|
| 22 |
SF_PASSWORD = os.getenv("SF_PASSWORD")
|
| 23 |
SF_SECURITY_TOKEN = os.getenv("SF_SECURITY_TOKEN")
|
| 24 |
+
SF_INSTANCE_URL = os.getenv("SF_INSTANCE_URL", "https://login.salesforce.com")
|
| 25 |
+
GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "AIzaSyBzr5vVpbe8CV1v70l3pGDp9vRJ76yCxdk")
|
| 26 |
+
|
| 27 |
+
# Initialize Salesforce
|
| 28 |
sf = None
|
| 29 |
+
try:
|
| 30 |
+
if all([SF_USERNAME, SF_PASSWORD, SF_SECURITY_TOKEN]):
|
| 31 |
sf = Salesforce(
|
| 32 |
username=SF_USERNAME,
|
| 33 |
password=SF_PASSWORD,
|
| 34 |
+
security_token=SF_SECURITY_TOKEN,
|
| 35 |
+
instance_url=SF_INSTANCE_URL
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
)
|
| 37 |
+
logger.info("Connected to Salesforce for user management")
|
| 38 |
+
else:
|
| 39 |
+
logger.warning("Salesforce credentials missing; user management disabled")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
except Exception as e:
|
| 41 |
+
logger.error(f"Salesforce connection failed: {str(e)}")
|
| 42 |
|
| 43 |
+
# Initialize Google Gemini
|
| 44 |
try:
|
| 45 |
+
genai.configure(api_key=GEMINI_API_KEY)
|
| 46 |
+
gemini_model = genai.GenerativeModel('gemini-1.5-flash')
|
| 47 |
+
chat = gemini_model.start_chat(history=[])
|
| 48 |
+
logger.info("Connected to Google Gemini for chatbot functionality")
|
| 49 |
except Exception as e:
|
| 50 |
+
logger.error(f"Google Gemini initialization failed: {str(e)}")
|
| 51 |
+
chat = None
|
| 52 |
+
|
| 53 |
+
# Load Whisper model for speech-to-text
|
| 54 |
+
whisper_processor = WhisperProcessor.from_pretrained("openai/whisper-tiny")
|
| 55 |
+
whisper_model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny")
|
| 56 |
+
whisper_model.config.forced_decoder_ids = whisper_processor.get_decoder_prompt_ids(language="english", task="transcribe")
|
| 57 |
+
|
| 58 |
+
# Initialize VAD
|
| 59 |
+
vad = webrtcvad.Vad(mode=2)
|
| 60 |
+
|
| 61 |
+
# Chatbot knowledge base
|
| 62 |
+
base_info = """
|
| 63 |
+
You are a highly advanced AI assistant named 'MindCare'.
|
| 64 |
+
Your role is to provide support in various aspects of health and well-being, including:
|
| 65 |
+
- **Mental health**: Emotional support, mindfulness, stress-relief exercises, anxiety management.
|
| 66 |
+
- **Medical guidance**: Basic symptom analysis, possible conditions, and medicine recommendations.
|
| 67 |
+
- **Decision-making support**: Helping users with personal, professional, and emotional choices.
|
| 68 |
+
- **General health advice**: Lifestyle improvements, nutrition, physical wellness, and mental well-being.
|
| 69 |
+
- **Emergency assistance**: If the user is in distress, suggest professional help or helpline numbers.
|
| 70 |
+
Your tone is always **empathetic, supportive, and informative**. You ensure users feel heard and cared for.
|
| 71 |
+
"""
|
| 72 |
+
mental_health = """
|
| 73 |
+
If the user is feeling stressed or anxious:
|
| 74 |
+
- Suggest mindfulness exercises, deep breathing techniques, or gratitude journaling.
|
| 75 |
+
- Encourage taking breaks, engaging in hobbies, and spending time in nature.
|
| 76 |
+
- Provide positive affirmations and self-care routines.
|
| 77 |
+
If the user is in distress:
|
| 78 |
+
- Offer emotional support and let them know they are not alone.
|
| 79 |
+
- Encourage them to reach out to a trusted person or professional.
|
| 80 |
+
- Provide emergency helpline numbers if needed.
|
| 81 |
+
"""
|
| 82 |
+
medical_assistance = """
|
| 83 |
+
If the user provides symptoms:
|
| 84 |
+
- Analyze symptoms and suggest possible conditions.
|
| 85 |
+
- Provide general advice but **never** replace a doctor’s consultation.
|
| 86 |
+
- Suggest lifestyle changes or basic home remedies if applicable.
|
| 87 |
+
- If symptoms are severe, advise them to visit a healthcare professional.
|
| 88 |
+
If the user asks about medicines:
|
| 89 |
+
- Suggest **common antibiotics** based on infection type (e.g., Amoxicillin for bacterial infections).
|
| 90 |
+
- Recommend **painkillers** like Paracetamol, Ibuprofen, or Diclofenac for pain relief.
|
| 91 |
+
- Mention precautions and possible side effects.
|
| 92 |
+
- Clearly **state that a doctor’s consultation is necessary before taking any medicine**.
|
| 93 |
+
"""
|
| 94 |
+
medicine_recommendation = """
|
| 95 |
+
If the user asks for a prescription, provide general guidance on **commonly used medicines**:
|
| 96 |
+
- **Antibiotics** (for bacterial infections): Amoxicillin, Azithromycin, Ciprofloxacin.
|
| 97 |
+
- **Painkillers**: Paracetamol (mild pain/fever), Ibuprofen (anti-inflammatory), Diclofenac (muscle pain).
|
| 98 |
+
- **Cold & Flu**: Antihistamines like Cetirizine, Cough syrups like Dextromethorphan.
|
| 99 |
+
- **Stomach Issues**: Antacids like Ranitidine, PPI like Omeprazole.
|
| 100 |
+
Always remind the user that **only a licensed doctor can prescribe medicines, and misuse can be harmful**.
|
| 101 |
+
"""
|
| 102 |
+
decision_guidance = """
|
| 103 |
+
If the user is struggling with a decision:
|
| 104 |
+
- Help them weigh pros and cons logically.
|
| 105 |
+
- Suggest considering their values, long-term goals, and emotions.
|
| 106 |
+
- Provide structured approaches like decision matrices or intuitive checks.
|
| 107 |
+
- Encourage seeking advice from trusted people if needed.
|
| 108 |
+
"""
|
| 109 |
+
emergency_help = """
|
| 110 |
+
If the user mentions severe mental distress:
|
| 111 |
+
- Respond with immediate emotional support.
|
| 112 |
+
- Provide crisis helpline numbers (if applicable to the region).
|
| 113 |
+
- Encourage talking to a trusted friend, family member, or professional.
|
| 114 |
+
- Remind them that they are not alone and help is available.
|
| 115 |
+
"""
|
| 116 |
+
context = [base_info, mental_health, medical_assistance, medicine_recommendation, decision_guidance, emergency_help]
|
| 117 |
+
|
| 118 |
+
def extract_health_features(audio, sr):
|
| 119 |
try:
|
| 120 |
+
audio = audio / np.max(np.abs(audio)) if np.max(np.abs(audio)) != 0 else audio
|
| 121 |
+
frame_duration = 30
|
| 122 |
+
frame_samples = int(sr * frame_duration / 1000)
|
| 123 |
+
frames = [audio[i:i + frame_samples] for i in range(0, len(audio), frame_samples)]
|
| 124 |
+
voiced_frames = [
|
| 125 |
+
frame for frame in frames
|
| 126 |
+
if len(frame) == frame_samples and vad.is_speech((frame * 32768).astype(np.int16).tobytes(), sr)
|
| 127 |
+
]
|
| 128 |
+
if not voiced_frames:
|
| 129 |
+
raise ValueError("No voiced segments detected")
|
| 130 |
+
voiced_audio = np.concatenate(voiced_frames)
|
| 131 |
+
pitches, magnitudes = librosa.piptrack(y=voiced_audio, sr=sr, fmin=75, fmax=300)
|
| 132 |
+
valid_pitches = [p for p in pitches[magnitudes > 0] if 75 <= p <= 300]
|
| 133 |
+
pitch = np.mean(valid_pitches) if valid_pitches else 0
|
| 134 |
+
jitter = np.std(valid_pitches) / pitch if pitch and valid_pitches else 0
|
| 135 |
+
if jitter > 10:
|
| 136 |
+
jitter = 10
|
| 137 |
+
logger.warning("Jitter capped at 10%")
|
| 138 |
+
amplitudes = librosa.feature.rms(y=voiced_audio, frame_length=2048, hop_length=512)[0]
|
| 139 |
+
shimmer = np.std(amplitudes) / np.mean(amplitudes) if np.mean(amplitudes) else 0
|
| 140 |
+
if shimmer > 10:
|
| 141 |
+
shimmer = 10
|
| 142 |
+
logger.warning("Shimmer capped at 10%")
|
| 143 |
+
energy = np.mean(librosa.feature.rms(y=voiced_audio, frame_length=2048, hop_length=512)[0])
|
| 144 |
+
return {
|
| 145 |
+
"pitch": pitch,
|
| 146 |
+
"jitter": jitter * 100,
|
| 147 |
+
"shimmer": shimmer * 100,
|
| 148 |
+
"energy": energy
|
| 149 |
+
}
|
| 150 |
except Exception as e:
|
| 151 |
+
logger.error(f"Feature extraction failed: {str(e)}")
|
| 152 |
+
raise
|
| 153 |
|
| 154 |
+
def transcribe_audio(audio):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 155 |
try:
|
| 156 |
+
inputs = whisper_processor(audio, sampling_rate=16000, return_tensors="pt")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 157 |
with torch.no_grad():
|
| 158 |
+
generated_ids = whisper_model.generate(inputs["input_features"])
|
| 159 |
+
transcription = whisper_processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
|
| 160 |
logger.info(f"Transcription: {transcription}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 161 |
return transcription
|
| 162 |
except Exception as e:
|
| 163 |
logger.error(f"Transcription failed: {str(e)}")
|
| 164 |
+
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 165 |
|
| 166 |
+
def get_chatbot_response(message):
|
| 167 |
+
if not chat or not message:
|
| 168 |
+
return "Unable to generate chatbot response due to missing input or model.", None
|
| 169 |
+
full_context = "\n".join(context) + f"\nUser: {message}\nMindCare:"
|
|
|
|
| 170 |
try:
|
| 171 |
+
response = chat.send_message(full_context).text
|
| 172 |
+
with tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") as temp_audio:
|
| 173 |
+
tts = gTTS(text=response, lang="en", slow=False)
|
| 174 |
+
tts.save(temp_audio.name)
|
| 175 |
+
audio_path = temp_audio.name
|
| 176 |
+
return response, audio_path
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 177 |
except Exception as e:
|
| 178 |
+
logger.error(f"Chatbot response failed: {str(e)}")
|
| 179 |
+
return "Error generating chatbot response.", None
|
| 180 |
|
| 181 |
+
def analyze_symptoms(text):
|
| 182 |
+
text = text.lower()
|
| 183 |
+
feedback = []
|
| 184 |
+
if "cough" in text or "difficulty breathing" in text:
|
| 185 |
+
feedback.append("Based on your input, you may have a respiratory issue, such as bronchitis or asthma. Please consult a doctor.")
|
| 186 |
+
elif "stressed" in text or "stress" in text or "tired" in text or "fatigue" in text:
|
| 187 |
+
feedback.append("Your description suggests possible stress or fatigue, potentially linked to anxiety or exhaustion. Consider seeking medical advice.")
|
| 188 |
+
else:
|
| 189 |
+
feedback.append("Your input didn’t clearly indicate specific symptoms. Please describe any health concerns (e.g., cough, stress) and consult a healthcare provider.")
|
| 190 |
+
return "\n".join(feedback)
|
| 191 |
+
|
| 192 |
+
def analyze_voice(audio_file=None):
|
| 193 |
+
global usage_metrics
|
| 194 |
+
usage_metrics["total_assessments"] += 1
|
| 195 |
+
logger.info(f"Total assessments: {usage_metrics['total_assessments']}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 196 |
|
|
|
|
|
|
|
| 197 |
try:
|
| 198 |
+
if audio_file and os.path.exists(audio_file):
|
| 199 |
+
audio, sr = librosa.load(audio_file, sr=16000)
|
| 200 |
+
else:
|
| 201 |
+
raise ValueError("No valid audio file provided for analysis")
|
| 202 |
+
|
| 203 |
+
if len(audio) < sr:
|
| 204 |
+
raise ValueError("Audio too short (minimum 1 second)")
|
| 205 |
+
|
| 206 |
+
features = extract_health_features(audio, sr)
|
| 207 |
+
transcription = transcribe_audio(audio)
|
| 208 |
+
symptom_feedback = analyze_symptoms(transcription) if transcription else "No transcription available. Please record again with clear speech."
|
| 209 |
+
|
| 210 |
+
feedback = []
|
| 211 |
+
respiratory_score = features["jitter"]
|
| 212 |
+
mental_health_score = features["shimmer"]
|
| 213 |
+
|
| 214 |
+
if respiratory_score > 1.0:
|
| 215 |
+
feedback.append(f"Your voice indicates elevated jitter ({respiratory_score:.2f}%), which may suggest respiratory issues. Consult a doctor.")
|
| 216 |
+
if mental_health_score > 5.0:
|
| 217 |
+
feedback.append(f"Your voice shows elevated shimmer ({mental_health_score:.2f}%), possibly indicating stress or emotional strain. Consider a health check.")
|
| 218 |
+
if features["energy"] < 0.01:
|
| 219 |
+
feedback.append(f"Your vocal energy is low ({features['energy']:.4f}), which might point to fatigue. Seek medical advice if this persists.")
|
| 220 |
+
|
| 221 |
+
if not feedback and not symptom_feedback.startswith("No transcription"):
|
| 222 |
+
feedback.append("Your voice analysis shows no immediate health concerns based on current data.")
|
| 223 |
+
|
| 224 |
+
feedback.append("\n**Symptom Feedback (Based on Your Input)**:")
|
| 225 |
+
feedback.append(symptom_feedback)
|
| 226 |
+
feedback.append("\n**Voice Analysis Details**:")
|
| 227 |
+
feedback.append(f"Pitch: {features['pitch']:.2f} Hz (average fundamental frequency)")
|
| 228 |
+
feedback.append(f"Jitter: {respiratory_score:.2f}% (pitch variation, higher values may indicate respiratory issues)")
|
| 229 |
+
feedback.append(f"Shimmer: {mental_health_score:.2f}% (amplitude variation, higher values may indicate stress)")
|
| 230 |
+
feedback.append(f"Energy: {features['energy']:.4f} (vocal intensity, lower values may indicate fatigue)")
|
| 231 |
+
feedback.append(f"Transcription: {transcription if transcription else 'None'}")
|
| 232 |
+
feedback.append("\n**Disclaimer**: This is a preliminary analysis, not a medical diagnosis. Always consult a healthcare provider for professional evaluation.")
|
| 233 |
+
|
| 234 |
+
feedback_str = "\n".join(feedback)
|
| 235 |
+
|
| 236 |
+
if sf:
|
| 237 |
+
store_in_salesforce(audio_file, feedback_str, respiratory_score, mental_health_score, features, transcription)
|
| 238 |
+
|
| 239 |
+
if audio_file and os.path.exists(audio_file):
|
| 240 |
+
try:
|
| 241 |
+
os.remove(audio_file)
|
| 242 |
+
logger.info(f"Deleted audio file: {audio_file} for compliance")
|
| 243 |
+
except Exception as e:
|
| 244 |
+
logger.error(f"Failed to delete audio file: {str(e)}")
|
| 245 |
+
|
| 246 |
+
return feedback_str
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 247 |
except Exception as e:
|
| 248 |
+
logger.error(f"Audio processing failed: {str(e)}")
|
| 249 |
return f"Error: {str(e)}"
|
| 250 |
|
| 251 |
+
def store_in_salesforce(audio_file, feedback, respiratory_score, mental_health_score, features, transcription):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 252 |
try:
|
| 253 |
+
sf.HealthAssessment__c.create({
|
| 254 |
+
"AssessmentDate__c": datetime.utcnow().isoformat(),
|
| 255 |
+
"Feedback__c": feedback,
|
| 256 |
+
"RespiratoryScore__c": float(respiratory_score),
|
| 257 |
+
"MentalHealthScore__c": float(mental_health_score),
|
| 258 |
+
"AudioFileName__c": os.path.basename(audio_file) if audio_file else "user_recorded_audio",
|
| 259 |
+
"Pitch__c": float(features["pitch"]),
|
| 260 |
+
"Jitter__c": float(features["jitter"]),
|
| 261 |
+
"Shimmer__c": float(features["shimmer"]),
|
| 262 |
+
"Energy__c": float(features["energy"]),
|
| 263 |
+
"Transcription__c": transcription or "None"
|
| 264 |
+
})
|
| 265 |
+
logger.info("Stored assessment in Salesforce")
|
| 266 |
except Exception as e:
|
| 267 |
+
logger.error(f"Salesforce storage failed: {str(e)}")
|
| 268 |
+
|
| 269 |
+
# Combined interface with voice analysis and chatbot suggestions
|
| 270 |
+
with gr.Blocks(title="MindCare Health Assistant") as demo:
|
| 271 |
+
gr.Markdown("# MindCare Health Assistant")
|
| 272 |
+
gr.Markdown("This tool is accessible via web and mobile. Use the sections below for health assessments and suggestions.")
|
| 273 |
+
|
| 274 |
+
with gr.Row():
|
| 275 |
+
with gr.Column():
|
| 276 |
+
gr.Markdown("### Voice Analysis")
|
| 277 |
+
gr.Markdown("Record or upload your voice (minimum 1 second) to receive a preliminary health check. Speak clearly in English about your symptoms (e.g., 'I have a cough' or 'I feel stressed').")
|
| 278 |
+
audio_input = gr.Audio(type="filepath", label="Record or Upload Your Voice (WAV, MP3, FLAC, 1+ sec)", format="wav")
|
| 279 |
+
voice_output = gr.Textbox(label="Health Assessment Results", elem_id="health-results")
|
| 280 |
+
submit_btn = gr.Button("Submit")
|
| 281 |
+
clear_btn = gr.Button("Clear")
|
| 282 |
+
|
| 283 |
+
with gr.Column():
|
| 284 |
+
gr.Markdown("### Health Suggestions")
|
| 285 |
+
gr.Markdown("Enter a message to get personalized health suggestions from MindCare.")
|
| 286 |
+
text_input = gr.Textbox(label="Enter your message")
|
| 287 |
+
text_output = gr.Textbox(label="Response")
|
| 288 |
+
audio_output = gr.Audio(label="Response Audio")
|
| 289 |
+
suggest_submit_btn = gr.Button("Submit")
|
| 290 |
+
suggest_clear_btn = gr.Button("Clear")
|
| 291 |
+
|
| 292 |
+
# Voice analysis event
|
| 293 |
+
submit_btn.click(
|
| 294 |
+
fn=analyze_voice,
|
| 295 |
+
inputs=[audio_input],
|
| 296 |
+
outputs=[voice_output]
|
| 297 |
+
)
|
| 298 |
+
clear_btn.click(
|
| 299 |
+
fn=lambda: (gr.update(value=None), gr.update(value="")),
|
| 300 |
+
inputs=None,
|
| 301 |
+
outputs=[audio_input, voice_output]
|
| 302 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 303 |
|
| 304 |
+
# Chatbot suggestion event
|
| 305 |
+
suggest_submit_btn.click(
|
| 306 |
+
fn=get_chatbot_response,
|
| 307 |
+
inputs=[text_input],
|
| 308 |
+
outputs=[text_output, audio_output]
|
| 309 |
+
)
|
| 310 |
+
suggest_clear_btn.click(
|
| 311 |
+
fn=lambda: (gr.update(value=""), gr.update(value=""), gr.update(value=None)),
|
| 312 |
+
inputs=None,
|
| 313 |
+
outputs=[text_input, text_output, audio_output]
|
| 314 |
+
)
|
|
|
|
|
|
|
| 315 |
|
| 316 |
if __name__ == "__main__":
|
| 317 |
+
logger.info("Starting MindCare Health Analyzer at 02:21 PM IST, June 23, 2025")
|
| 318 |
+
demo.launch(server_name="0.0.0.0", server_port=7860)
|
|
|
|
|
|
|
|
|