Glitche commited on
Upload 12 files
Browse files- Dockerfile +35 -0
- Procfile +1 -0
- README.md +18 -5
- app.py +186 -0
- database.py +138 -0
- nlu.py +71 -0
- requirements.txt +11 -0
- schema.sql +27 -0
- schema_postgres.sql +25 -0
- stt.py +51 -0
- translation.py +69 -0
- tts.py +138 -0
Dockerfile
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# TalkToDoc production image.
|
| 2 |
+
# Built to be interchangeable across hosts: Hugging Face Spaces, Render
|
| 3 |
+
# (Docker deploy), Fly.io, Cloud Run, or any other platform that runs a
|
| 4 |
+
# standard Docker image. Nothing here is tied to one specific host.
|
| 5 |
+
# Configuration comes entirely from environment variables set by
|
| 6 |
+
# whichever platform runs it (PORT, DATABASE_URL, ANTHROPIC_API_KEY,
|
| 7 |
+
# SECRET_KEY, ENVIRONMENT).
|
| 8 |
+
|
| 9 |
+
FROM python:3.11-slim
|
| 10 |
+
|
| 11 |
+
# Whisper needs ffmpeg to read audio files, this installs it at the
|
| 12 |
+
# system level since it can't come from pip.
|
| 13 |
+
RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg \
|
| 14 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 15 |
+
|
| 16 |
+
# Run as a non-root user, standard practice and specifically expected by
|
| 17 |
+
# Hugging Face Spaces' Docker SDK.
|
| 18 |
+
RUN useradd -m -u 1000 user
|
| 19 |
+
USER user
|
| 20 |
+
ENV PATH="/home/user/.local/bin:$PATH"
|
| 21 |
+
|
| 22 |
+
WORKDIR /home/user/app
|
| 23 |
+
|
| 24 |
+
COPY --chown=user requirements.txt .
|
| 25 |
+
RUN pip install --no-cache-dir --upgrade -r requirements.txt
|
| 26 |
+
|
| 27 |
+
COPY --chown=user . .
|
| 28 |
+
|
| 29 |
+
ENV ENVIRONMENT=production
|
| 30 |
+
|
| 31 |
+
# 7860 is Hugging Face Spaces' default port. Other platforms (like
|
| 32 |
+
# Render) set $PORT themselves, and this falls back to that automatically.
|
| 33 |
+
EXPOSE 7860
|
| 34 |
+
|
| 35 |
+
CMD gunicorn app:app --bind 0.0.0.0:${PORT:-7860} --timeout 120 --workers 1
|
Procfile
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
web: gunicorn app:app --bind 0.0.0.0:$PORT --timeout 120 --workers 1
|
README.md
CHANGED
|
@@ -1,10 +1,23 @@
|
|
| 1 |
---
|
| 2 |
title: TalkToDoc
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: docker
|
| 7 |
-
|
| 8 |
---
|
| 9 |
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
title: TalkToDoc
|
| 3 |
+
emoji: 🩺
|
| 4 |
+
colorFrom: green
|
| 5 |
+
colorTo: gray
|
| 6 |
sdk: docker
|
| 7 |
+
app_port: 7860
|
| 8 |
---
|
| 9 |
|
| 10 |
+
# TalkToDoc
|
| 11 |
+
|
| 12 |
+
A multilingual communication tool that helps patients and healthcare
|
| 13 |
+
providers understand each other across English, Yoruba, Hausa, Igbo, and
|
| 14 |
+
Nigerian Pidgin. Built for rural Nigerian healthcare settings where
|
| 15 |
+
doctor shortages and language barriers contribute to misdiagnosis and
|
| 16 |
+
communication breakdowns.
|
| 17 |
+
|
| 18 |
+
This Space runs the full application: speech-to-text, translation,
|
| 19 |
+
symptom summarization, and text-to-speech, in the patient's own
|
| 20 |
+
language.
|
| 21 |
+
|
| 22 |
+
Requires ANTHROPIC_API_KEY, SECRET_KEY, and DATABASE_URL to be set as
|
| 23 |
+
secrets in this Space's settings.
|
app.py
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
TalkToDoc Flask application.
|
| 3 |
+
Wires together speech-to-text, translation, natural language understanding,
|
| 4 |
+
and text-to-speech into the patient-provider communication flow described
|
| 5 |
+
in the research document.
|
| 6 |
+
|
| 7 |
+
Two-step flow:
|
| 8 |
+
1. Patient submits input (text or audio) in their chosen language.
|
| 9 |
+
The system transcribes it if needed, translates it to English, and
|
| 10 |
+
summarizes the likely symptoms or intent for the provider.
|
| 11 |
+
2. Provider submits a reply in English. The system translates it back
|
| 12 |
+
into the patient's language and generates spoken audio for it.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
import os
|
| 16 |
+
import uuid
|
| 17 |
+
from datetime import datetime, timezone
|
| 18 |
+
|
| 19 |
+
from dotenv import load_dotenv
|
| 20 |
+
from flask import Flask, request, jsonify, render_template, session
|
| 21 |
+
|
| 22 |
+
import database
|
| 23 |
+
import stt
|
| 24 |
+
import translation
|
| 25 |
+
import nlu
|
| 26 |
+
import tts
|
| 27 |
+
|
| 28 |
+
load_dotenv()
|
| 29 |
+
|
| 30 |
+
AUDIO_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "static", "audio")
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def create_app():
|
| 34 |
+
flask_app = Flask(__name__)
|
| 35 |
+
flask_app.secret_key = os.environ["SECRET_KEY"]
|
| 36 |
+
os.makedirs(AUDIO_DIR, exist_ok=True)
|
| 37 |
+
return flask_app
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
app = create_app()
|
| 41 |
+
|
| 42 |
+
# Runs whether the app is started directly (python app.py, local development)
|
| 43 |
+
# or imported by a production server like gunicorn (Render deployment).
|
| 44 |
+
# Initialization code inside "if __name__ == '__main__'" only runs on a
|
| 45 |
+
# direct start, gunicorn never triggers it, so anything the app needs to
|
| 46 |
+
# work at all has to happen here instead.
|
| 47 |
+
database.init_db()
|
| 48 |
+
tts.preload_models()
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def get_current_user_id(language):
|
| 52 |
+
"""
|
| 53 |
+
Returns the user_id for the current browser session, creating a new
|
| 54 |
+
user and session record the first time this browser is seen. This is
|
| 55 |
+
what keeps a patient's messages tied to the same person instead of
|
| 56 |
+
creating a brand new anonymous patient on every single submission.
|
| 57 |
+
"""
|
| 58 |
+
if "user_id" not in session:
|
| 59 |
+
user_id = database.add_user(name="Patient", preferred_language=language, role="patient")
|
| 60 |
+
session_id = database.add_session(
|
| 61 |
+
user_id=user_id,
|
| 62 |
+
start_time=datetime.now(timezone.utc).isoformat(),
|
| 63 |
+
)
|
| 64 |
+
session["user_id"] = user_id
|
| 65 |
+
session["session_id"] = session_id
|
| 66 |
+
|
| 67 |
+
return session["user_id"]
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
@app.route("/")
|
| 71 |
+
def index():
|
| 72 |
+
return render_template("patient.html", languages=translation.SUPPORTED_LANGUAGES)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
@app.route("/provider")
|
| 76 |
+
def provider_page():
|
| 77 |
+
return render_template("provider.html")
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
@app.route("/patient-input", methods=["POST"])
|
| 81 |
+
def patient_input():
|
| 82 |
+
language = request.form.get("language")
|
| 83 |
+
if not language or language.lower() not in translation.SUPPORTED_LANGUAGES:
|
| 84 |
+
return jsonify({"error": f"language must be one of {translation.SUPPORTED_LANGUAGES}"}), 400
|
| 85 |
+
language = language.lower()
|
| 86 |
+
|
| 87 |
+
text = request.form.get("text")
|
| 88 |
+
audio_file = request.files.get("audio")
|
| 89 |
+
|
| 90 |
+
if not text and not audio_file:
|
| 91 |
+
return jsonify({"error": "provide either text or audio"}), 400
|
| 92 |
+
|
| 93 |
+
if audio_file:
|
| 94 |
+
temp_path = os.path.join(AUDIO_DIR, f"upload_{uuid.uuid4().hex}.wav")
|
| 95 |
+
audio_file.save(temp_path)
|
| 96 |
+
try:
|
| 97 |
+
patient_text = stt.transcribe_audio(temp_path, language)
|
| 98 |
+
finally:
|
| 99 |
+
os.remove(temp_path)
|
| 100 |
+
else:
|
| 101 |
+
patient_text = text
|
| 102 |
+
|
| 103 |
+
english_text = translation.translate(patient_text, language, "english")
|
| 104 |
+
nlu_summary = nlu.interpret_query(english_text)
|
| 105 |
+
|
| 106 |
+
user_id = get_current_user_id(language)
|
| 107 |
+
interaction_id = database.add_interaction(
|
| 108 |
+
user_id=user_id,
|
| 109 |
+
input_text=patient_text,
|
| 110 |
+
detected_language=language,
|
| 111 |
+
translated_text=english_text,
|
| 112 |
+
nlu_summary=nlu_summary,
|
| 113 |
+
timestamp=datetime.now(timezone.utc).isoformat(),
|
| 114 |
+
)
|
| 115 |
+
|
| 116 |
+
return jsonify({
|
| 117 |
+
"interaction_id": interaction_id,
|
| 118 |
+
"patient_text": patient_text,
|
| 119 |
+
"translated_text": english_text,
|
| 120 |
+
"nlu_summary": nlu_summary,
|
| 121 |
+
})
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
@app.route("/provider-response", methods=["POST"])
|
| 125 |
+
def provider_response():
|
| 126 |
+
interaction_id = request.form.get("interaction_id")
|
| 127 |
+
response_text = request.form.get("response_text")
|
| 128 |
+
|
| 129 |
+
if not interaction_id or not response_text:
|
| 130 |
+
return jsonify({"error": "interaction_id and response_text are required"}), 400
|
| 131 |
+
|
| 132 |
+
interaction = database.get_interaction(interaction_id)
|
| 133 |
+
if not interaction:
|
| 134 |
+
return jsonify({"error": "interaction not found"}), 404
|
| 135 |
+
|
| 136 |
+
patient_language = interaction["detected_language"]
|
| 137 |
+
translated_response = translation.translate(response_text, "english", patient_language)
|
| 138 |
+
|
| 139 |
+
audio_filename = f"response_{interaction_id}.wav"
|
| 140 |
+
audio_path = os.path.join(AUDIO_DIR, audio_filename)
|
| 141 |
+
tts.synthesize_speech(translated_response, patient_language, audio_path)
|
| 142 |
+
|
| 143 |
+
database.update_interaction_response(interaction_id, response_text, translated_response)
|
| 144 |
+
|
| 145 |
+
return jsonify({
|
| 146 |
+
"translated_response": translated_response,
|
| 147 |
+
"audio_url": f"/static/audio/{audio_filename}",
|
| 148 |
+
})
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
@app.route("/interaction/<interaction_id>")
|
| 152 |
+
def get_interaction(interaction_id):
|
| 153 |
+
interaction = database.get_interaction(interaction_id)
|
| 154 |
+
if not interaction:
|
| 155 |
+
return jsonify({"error": "interaction not found"}), 404
|
| 156 |
+
|
| 157 |
+
audio_filename = f"response_{interaction_id}.wav"
|
| 158 |
+
audio_path = os.path.join(AUDIO_DIR, audio_filename)
|
| 159 |
+
interaction["audio_url"] = f"/static/audio/{audio_filename}" if os.path.exists(audio_path) else None
|
| 160 |
+
|
| 161 |
+
return jsonify(interaction)
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
@app.route("/pending-interactions")
|
| 165 |
+
def pending_interactions():
|
| 166 |
+
return jsonify(database.get_pending_interactions())
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
@app.route("/history")
|
| 170 |
+
def history():
|
| 171 |
+
if "user_id" not in session:
|
| 172 |
+
return jsonify([])
|
| 173 |
+
return jsonify(database.get_interactions_for_user(session["user_id"]))
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
@app.route("/end-session", methods=["POST"])
|
| 177 |
+
def end_session_route():
|
| 178 |
+
if "session_id" in session:
|
| 179 |
+
database.end_session(session["session_id"], datetime.now(timezone.utc).isoformat())
|
| 180 |
+
session.clear()
|
| 181 |
+
return jsonify({"status": "ended"})
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
if __name__ == "__main__":
|
| 185 |
+
is_production = os.environ.get("ENVIRONMENT") == "production"
|
| 186 |
+
app.run(debug=not is_production, host="0.0.0.0", port=int(os.environ.get("PORT", 5000)))
|
database.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Database layer for TalkToDoc.
|
| 3 |
+
Local development and testing use SQLite (no setup needed). When deployed,
|
| 4 |
+
DATABASE_URL is set by Render and the app uses Postgres instead, since
|
| 5 |
+
Render's free tier wipes local files like a SQLite database on every
|
| 6 |
+
restart.
|
| 7 |
+
|
| 8 |
+
Queries are written once using SQLite-style '?' placeholders and adapted
|
| 9 |
+
automatically for Postgres, so there's a single query per function rather
|
| 10 |
+
than two versions of everything.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
import os
|
| 14 |
+
from pathlib import Path
|
| 15 |
+
|
| 16 |
+
import sqlite3
|
| 17 |
+
|
| 18 |
+
DB_PATH = Path(__file__).parent / "talktodoc.db"
|
| 19 |
+
SCHEMA_PATH = Path(__file__).parent / "schema.sql"
|
| 20 |
+
SCHEMA_PATH_POSTGRES = Path(__file__).parent / "schema_postgres.sql"
|
| 21 |
+
|
| 22 |
+
DATABASE_URL = os.environ.get("DATABASE_URL")
|
| 23 |
+
|
| 24 |
+
if DATABASE_URL:
|
| 25 |
+
import psycopg2
|
| 26 |
+
import psycopg2.extras
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def get_connection():
|
| 30 |
+
if DATABASE_URL:
|
| 31 |
+
return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
|
| 32 |
+
connection = sqlite3.connect(DB_PATH)
|
| 33 |
+
connection.row_factory = sqlite3.Row
|
| 34 |
+
connection.execute("PRAGMA foreign_keys = ON")
|
| 35 |
+
return connection
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def _run(connection, query, params=()):
|
| 39 |
+
if DATABASE_URL:
|
| 40 |
+
query = query.replace("?", "%s")
|
| 41 |
+
cursor = connection.cursor()
|
| 42 |
+
cursor.execute(query, params)
|
| 43 |
+
return cursor
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def _insert_and_get_id(connection, query, params):
|
| 47 |
+
if DATABASE_URL:
|
| 48 |
+
cursor = _run(connection, query + " RETURNING id", params)
|
| 49 |
+
return cursor.fetchone()["id"]
|
| 50 |
+
cursor = _run(connection, query, params)
|
| 51 |
+
return cursor.lastrowid
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def init_db():
|
| 55 |
+
schema_path = SCHEMA_PATH_POSTGRES if DATABASE_URL else SCHEMA_PATH
|
| 56 |
+
schema_sql = schema_path.read_text()
|
| 57 |
+
with get_connection() as connection:
|
| 58 |
+
if DATABASE_URL:
|
| 59 |
+
connection.cursor().execute(schema_sql)
|
| 60 |
+
else:
|
| 61 |
+
connection.executescript(schema_sql)
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def add_user(name, preferred_language, role):
|
| 65 |
+
with get_connection() as connection:
|
| 66 |
+
return _insert_and_get_id(
|
| 67 |
+
connection,
|
| 68 |
+
"INSERT INTO app_user (name, preferred_language, role) VALUES (?, ?, ?)",
|
| 69 |
+
(name, preferred_language, role),
|
| 70 |
+
)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def add_session(user_id, start_time):
|
| 74 |
+
with get_connection() as connection:
|
| 75 |
+
return _insert_and_get_id(
|
| 76 |
+
connection,
|
| 77 |
+
"INSERT INTO session (user_id, start_time) VALUES (?, ?)",
|
| 78 |
+
(user_id, start_time),
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def end_session(session_id, end_time):
|
| 83 |
+
with get_connection() as connection:
|
| 84 |
+
_run(connection, "UPDATE session SET end_time = ? WHERE id = ?", (end_time, session_id))
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def add_interaction(user_id, input_text, detected_language, translated_text, nlu_summary, timestamp):
|
| 88 |
+
with get_connection() as connection:
|
| 89 |
+
return _insert_and_get_id(
|
| 90 |
+
connection,
|
| 91 |
+
"""INSERT INTO interaction
|
| 92 |
+
(user_id, input_text, detected_language, translated_text, nlu_summary, timestamp)
|
| 93 |
+
VALUES (?, ?, ?, ?, ?, ?)""",
|
| 94 |
+
(user_id, input_text, detected_language, translated_text, nlu_summary, timestamp),
|
| 95 |
+
)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def get_interactions_for_user(user_id):
|
| 99 |
+
with get_connection() as connection:
|
| 100 |
+
cursor = _run(
|
| 101 |
+
connection,
|
| 102 |
+
"SELECT * FROM interaction WHERE user_id = ? ORDER BY timestamp",
|
| 103 |
+
(user_id,),
|
| 104 |
+
)
|
| 105 |
+
return [dict(row) for row in cursor.fetchall()]
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def get_pending_interactions():
|
| 109 |
+
with get_connection() as connection:
|
| 110 |
+
cursor = _run(
|
| 111 |
+
connection,
|
| 112 |
+
"SELECT * FROM interaction WHERE provider_response IS NULL ORDER BY timestamp",
|
| 113 |
+
)
|
| 114 |
+
return [dict(row) for row in cursor.fetchall()]
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def get_interaction(interaction_id):
|
| 118 |
+
with get_connection() as connection:
|
| 119 |
+
cursor = _run(connection, "SELECT * FROM interaction WHERE id = ?", (interaction_id,))
|
| 120 |
+
row = cursor.fetchone()
|
| 121 |
+
return dict(row) if row else None
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def update_interaction_response(interaction_id, provider_response, translated_response):
|
| 125 |
+
with get_connection() as connection:
|
| 126 |
+
_run(
|
| 127 |
+
connection,
|
| 128 |
+
"UPDATE interaction SET provider_response = ?, translated_response = ? WHERE id = ?",
|
| 129 |
+
(provider_response, translated_response, interaction_id),
|
| 130 |
+
)
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
if __name__ == "__main__":
|
| 134 |
+
init_db()
|
| 135 |
+
if DATABASE_URL:
|
| 136 |
+
print("Database initialized (Postgres)")
|
| 137 |
+
else:
|
| 138 |
+
print("Database created at:", DB_PATH)
|
nlu.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Natural language understanding for TalkToDoc.
|
| 3 |
+
Interprets basic health-related queries to identify symptoms, intent, or
|
| 4 |
+
information requests, per functional requirement 5 in the research
|
| 5 |
+
document.
|
| 6 |
+
|
| 7 |
+
This is a communication aid, not a diagnostic tool. The document is
|
| 8 |
+
explicit that the system does not provide formal medical diagnosis, so
|
| 9 |
+
this module only summarizes what the patient is communicating, it never
|
| 10 |
+
suggests a diagnosis or treatment.
|
| 11 |
+
|
| 12 |
+
Uses the Claude API (Anthropic), same as translation.py.
|
| 13 |
+
|
| 14 |
+
MOCK_MODE: if set to "true" in .env, this uses simple keyword matching
|
| 15 |
+
instead of a real API call, so the rest of the app can be tested for
|
| 16 |
+
free, with no API key. Not a substitute for testing real NLU quality.
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
import os
|
| 20 |
+
from dotenv import load_dotenv
|
| 21 |
+
|
| 22 |
+
load_dotenv()
|
| 23 |
+
|
| 24 |
+
MOCK_MODE = os.environ.get("MOCK_MODE", "").lower() == "true"
|
| 25 |
+
|
| 26 |
+
if not MOCK_MODE:
|
| 27 |
+
from anthropic import Anthropic
|
| 28 |
+
|
| 29 |
+
client = Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"])
|
| 30 |
+
MODEL = "claude-haiku-4-5-20251001"
|
| 31 |
+
|
| 32 |
+
_MOCK_KEYWORDS = ["headache", "fever", "stomach", "cough", "dizzy", "pain", "vomit", "rash"]
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def interpret_query(english_text):
|
| 36 |
+
"""
|
| 37 |
+
english_text: the patient's message, already translated to English
|
| 38 |
+
Returns a short plain-language summary of the likely symptoms, intent,
|
| 39 |
+
or information request, to help the provider quickly understand what
|
| 40 |
+
the patient needs. Not a diagnosis.
|
| 41 |
+
"""
|
| 42 |
+
if MOCK_MODE:
|
| 43 |
+
text_lower = english_text.lower()
|
| 44 |
+
found = [word for word in _MOCK_KEYWORDS if word in text_lower]
|
| 45 |
+
symptoms = ", ".join(found) if found else "an unspecified concern"
|
| 46 |
+
return f"Patient reports {symptoms}. Requesting guidance. (Mock summary, no AI used.)"
|
| 47 |
+
|
| 48 |
+
prompt = (
|
| 49 |
+
"A patient sent the following message to a healthcare provider. "
|
| 50 |
+
"In two sentences or less, summarize the likely symptoms, intent, "
|
| 51 |
+
"or information request. Do not diagnose or suggest treatment, "
|
| 52 |
+
"only summarize what the patient is communicating.\n\n"
|
| 53 |
+
f"Message: {english_text}"
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
response = client.messages.create(
|
| 57 |
+
model=MODEL,
|
| 58 |
+
max_tokens=200,
|
| 59 |
+
messages=[{"role": "user", "content": prompt}],
|
| 60 |
+
)
|
| 61 |
+
|
| 62 |
+
return response.content[0].text.strip()
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
if __name__ == "__main__":
|
| 66 |
+
import sys
|
| 67 |
+
|
| 68 |
+
if len(sys.argv) < 2:
|
| 69 |
+
print('Usage: python nlu.py "english text"')
|
| 70 |
+
else:
|
| 71 |
+
print(interpret_query(sys.argv[1]))
|
requirements.txt
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
flask==3.1.3
|
| 2 |
+
python-dotenv==1.2.2
|
| 3 |
+
anthropic==0.117.0
|
| 4 |
+
openai-whisper==20250625
|
| 5 |
+
transformers==5.14.1
|
| 6 |
+
torch==2.11.0
|
| 7 |
+
torchaudio==2.11.0
|
| 8 |
+
scipy==1.18.0
|
| 9 |
+
yarngpt==0.2.0
|
| 10 |
+
psycopg2-binary==2.9.12
|
| 11 |
+
gunicorn==26.0.0
|
schema.sql
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
CREATE TABLE IF NOT EXISTS app_user (
|
| 2 |
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 3 |
+
name TEXT NOT NULL,
|
| 4 |
+
preferred_language TEXT NOT NULL,
|
| 5 |
+
role TEXT NOT NULL CHECK (role IN ('patient', 'provider'))
|
| 6 |
+
);
|
| 7 |
+
|
| 8 |
+
CREATE TABLE IF NOT EXISTS session (
|
| 9 |
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 10 |
+
user_id INTEGER NOT NULL,
|
| 11 |
+
start_time TEXT NOT NULL,
|
| 12 |
+
end_time TEXT,
|
| 13 |
+
FOREIGN KEY (user_id) REFERENCES app_user (id)
|
| 14 |
+
);
|
| 15 |
+
|
| 16 |
+
CREATE TABLE IF NOT EXISTS interaction (
|
| 17 |
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 18 |
+
user_id INTEGER NOT NULL,
|
| 19 |
+
input_text TEXT,
|
| 20 |
+
detected_language TEXT,
|
| 21 |
+
translated_text TEXT,
|
| 22 |
+
nlu_summary TEXT,
|
| 23 |
+
provider_response TEXT,
|
| 24 |
+
translated_response TEXT,
|
| 25 |
+
timestamp TEXT NOT NULL,
|
| 26 |
+
FOREIGN KEY (user_id) REFERENCES app_user (id)
|
| 27 |
+
);
|
schema_postgres.sql
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
CREATE TABLE IF NOT EXISTS app_user (
|
| 2 |
+
id SERIAL PRIMARY KEY,
|
| 3 |
+
name TEXT NOT NULL,
|
| 4 |
+
preferred_language TEXT NOT NULL,
|
| 5 |
+
role TEXT NOT NULL CHECK (role IN ('patient', 'provider'))
|
| 6 |
+
);
|
| 7 |
+
|
| 8 |
+
CREATE TABLE IF NOT EXISTS session (
|
| 9 |
+
id SERIAL PRIMARY KEY,
|
| 10 |
+
user_id INTEGER NOT NULL REFERENCES app_user (id),
|
| 11 |
+
start_time TEXT NOT NULL,
|
| 12 |
+
end_time TEXT
|
| 13 |
+
);
|
| 14 |
+
|
| 15 |
+
CREATE TABLE IF NOT EXISTS interaction (
|
| 16 |
+
id SERIAL PRIMARY KEY,
|
| 17 |
+
user_id INTEGER NOT NULL REFERENCES app_user (id),
|
| 18 |
+
input_text TEXT,
|
| 19 |
+
detected_language TEXT,
|
| 20 |
+
translated_text TEXT,
|
| 21 |
+
nlu_summary TEXT,
|
| 22 |
+
provider_response TEXT,
|
| 23 |
+
translated_response TEXT,
|
| 24 |
+
timestamp TEXT NOT NULL
|
| 25 |
+
);
|
stt.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Speech-to-text for TalkToDoc.
|
| 3 |
+
Converts a patient's spoken audio into text using a local Whisper model.
|
| 4 |
+
Language is passed in from the manual language selector, not auto-detected.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import whisper
|
| 8 |
+
|
| 9 |
+
_model = None
|
| 10 |
+
|
| 11 |
+
# Maps the app's language selection to Whisper's language codes.
|
| 12 |
+
# Whisper has no dedicated Nigerian Pidgin code, so Pidgin is passed as
|
| 13 |
+
# English, since Pidgin is close enough to English that this gives Whisper
|
| 14 |
+
# the best chance of an accurate transcription.
|
| 15 |
+
LANGUAGE_MAP = {
|
| 16 |
+
"english": "en",
|
| 17 |
+
"yoruba": "yo",
|
| 18 |
+
"hausa": "ha",
|
| 19 |
+
"igbo": "ig",
|
| 20 |
+
"pidgin": "en",
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _get_model():
|
| 25 |
+
global _model
|
| 26 |
+
if _model is None:
|
| 27 |
+
_model = whisper.load_model("base")
|
| 28 |
+
return _model
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def transcribe_audio(audio_path, language=None):
|
| 32 |
+
"""
|
| 33 |
+
audio_path: path to an audio file (wav, mp3, m4a, etc.)
|
| 34 |
+
language: one of "english", "yoruba", "hausa", "igbo", "pidgin"
|
| 35 |
+
Returns the transcribed text.
|
| 36 |
+
"""
|
| 37 |
+
model = _get_model()
|
| 38 |
+
whisper_language = LANGUAGE_MAP.get(language.lower()) if language else None
|
| 39 |
+
result = model.transcribe(audio_path, language=whisper_language)
|
| 40 |
+
return result["text"].strip()
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
if __name__ == "__main__":
|
| 44 |
+
import sys
|
| 45 |
+
|
| 46 |
+
if len(sys.argv) < 2:
|
| 47 |
+
print("Usage: python stt.py <audio_file> [language]")
|
| 48 |
+
else:
|
| 49 |
+
audio_file = sys.argv[1]
|
| 50 |
+
selected_language = sys.argv[2] if len(sys.argv) > 2 else None
|
| 51 |
+
print(transcribe_audio(audio_file, selected_language))
|
translation.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Translation for TalkToDoc.
|
| 3 |
+
Translates patient input into English for the provider, and translates the
|
| 4 |
+
provider's reply back into the patient's selected language.
|
| 5 |
+
|
| 6 |
+
Uses the Claude API (Anthropic), as specified in the research document's
|
| 7 |
+
software requirements (3.5.4).
|
| 8 |
+
|
| 9 |
+
MOCK_MODE: if set to "true" in .env, this skips the real API call
|
| 10 |
+
entirely and returns the input text unchanged instead. This exists so
|
| 11 |
+
the whole app (Whisper, YarnGPT, MMS-TTS, the database, all the routing
|
| 12 |
+
and session logic) can be run and tested locally with zero cost and no
|
| 13 |
+
API key, before deciding to actually acquire one. It is not a substitute
|
| 14 |
+
for testing real translation quality, only for testing everything else.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
import os
|
| 18 |
+
from dotenv import load_dotenv
|
| 19 |
+
|
| 20 |
+
load_dotenv()
|
| 21 |
+
|
| 22 |
+
MOCK_MODE = os.environ.get("MOCK_MODE", "").lower() == "true"
|
| 23 |
+
|
| 24 |
+
if not MOCK_MODE:
|
| 25 |
+
from anthropic import Anthropic
|
| 26 |
+
|
| 27 |
+
client = Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"])
|
| 28 |
+
MODEL = "claude-haiku-4-5-20251001"
|
| 29 |
+
|
| 30 |
+
SUPPORTED_LANGUAGES = ["english", "yoruba", "hausa", "igbo", "pidgin"]
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def translate(text, source_language, target_language):
|
| 34 |
+
"""
|
| 35 |
+
text: the text to translate
|
| 36 |
+
source_language: one of "english", "yoruba", "hausa", "igbo", "pidgin"
|
| 37 |
+
target_language: one of "english", "yoruba", "hausa", "igbo", "pidgin"
|
| 38 |
+
Returns the translated text only.
|
| 39 |
+
"""
|
| 40 |
+
if MOCK_MODE:
|
| 41 |
+
if source_language.lower() == target_language.lower():
|
| 42 |
+
return text
|
| 43 |
+
return f"[MOCK, no real translation: {source_language} -> {target_language}] {text}"
|
| 44 |
+
|
| 45 |
+
prompt = (
|
| 46 |
+
f"Translate the following text from {source_language} to {target_language}. "
|
| 47 |
+
f"Reply with only the translated text and nothing else, no explanation.\n\n"
|
| 48 |
+
f"Text: {text}"
|
| 49 |
+
)
|
| 50 |
+
|
| 51 |
+
response = client.messages.create(
|
| 52 |
+
model=MODEL,
|
| 53 |
+
max_tokens=500,
|
| 54 |
+
messages=[{"role": "user", "content": prompt}],
|
| 55 |
+
)
|
| 56 |
+
|
| 57 |
+
return response.content[0].text.strip()
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
if __name__ == "__main__":
|
| 61 |
+
import sys
|
| 62 |
+
|
| 63 |
+
if len(sys.argv) < 4:
|
| 64 |
+
print('Usage: python translation.py "text" source_language target_language')
|
| 65 |
+
else:
|
| 66 |
+
input_text = sys.argv[1]
|
| 67 |
+
source_lang = sys.argv[2]
|
| 68 |
+
target_lang = sys.argv[3]
|
| 69 |
+
print(translate(input_text, source_lang, target_lang))
|
tts.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Text-to-speech for TalkToDoc.
|
| 3 |
+
Converts translated text responses into audible speech in the patient's
|
| 4 |
+
selected language.
|
| 5 |
+
|
| 6 |
+
English, Yoruba, Igbo, and Hausa use YarnGPT, a local model trained
|
| 7 |
+
specifically on Nigerian-accented speech.
|
| 8 |
+
|
| 9 |
+
Nigerian Pidgin uses MMS-TTS (facebook/mms-tts-pcm) instead, since YarnGPT
|
| 10 |
+
has no Pidgin support. MMS-TTS is the only free option found with a
|
| 11 |
+
dedicated Pidgin checkpoint.
|
| 12 |
+
|
| 13 |
+
Note: importing this file triggers a one-time download of YarnGPT's audio
|
| 14 |
+
tokenizer files. This is expected and only happens once.
|
| 15 |
+
|
| 16 |
+
Performance note: YarnGPT's own generate_speech() function reloads its
|
| 17 |
+
full model from disk on every call, which is too slow for a live app.
|
| 18 |
+
This file loads the model once and reuses it, using the same generation
|
| 19 |
+
steps YarnGPT's own function uses internally, just without the reload.
|
| 20 |
+
Call preload_models() once when the app starts, so the first real request
|
| 21 |
+
isn't slow either.
|
| 22 |
+
"""
|
| 23 |
+
|
| 24 |
+
import torch
|
| 25 |
+
import torchaudio
|
| 26 |
+
import scipy.io.wavfile
|
| 27 |
+
from transformers import VitsModel, AutoTokenizer
|
| 28 |
+
from yarngpt.core import load_model_and_tokenizer, SPEAKER_MAPPING, AVAILABLE_SPEAKERS
|
| 29 |
+
|
| 30 |
+
# Default speaker used for each YarnGPT-supported language.
|
| 31 |
+
YARNGPT_SPEAKERS = {
|
| 32 |
+
"english": "idera",
|
| 33 |
+
"yoruba": "abayomi",
|
| 34 |
+
"igbo": "chioma",
|
| 35 |
+
"hausa": "amina",
|
| 36 |
+
}
|
| 37 |
+
|
| 38 |
+
_yarngpt_model = None
|
| 39 |
+
_yarngpt_tokenizer = None
|
| 40 |
+
|
| 41 |
+
_pidgin_model = None
|
| 42 |
+
_pidgin_tokenizer = None
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _get_yarngpt_model():
|
| 46 |
+
global _yarngpt_model, _yarngpt_tokenizer
|
| 47 |
+
if _yarngpt_model is None:
|
| 48 |
+
_yarngpt_model, _yarngpt_tokenizer = load_model_and_tokenizer()
|
| 49 |
+
return _yarngpt_model, _yarngpt_tokenizer
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _get_pidgin_model():
|
| 53 |
+
global _pidgin_model, _pidgin_tokenizer
|
| 54 |
+
if _pidgin_model is None:
|
| 55 |
+
_pidgin_tokenizer = AutoTokenizer.from_pretrained("facebook/mms-tts-pcm")
|
| 56 |
+
_pidgin_model = VitsModel.from_pretrained("facebook/mms-tts-pcm")
|
| 57 |
+
return _pidgin_model, _pidgin_tokenizer
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def preload_models():
|
| 61 |
+
"""Loads both TTS backends into memory ahead of time. Call this once
|
| 62 |
+
when the Flask app starts, so the first real request isn't slow."""
|
| 63 |
+
_get_yarngpt_model()
|
| 64 |
+
_get_pidgin_model()
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def _generate_yarngpt_speech(text, speaker, language, temperature=0.1, repetition_penalty=1.1, max_length=4000):
|
| 68 |
+
"""
|
| 69 |
+
Same steps as yarngpt's own generate_speech(), but reuses the model
|
| 70 |
+
already loaded by _get_yarngpt_model() instead of reloading it.
|
| 71 |
+
"""
|
| 72 |
+
model_speaker = SPEAKER_MAPPING.get(speaker, speaker)
|
| 73 |
+
if model_speaker not in AVAILABLE_SPEAKERS:
|
| 74 |
+
raise ValueError(f"Unknown speaker: {speaker}")
|
| 75 |
+
|
| 76 |
+
model, audio_tokenizer = _get_yarngpt_model()
|
| 77 |
+
|
| 78 |
+
prompt = audio_tokenizer.create_prompt(text, language, model_speaker)
|
| 79 |
+
input_ids = audio_tokenizer.tokenize_prompt(prompt)
|
| 80 |
+
attention_mask = torch.ones_like(input_ids)
|
| 81 |
+
|
| 82 |
+
output = model.generate(
|
| 83 |
+
input_ids=input_ids,
|
| 84 |
+
attention_mask=attention_mask,
|
| 85 |
+
do_sample=True,
|
| 86 |
+
temperature=temperature,
|
| 87 |
+
repetition_penalty=repetition_penalty,
|
| 88 |
+
max_length=max_length,
|
| 89 |
+
pad_token_id=model.config.eos_token_id,
|
| 90 |
+
eos_token_id=model.config.eos_token_id,
|
| 91 |
+
)
|
| 92 |
+
|
| 93 |
+
codes = audio_tokenizer.get_codes(output)
|
| 94 |
+
audio = audio_tokenizer.get_audio(codes)
|
| 95 |
+
return audio
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def synthesize_speech(text, language, output_path):
|
| 99 |
+
"""
|
| 100 |
+
text: the text to speak
|
| 101 |
+
language: one of "english", "yoruba", "hausa", "igbo", "pidgin"
|
| 102 |
+
output_path: where to save the resulting .wav file
|
| 103 |
+
Returns output_path.
|
| 104 |
+
"""
|
| 105 |
+
language = language.lower()
|
| 106 |
+
|
| 107 |
+
if language in YARNGPT_SPEAKERS:
|
| 108 |
+
speaker = YARNGPT_SPEAKERS[language]
|
| 109 |
+
audio = _generate_yarngpt_speech(text, speaker=speaker, language=language)
|
| 110 |
+
torchaudio.save(output_path, audio, sample_rate=24000)
|
| 111 |
+
return output_path
|
| 112 |
+
|
| 113 |
+
if language == "pidgin":
|
| 114 |
+
model, tokenizer = _get_pidgin_model()
|
| 115 |
+
inputs = tokenizer(text, return_tensors="pt")
|
| 116 |
+
with torch.no_grad():
|
| 117 |
+
output = model(**inputs).waveform
|
| 118 |
+
scipy.io.wavfile.write(
|
| 119 |
+
output_path,
|
| 120 |
+
rate=model.config.sampling_rate,
|
| 121 |
+
data=output.numpy().squeeze(),
|
| 122 |
+
)
|
| 123 |
+
return output_path
|
| 124 |
+
|
| 125 |
+
raise ValueError(f"Unsupported language: {language}")
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
if __name__ == "__main__":
|
| 129 |
+
import sys
|
| 130 |
+
|
| 131 |
+
if len(sys.argv) < 3:
|
| 132 |
+
print('Usage: python tts.py "text" language [output_file]')
|
| 133 |
+
else:
|
| 134 |
+
input_text = sys.argv[1]
|
| 135 |
+
selected_language = sys.argv[2]
|
| 136 |
+
output_file = sys.argv[3] if len(sys.argv) > 3 else "output.wav"
|
| 137 |
+
synthesize_speech(input_text, selected_language, output_file)
|
| 138 |
+
print("Saved to", output_file)
|