Glitche commited on
Commit
4fcd019
·
verified ·
1 Parent(s): 33fd8e7

Upload 12 files

Browse files
Files changed (12) hide show
  1. Dockerfile +35 -0
  2. Procfile +1 -0
  3. README.md +18 -5
  4. app.py +186 -0
  5. database.py +138 -0
  6. nlu.py +71 -0
  7. requirements.txt +11 -0
  8. schema.sql +27 -0
  9. schema_postgres.sql +25 -0
  10. stt.py +51 -0
  11. translation.py +69 -0
  12. tts.py +138 -0
Dockerfile ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # TalkToDoc production image.
2
+ # Built to be interchangeable across hosts: Hugging Face Spaces, Render
3
+ # (Docker deploy), Fly.io, Cloud Run, or any other platform that runs a
4
+ # standard Docker image. Nothing here is tied to one specific host.
5
+ # Configuration comes entirely from environment variables set by
6
+ # whichever platform runs it (PORT, DATABASE_URL, ANTHROPIC_API_KEY,
7
+ # SECRET_KEY, ENVIRONMENT).
8
+
9
+ FROM python:3.11-slim
10
+
11
+ # Whisper needs ffmpeg to read audio files, this installs it at the
12
+ # system level since it can't come from pip.
13
+ RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg \
14
+ && rm -rf /var/lib/apt/lists/*
15
+
16
+ # Run as a non-root user, standard practice and specifically expected by
17
+ # Hugging Face Spaces' Docker SDK.
18
+ RUN useradd -m -u 1000 user
19
+ USER user
20
+ ENV PATH="/home/user/.local/bin:$PATH"
21
+
22
+ WORKDIR /home/user/app
23
+
24
+ COPY --chown=user requirements.txt .
25
+ RUN pip install --no-cache-dir --upgrade -r requirements.txt
26
+
27
+ COPY --chown=user . .
28
+
29
+ ENV ENVIRONMENT=production
30
+
31
+ # 7860 is Hugging Face Spaces' default port. Other platforms (like
32
+ # Render) set $PORT themselves, and this falls back to that automatically.
33
+ EXPOSE 7860
34
+
35
+ CMD gunicorn app:app --bind 0.0.0.0:${PORT:-7860} --timeout 120 --workers 1
Procfile ADDED
@@ -0,0 +1 @@
 
 
1
+ web: gunicorn app:app --bind 0.0.0.0:$PORT --timeout 120 --workers 1
README.md CHANGED
@@ -1,10 +1,23 @@
1
  ---
2
  title: TalkToDoc
3
- emoji: 📊
4
- colorFrom: yellow
5
- colorTo: blue
6
  sdk: docker
7
- pinned: false
8
  ---
9
 
10
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
  title: TalkToDoc
3
+ emoji: 🩺
4
+ colorFrom: green
5
+ colorTo: gray
6
  sdk: docker
7
+ app_port: 7860
8
  ---
9
 
10
+ # TalkToDoc
11
+
12
+ A multilingual communication tool that helps patients and healthcare
13
+ providers understand each other across English, Yoruba, Hausa, Igbo, and
14
+ Nigerian Pidgin. Built for rural Nigerian healthcare settings where
15
+ doctor shortages and language barriers contribute to misdiagnosis and
16
+ communication breakdowns.
17
+
18
+ This Space runs the full application: speech-to-text, translation,
19
+ symptom summarization, and text-to-speech, in the patient's own
20
+ language.
21
+
22
+ Requires ANTHROPIC_API_KEY, SECRET_KEY, and DATABASE_URL to be set as
23
+ secrets in this Space's settings.
app.py ADDED
@@ -0,0 +1,186 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ TalkToDoc Flask application.
3
+ Wires together speech-to-text, translation, natural language understanding,
4
+ and text-to-speech into the patient-provider communication flow described
5
+ in the research document.
6
+
7
+ Two-step flow:
8
+ 1. Patient submits input (text or audio) in their chosen language.
9
+ The system transcribes it if needed, translates it to English, and
10
+ summarizes the likely symptoms or intent for the provider.
11
+ 2. Provider submits a reply in English. The system translates it back
12
+ into the patient's language and generates spoken audio for it.
13
+ """
14
+
15
+ import os
16
+ import uuid
17
+ from datetime import datetime, timezone
18
+
19
+ from dotenv import load_dotenv
20
+ from flask import Flask, request, jsonify, render_template, session
21
+
22
+ import database
23
+ import stt
24
+ import translation
25
+ import nlu
26
+ import tts
27
+
28
+ load_dotenv()
29
+
30
+ AUDIO_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "static", "audio")
31
+
32
+
33
+ def create_app():
34
+ flask_app = Flask(__name__)
35
+ flask_app.secret_key = os.environ["SECRET_KEY"]
36
+ os.makedirs(AUDIO_DIR, exist_ok=True)
37
+ return flask_app
38
+
39
+
40
+ app = create_app()
41
+
42
+ # Runs whether the app is started directly (python app.py, local development)
43
+ # or imported by a production server like gunicorn (Render deployment).
44
+ # Initialization code inside "if __name__ == '__main__'" only runs on a
45
+ # direct start, gunicorn never triggers it, so anything the app needs to
46
+ # work at all has to happen here instead.
47
+ database.init_db()
48
+ tts.preload_models()
49
+
50
+
51
+ def get_current_user_id(language):
52
+ """
53
+ Returns the user_id for the current browser session, creating a new
54
+ user and session record the first time this browser is seen. This is
55
+ what keeps a patient's messages tied to the same person instead of
56
+ creating a brand new anonymous patient on every single submission.
57
+ """
58
+ if "user_id" not in session:
59
+ user_id = database.add_user(name="Patient", preferred_language=language, role="patient")
60
+ session_id = database.add_session(
61
+ user_id=user_id,
62
+ start_time=datetime.now(timezone.utc).isoformat(),
63
+ )
64
+ session["user_id"] = user_id
65
+ session["session_id"] = session_id
66
+
67
+ return session["user_id"]
68
+
69
+
70
+ @app.route("/")
71
+ def index():
72
+ return render_template("patient.html", languages=translation.SUPPORTED_LANGUAGES)
73
+
74
+
75
+ @app.route("/provider")
76
+ def provider_page():
77
+ return render_template("provider.html")
78
+
79
+
80
+ @app.route("/patient-input", methods=["POST"])
81
+ def patient_input():
82
+ language = request.form.get("language")
83
+ if not language or language.lower() not in translation.SUPPORTED_LANGUAGES:
84
+ return jsonify({"error": f"language must be one of {translation.SUPPORTED_LANGUAGES}"}), 400
85
+ language = language.lower()
86
+
87
+ text = request.form.get("text")
88
+ audio_file = request.files.get("audio")
89
+
90
+ if not text and not audio_file:
91
+ return jsonify({"error": "provide either text or audio"}), 400
92
+
93
+ if audio_file:
94
+ temp_path = os.path.join(AUDIO_DIR, f"upload_{uuid.uuid4().hex}.wav")
95
+ audio_file.save(temp_path)
96
+ try:
97
+ patient_text = stt.transcribe_audio(temp_path, language)
98
+ finally:
99
+ os.remove(temp_path)
100
+ else:
101
+ patient_text = text
102
+
103
+ english_text = translation.translate(patient_text, language, "english")
104
+ nlu_summary = nlu.interpret_query(english_text)
105
+
106
+ user_id = get_current_user_id(language)
107
+ interaction_id = database.add_interaction(
108
+ user_id=user_id,
109
+ input_text=patient_text,
110
+ detected_language=language,
111
+ translated_text=english_text,
112
+ nlu_summary=nlu_summary,
113
+ timestamp=datetime.now(timezone.utc).isoformat(),
114
+ )
115
+
116
+ return jsonify({
117
+ "interaction_id": interaction_id,
118
+ "patient_text": patient_text,
119
+ "translated_text": english_text,
120
+ "nlu_summary": nlu_summary,
121
+ })
122
+
123
+
124
+ @app.route("/provider-response", methods=["POST"])
125
+ def provider_response():
126
+ interaction_id = request.form.get("interaction_id")
127
+ response_text = request.form.get("response_text")
128
+
129
+ if not interaction_id or not response_text:
130
+ return jsonify({"error": "interaction_id and response_text are required"}), 400
131
+
132
+ interaction = database.get_interaction(interaction_id)
133
+ if not interaction:
134
+ return jsonify({"error": "interaction not found"}), 404
135
+
136
+ patient_language = interaction["detected_language"]
137
+ translated_response = translation.translate(response_text, "english", patient_language)
138
+
139
+ audio_filename = f"response_{interaction_id}.wav"
140
+ audio_path = os.path.join(AUDIO_DIR, audio_filename)
141
+ tts.synthesize_speech(translated_response, patient_language, audio_path)
142
+
143
+ database.update_interaction_response(interaction_id, response_text, translated_response)
144
+
145
+ return jsonify({
146
+ "translated_response": translated_response,
147
+ "audio_url": f"/static/audio/{audio_filename}",
148
+ })
149
+
150
+
151
+ @app.route("/interaction/<interaction_id>")
152
+ def get_interaction(interaction_id):
153
+ interaction = database.get_interaction(interaction_id)
154
+ if not interaction:
155
+ return jsonify({"error": "interaction not found"}), 404
156
+
157
+ audio_filename = f"response_{interaction_id}.wav"
158
+ audio_path = os.path.join(AUDIO_DIR, audio_filename)
159
+ interaction["audio_url"] = f"/static/audio/{audio_filename}" if os.path.exists(audio_path) else None
160
+
161
+ return jsonify(interaction)
162
+
163
+
164
+ @app.route("/pending-interactions")
165
+ def pending_interactions():
166
+ return jsonify(database.get_pending_interactions())
167
+
168
+
169
+ @app.route("/history")
170
+ def history():
171
+ if "user_id" not in session:
172
+ return jsonify([])
173
+ return jsonify(database.get_interactions_for_user(session["user_id"]))
174
+
175
+
176
+ @app.route("/end-session", methods=["POST"])
177
+ def end_session_route():
178
+ if "session_id" in session:
179
+ database.end_session(session["session_id"], datetime.now(timezone.utc).isoformat())
180
+ session.clear()
181
+ return jsonify({"status": "ended"})
182
+
183
+
184
+ if __name__ == "__main__":
185
+ is_production = os.environ.get("ENVIRONMENT") == "production"
186
+ app.run(debug=not is_production, host="0.0.0.0", port=int(os.environ.get("PORT", 5000)))
database.py ADDED
@@ -0,0 +1,138 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Database layer for TalkToDoc.
3
+ Local development and testing use SQLite (no setup needed). When deployed,
4
+ DATABASE_URL is set by Render and the app uses Postgres instead, since
5
+ Render's free tier wipes local files like a SQLite database on every
6
+ restart.
7
+
8
+ Queries are written once using SQLite-style '?' placeholders and adapted
9
+ automatically for Postgres, so there's a single query per function rather
10
+ than two versions of everything.
11
+ """
12
+
13
+ import os
14
+ from pathlib import Path
15
+
16
+ import sqlite3
17
+
18
+ DB_PATH = Path(__file__).parent / "talktodoc.db"
19
+ SCHEMA_PATH = Path(__file__).parent / "schema.sql"
20
+ SCHEMA_PATH_POSTGRES = Path(__file__).parent / "schema_postgres.sql"
21
+
22
+ DATABASE_URL = os.environ.get("DATABASE_URL")
23
+
24
+ if DATABASE_URL:
25
+ import psycopg2
26
+ import psycopg2.extras
27
+
28
+
29
+ def get_connection():
30
+ if DATABASE_URL:
31
+ return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
32
+ connection = sqlite3.connect(DB_PATH)
33
+ connection.row_factory = sqlite3.Row
34
+ connection.execute("PRAGMA foreign_keys = ON")
35
+ return connection
36
+
37
+
38
+ def _run(connection, query, params=()):
39
+ if DATABASE_URL:
40
+ query = query.replace("?", "%s")
41
+ cursor = connection.cursor()
42
+ cursor.execute(query, params)
43
+ return cursor
44
+
45
+
46
+ def _insert_and_get_id(connection, query, params):
47
+ if DATABASE_URL:
48
+ cursor = _run(connection, query + " RETURNING id", params)
49
+ return cursor.fetchone()["id"]
50
+ cursor = _run(connection, query, params)
51
+ return cursor.lastrowid
52
+
53
+
54
+ def init_db():
55
+ schema_path = SCHEMA_PATH_POSTGRES if DATABASE_URL else SCHEMA_PATH
56
+ schema_sql = schema_path.read_text()
57
+ with get_connection() as connection:
58
+ if DATABASE_URL:
59
+ connection.cursor().execute(schema_sql)
60
+ else:
61
+ connection.executescript(schema_sql)
62
+
63
+
64
+ def add_user(name, preferred_language, role):
65
+ with get_connection() as connection:
66
+ return _insert_and_get_id(
67
+ connection,
68
+ "INSERT INTO app_user (name, preferred_language, role) VALUES (?, ?, ?)",
69
+ (name, preferred_language, role),
70
+ )
71
+
72
+
73
+ def add_session(user_id, start_time):
74
+ with get_connection() as connection:
75
+ return _insert_and_get_id(
76
+ connection,
77
+ "INSERT INTO session (user_id, start_time) VALUES (?, ?)",
78
+ (user_id, start_time),
79
+ )
80
+
81
+
82
+ def end_session(session_id, end_time):
83
+ with get_connection() as connection:
84
+ _run(connection, "UPDATE session SET end_time = ? WHERE id = ?", (end_time, session_id))
85
+
86
+
87
+ def add_interaction(user_id, input_text, detected_language, translated_text, nlu_summary, timestamp):
88
+ with get_connection() as connection:
89
+ return _insert_and_get_id(
90
+ connection,
91
+ """INSERT INTO interaction
92
+ (user_id, input_text, detected_language, translated_text, nlu_summary, timestamp)
93
+ VALUES (?, ?, ?, ?, ?, ?)""",
94
+ (user_id, input_text, detected_language, translated_text, nlu_summary, timestamp),
95
+ )
96
+
97
+
98
+ def get_interactions_for_user(user_id):
99
+ with get_connection() as connection:
100
+ cursor = _run(
101
+ connection,
102
+ "SELECT * FROM interaction WHERE user_id = ? ORDER BY timestamp",
103
+ (user_id,),
104
+ )
105
+ return [dict(row) for row in cursor.fetchall()]
106
+
107
+
108
+ def get_pending_interactions():
109
+ with get_connection() as connection:
110
+ cursor = _run(
111
+ connection,
112
+ "SELECT * FROM interaction WHERE provider_response IS NULL ORDER BY timestamp",
113
+ )
114
+ return [dict(row) for row in cursor.fetchall()]
115
+
116
+
117
+ def get_interaction(interaction_id):
118
+ with get_connection() as connection:
119
+ cursor = _run(connection, "SELECT * FROM interaction WHERE id = ?", (interaction_id,))
120
+ row = cursor.fetchone()
121
+ return dict(row) if row else None
122
+
123
+
124
+ def update_interaction_response(interaction_id, provider_response, translated_response):
125
+ with get_connection() as connection:
126
+ _run(
127
+ connection,
128
+ "UPDATE interaction SET provider_response = ?, translated_response = ? WHERE id = ?",
129
+ (provider_response, translated_response, interaction_id),
130
+ )
131
+
132
+
133
+ if __name__ == "__main__":
134
+ init_db()
135
+ if DATABASE_URL:
136
+ print("Database initialized (Postgres)")
137
+ else:
138
+ print("Database created at:", DB_PATH)
nlu.py ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Natural language understanding for TalkToDoc.
3
+ Interprets basic health-related queries to identify symptoms, intent, or
4
+ information requests, per functional requirement 5 in the research
5
+ document.
6
+
7
+ This is a communication aid, not a diagnostic tool. The document is
8
+ explicit that the system does not provide formal medical diagnosis, so
9
+ this module only summarizes what the patient is communicating, it never
10
+ suggests a diagnosis or treatment.
11
+
12
+ Uses the Claude API (Anthropic), same as translation.py.
13
+
14
+ MOCK_MODE: if set to "true" in .env, this uses simple keyword matching
15
+ instead of a real API call, so the rest of the app can be tested for
16
+ free, with no API key. Not a substitute for testing real NLU quality.
17
+ """
18
+
19
+ import os
20
+ from dotenv import load_dotenv
21
+
22
+ load_dotenv()
23
+
24
+ MOCK_MODE = os.environ.get("MOCK_MODE", "").lower() == "true"
25
+
26
+ if not MOCK_MODE:
27
+ from anthropic import Anthropic
28
+
29
+ client = Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"])
30
+ MODEL = "claude-haiku-4-5-20251001"
31
+
32
+ _MOCK_KEYWORDS = ["headache", "fever", "stomach", "cough", "dizzy", "pain", "vomit", "rash"]
33
+
34
+
35
+ def interpret_query(english_text):
36
+ """
37
+ english_text: the patient's message, already translated to English
38
+ Returns a short plain-language summary of the likely symptoms, intent,
39
+ or information request, to help the provider quickly understand what
40
+ the patient needs. Not a diagnosis.
41
+ """
42
+ if MOCK_MODE:
43
+ text_lower = english_text.lower()
44
+ found = [word for word in _MOCK_KEYWORDS if word in text_lower]
45
+ symptoms = ", ".join(found) if found else "an unspecified concern"
46
+ return f"Patient reports {symptoms}. Requesting guidance. (Mock summary, no AI used.)"
47
+
48
+ prompt = (
49
+ "A patient sent the following message to a healthcare provider. "
50
+ "In two sentences or less, summarize the likely symptoms, intent, "
51
+ "or information request. Do not diagnose or suggest treatment, "
52
+ "only summarize what the patient is communicating.\n\n"
53
+ f"Message: {english_text}"
54
+ )
55
+
56
+ response = client.messages.create(
57
+ model=MODEL,
58
+ max_tokens=200,
59
+ messages=[{"role": "user", "content": prompt}],
60
+ )
61
+
62
+ return response.content[0].text.strip()
63
+
64
+
65
+ if __name__ == "__main__":
66
+ import sys
67
+
68
+ if len(sys.argv) < 2:
69
+ print('Usage: python nlu.py "english text"')
70
+ else:
71
+ print(interpret_query(sys.argv[1]))
requirements.txt ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ flask==3.1.3
2
+ python-dotenv==1.2.2
3
+ anthropic==0.117.0
4
+ openai-whisper==20250625
5
+ transformers==5.14.1
6
+ torch==2.11.0
7
+ torchaudio==2.11.0
8
+ scipy==1.18.0
9
+ yarngpt==0.2.0
10
+ psycopg2-binary==2.9.12
11
+ gunicorn==26.0.0
schema.sql ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ CREATE TABLE IF NOT EXISTS app_user (
2
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
3
+ name TEXT NOT NULL,
4
+ preferred_language TEXT NOT NULL,
5
+ role TEXT NOT NULL CHECK (role IN ('patient', 'provider'))
6
+ );
7
+
8
+ CREATE TABLE IF NOT EXISTS session (
9
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
10
+ user_id INTEGER NOT NULL,
11
+ start_time TEXT NOT NULL,
12
+ end_time TEXT,
13
+ FOREIGN KEY (user_id) REFERENCES app_user (id)
14
+ );
15
+
16
+ CREATE TABLE IF NOT EXISTS interaction (
17
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
18
+ user_id INTEGER NOT NULL,
19
+ input_text TEXT,
20
+ detected_language TEXT,
21
+ translated_text TEXT,
22
+ nlu_summary TEXT,
23
+ provider_response TEXT,
24
+ translated_response TEXT,
25
+ timestamp TEXT NOT NULL,
26
+ FOREIGN KEY (user_id) REFERENCES app_user (id)
27
+ );
schema_postgres.sql ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ CREATE TABLE IF NOT EXISTS app_user (
2
+ id SERIAL PRIMARY KEY,
3
+ name TEXT NOT NULL,
4
+ preferred_language TEXT NOT NULL,
5
+ role TEXT NOT NULL CHECK (role IN ('patient', 'provider'))
6
+ );
7
+
8
+ CREATE TABLE IF NOT EXISTS session (
9
+ id SERIAL PRIMARY KEY,
10
+ user_id INTEGER NOT NULL REFERENCES app_user (id),
11
+ start_time TEXT NOT NULL,
12
+ end_time TEXT
13
+ );
14
+
15
+ CREATE TABLE IF NOT EXISTS interaction (
16
+ id SERIAL PRIMARY KEY,
17
+ user_id INTEGER NOT NULL REFERENCES app_user (id),
18
+ input_text TEXT,
19
+ detected_language TEXT,
20
+ translated_text TEXT,
21
+ nlu_summary TEXT,
22
+ provider_response TEXT,
23
+ translated_response TEXT,
24
+ timestamp TEXT NOT NULL
25
+ );
stt.py ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Speech-to-text for TalkToDoc.
3
+ Converts a patient's spoken audio into text using a local Whisper model.
4
+ Language is passed in from the manual language selector, not auto-detected.
5
+ """
6
+
7
+ import whisper
8
+
9
+ _model = None
10
+
11
+ # Maps the app's language selection to Whisper's language codes.
12
+ # Whisper has no dedicated Nigerian Pidgin code, so Pidgin is passed as
13
+ # English, since Pidgin is close enough to English that this gives Whisper
14
+ # the best chance of an accurate transcription.
15
+ LANGUAGE_MAP = {
16
+ "english": "en",
17
+ "yoruba": "yo",
18
+ "hausa": "ha",
19
+ "igbo": "ig",
20
+ "pidgin": "en",
21
+ }
22
+
23
+
24
+ def _get_model():
25
+ global _model
26
+ if _model is None:
27
+ _model = whisper.load_model("base")
28
+ return _model
29
+
30
+
31
+ def transcribe_audio(audio_path, language=None):
32
+ """
33
+ audio_path: path to an audio file (wav, mp3, m4a, etc.)
34
+ language: one of "english", "yoruba", "hausa", "igbo", "pidgin"
35
+ Returns the transcribed text.
36
+ """
37
+ model = _get_model()
38
+ whisper_language = LANGUAGE_MAP.get(language.lower()) if language else None
39
+ result = model.transcribe(audio_path, language=whisper_language)
40
+ return result["text"].strip()
41
+
42
+
43
+ if __name__ == "__main__":
44
+ import sys
45
+
46
+ if len(sys.argv) < 2:
47
+ print("Usage: python stt.py <audio_file> [language]")
48
+ else:
49
+ audio_file = sys.argv[1]
50
+ selected_language = sys.argv[2] if len(sys.argv) > 2 else None
51
+ print(transcribe_audio(audio_file, selected_language))
translation.py ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Translation for TalkToDoc.
3
+ Translates patient input into English for the provider, and translates the
4
+ provider's reply back into the patient's selected language.
5
+
6
+ Uses the Claude API (Anthropic), as specified in the research document's
7
+ software requirements (3.5.4).
8
+
9
+ MOCK_MODE: if set to "true" in .env, this skips the real API call
10
+ entirely and returns the input text unchanged instead. This exists so
11
+ the whole app (Whisper, YarnGPT, MMS-TTS, the database, all the routing
12
+ and session logic) can be run and tested locally with zero cost and no
13
+ API key, before deciding to actually acquire one. It is not a substitute
14
+ for testing real translation quality, only for testing everything else.
15
+ """
16
+
17
+ import os
18
+ from dotenv import load_dotenv
19
+
20
+ load_dotenv()
21
+
22
+ MOCK_MODE = os.environ.get("MOCK_MODE", "").lower() == "true"
23
+
24
+ if not MOCK_MODE:
25
+ from anthropic import Anthropic
26
+
27
+ client = Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"])
28
+ MODEL = "claude-haiku-4-5-20251001"
29
+
30
+ SUPPORTED_LANGUAGES = ["english", "yoruba", "hausa", "igbo", "pidgin"]
31
+
32
+
33
+ def translate(text, source_language, target_language):
34
+ """
35
+ text: the text to translate
36
+ source_language: one of "english", "yoruba", "hausa", "igbo", "pidgin"
37
+ target_language: one of "english", "yoruba", "hausa", "igbo", "pidgin"
38
+ Returns the translated text only.
39
+ """
40
+ if MOCK_MODE:
41
+ if source_language.lower() == target_language.lower():
42
+ return text
43
+ return f"[MOCK, no real translation: {source_language} -> {target_language}] {text}"
44
+
45
+ prompt = (
46
+ f"Translate the following text from {source_language} to {target_language}. "
47
+ f"Reply with only the translated text and nothing else, no explanation.\n\n"
48
+ f"Text: {text}"
49
+ )
50
+
51
+ response = client.messages.create(
52
+ model=MODEL,
53
+ max_tokens=500,
54
+ messages=[{"role": "user", "content": prompt}],
55
+ )
56
+
57
+ return response.content[0].text.strip()
58
+
59
+
60
+ if __name__ == "__main__":
61
+ import sys
62
+
63
+ if len(sys.argv) < 4:
64
+ print('Usage: python translation.py "text" source_language target_language')
65
+ else:
66
+ input_text = sys.argv[1]
67
+ source_lang = sys.argv[2]
68
+ target_lang = sys.argv[3]
69
+ print(translate(input_text, source_lang, target_lang))
tts.py ADDED
@@ -0,0 +1,138 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Text-to-speech for TalkToDoc.
3
+ Converts translated text responses into audible speech in the patient's
4
+ selected language.
5
+
6
+ English, Yoruba, Igbo, and Hausa use YarnGPT, a local model trained
7
+ specifically on Nigerian-accented speech.
8
+
9
+ Nigerian Pidgin uses MMS-TTS (facebook/mms-tts-pcm) instead, since YarnGPT
10
+ has no Pidgin support. MMS-TTS is the only free option found with a
11
+ dedicated Pidgin checkpoint.
12
+
13
+ Note: importing this file triggers a one-time download of YarnGPT's audio
14
+ tokenizer files. This is expected and only happens once.
15
+
16
+ Performance note: YarnGPT's own generate_speech() function reloads its
17
+ full model from disk on every call, which is too slow for a live app.
18
+ This file loads the model once and reuses it, using the same generation
19
+ steps YarnGPT's own function uses internally, just without the reload.
20
+ Call preload_models() once when the app starts, so the first real request
21
+ isn't slow either.
22
+ """
23
+
24
+ import torch
25
+ import torchaudio
26
+ import scipy.io.wavfile
27
+ from transformers import VitsModel, AutoTokenizer
28
+ from yarngpt.core import load_model_and_tokenizer, SPEAKER_MAPPING, AVAILABLE_SPEAKERS
29
+
30
+ # Default speaker used for each YarnGPT-supported language.
31
+ YARNGPT_SPEAKERS = {
32
+ "english": "idera",
33
+ "yoruba": "abayomi",
34
+ "igbo": "chioma",
35
+ "hausa": "amina",
36
+ }
37
+
38
+ _yarngpt_model = None
39
+ _yarngpt_tokenizer = None
40
+
41
+ _pidgin_model = None
42
+ _pidgin_tokenizer = None
43
+
44
+
45
+ def _get_yarngpt_model():
46
+ global _yarngpt_model, _yarngpt_tokenizer
47
+ if _yarngpt_model is None:
48
+ _yarngpt_model, _yarngpt_tokenizer = load_model_and_tokenizer()
49
+ return _yarngpt_model, _yarngpt_tokenizer
50
+
51
+
52
+ def _get_pidgin_model():
53
+ global _pidgin_model, _pidgin_tokenizer
54
+ if _pidgin_model is None:
55
+ _pidgin_tokenizer = AutoTokenizer.from_pretrained("facebook/mms-tts-pcm")
56
+ _pidgin_model = VitsModel.from_pretrained("facebook/mms-tts-pcm")
57
+ return _pidgin_model, _pidgin_tokenizer
58
+
59
+
60
+ def preload_models():
61
+ """Loads both TTS backends into memory ahead of time. Call this once
62
+ when the Flask app starts, so the first real request isn't slow."""
63
+ _get_yarngpt_model()
64
+ _get_pidgin_model()
65
+
66
+
67
+ def _generate_yarngpt_speech(text, speaker, language, temperature=0.1, repetition_penalty=1.1, max_length=4000):
68
+ """
69
+ Same steps as yarngpt's own generate_speech(), but reuses the model
70
+ already loaded by _get_yarngpt_model() instead of reloading it.
71
+ """
72
+ model_speaker = SPEAKER_MAPPING.get(speaker, speaker)
73
+ if model_speaker not in AVAILABLE_SPEAKERS:
74
+ raise ValueError(f"Unknown speaker: {speaker}")
75
+
76
+ model, audio_tokenizer = _get_yarngpt_model()
77
+
78
+ prompt = audio_tokenizer.create_prompt(text, language, model_speaker)
79
+ input_ids = audio_tokenizer.tokenize_prompt(prompt)
80
+ attention_mask = torch.ones_like(input_ids)
81
+
82
+ output = model.generate(
83
+ input_ids=input_ids,
84
+ attention_mask=attention_mask,
85
+ do_sample=True,
86
+ temperature=temperature,
87
+ repetition_penalty=repetition_penalty,
88
+ max_length=max_length,
89
+ pad_token_id=model.config.eos_token_id,
90
+ eos_token_id=model.config.eos_token_id,
91
+ )
92
+
93
+ codes = audio_tokenizer.get_codes(output)
94
+ audio = audio_tokenizer.get_audio(codes)
95
+ return audio
96
+
97
+
98
+ def synthesize_speech(text, language, output_path):
99
+ """
100
+ text: the text to speak
101
+ language: one of "english", "yoruba", "hausa", "igbo", "pidgin"
102
+ output_path: where to save the resulting .wav file
103
+ Returns output_path.
104
+ """
105
+ language = language.lower()
106
+
107
+ if language in YARNGPT_SPEAKERS:
108
+ speaker = YARNGPT_SPEAKERS[language]
109
+ audio = _generate_yarngpt_speech(text, speaker=speaker, language=language)
110
+ torchaudio.save(output_path, audio, sample_rate=24000)
111
+ return output_path
112
+
113
+ if language == "pidgin":
114
+ model, tokenizer = _get_pidgin_model()
115
+ inputs = tokenizer(text, return_tensors="pt")
116
+ with torch.no_grad():
117
+ output = model(**inputs).waveform
118
+ scipy.io.wavfile.write(
119
+ output_path,
120
+ rate=model.config.sampling_rate,
121
+ data=output.numpy().squeeze(),
122
+ )
123
+ return output_path
124
+
125
+ raise ValueError(f"Unsupported language: {language}")
126
+
127
+
128
+ if __name__ == "__main__":
129
+ import sys
130
+
131
+ if len(sys.argv) < 3:
132
+ print('Usage: python tts.py "text" language [output_file]')
133
+ else:
134
+ input_text = sys.argv[1]
135
+ selected_language = sys.argv[2]
136
+ output_file = sys.argv[3] if len(sys.argv) > 3 else "output.wav"
137
+ synthesize_speech(input_text, selected_language, output_file)
138
+ print("Saved to", output_file)