Fred-Rcky commited on
Commit
e1242f6
·
1 Parent(s): c32bf13

refactored for hugging face hosting

Browse files
Files changed (5) hide show
  1. README.md +41 -33
  2. agents/symptom_agent.py +2 -67
  3. app.py +49 -103
  4. requirements.txt +0 -4
  5. transcription/transcriber.py +10 -111
README.md CHANGED
@@ -1,19 +1,30 @@
1
- # Hospital Copilot
 
 
 
 
 
 
 
 
 
 
 
2
 
3
  AI-powered medical documentation assistant for the **Gemma 4 for Good** hackathon.
4
 
5
- Listens to doctor-patient consultations and automatically generates SOAP notes, patient summaries, symptom extractions, and Twi (Akan) translations reducing paperwork and language barriers in Ghanaian healthcare.
6
 
7
  ## Features
8
 
9
- - **Live transcription** via faster-whisper (runs on CPU)
10
- - **Symptom extraction** via Gemma 4 E2B (local, Ollama, CPU)
11
- - **SOAP note generation** via Gemma 4 27B (Google AI Studio)
12
- - **Patient summary** in plain English
13
- - **English Twi translation** for Ghanaian patients
14
- - **Patient records** stored in local SQLite
15
 
16
- ## Setup
17
 
18
  ### 1. Install dependencies
19
 
@@ -21,53 +32,50 @@ Listens to doctor-patient consultations and automatically generates SOAP notes,
21
  pip install -r requirements.txt
22
  ```
23
 
24
- ### 2. Install Ollama and pull Gemma 4
25
-
26
- ```bash
27
- # Install Ollama: https://ollama.com
28
- ollama pull gemma4:e2b
29
- ```
30
-
31
- ### 3. Configure environment
32
 
33
  ```bash
34
  cp .env.example .env
35
- # Edit .env and add your Google AI Studio API key
36
  ```
37
 
38
- Get a free API key at https://aistudio.google.com
39
 
40
- ### 4. Run
41
 
42
  ```bash
43
  python app.py
44
  ```
45
 
46
- Open http://localhost:7860 in your browser.
 
 
47
 
48
  ## Project Structure
49
 
50
  ```
51
- hosptial_copilot/
52
  ├── app.py # Gradio UI + app logic
53
  ├── agents/
54
- │ ├── symptom_agent.py # Local Gemma 4 (Ollama) symptom extractor
55
- │ └── cloud_agents.py # Cloud Gemma 4: SOAP, summary, translation
56
  ├── transcription/
57
- │ └── transcriber.py # faster-whisper live mic transcription
 
 
 
58
  ├── database/
59
  │ └── db.py # SQLite helpers
60
- ── requirements.txt
61
- └── .env.example
62
  ```
63
 
64
  ## Architecture
65
 
66
  ```
67
- Microphone
68
- └─► faster-whisper (local, CPU) → raw transcript
69
- ─► Gemma 4 E2B via Ollama symptom JSON (local CPU)
70
- ─► Gemma 4 27B via API SOAP note + summary + Twi translation
71
- ─► SQLite patient records
72
- └─► Gradio UI doctor dashboard
 
73
  ```
 
1
+ ---
2
+ title: MediScribe AI
3
+ emoji: 🏥
4
+ colorFrom: blue
5
+ colorTo: indigo
6
+ sdk: gradio
7
+ sdk_version: 4.44.0
8
+ app_file: app.py
9
+ pinned: false
10
+ ---
11
+
12
+ # MediScribe AI
13
 
14
  AI-powered medical documentation assistant for the **Gemma 4 for Good** hackathon.
15
 
16
+ Record a doctor-patient consultation via your browser mic. MediScribe transcribes it, repairs ASR errors, extracts structured clinical data, and generates a professional SOAP note and patient summary powered by Gemma 4.
17
 
18
  ## Features
19
 
20
+ - **Browser mic recording** no software install needed
21
+ - **Transcript repair + speaker labelling** via Gemma 4
22
+ - **Structured symptom extraction** via Gemma 4 function calling
23
+ - **RAG-grounded SOAP notes** with ICD-10 codes and WHO drug references
24
+ - **Multimodal document analysis** upload lab results or prescriptions
25
+ - **Patient records** stored in SQLite
26
 
27
+ ## Setup (local)
28
 
29
  ### 1. Install dependencies
30
 
 
32
  pip install -r requirements.txt
33
  ```
34
 
35
+ ### 2. Configure environment
 
 
 
 
 
 
 
36
 
37
  ```bash
38
  cp .env.example .env
39
+ # Add your Google AI Studio API key
40
  ```
41
 
42
+ Get a free key at https://aistudio.google.com
43
 
44
+ ### 3. Run
45
 
46
  ```bash
47
  python app.py
48
  ```
49
 
50
+ ## Hugging Face Spaces
51
+
52
+ Set `GEMINI_API_KEY` as a Space secret in Settings → Variables and secrets.
53
 
54
  ## Project Structure
55
 
56
  ```
 
57
  ├── app.py # Gradio UI + app logic
58
  ├── agents/
59
+ │ ├── symptom_agent.py # Symptom extractor (Gemma 4 function calling)
60
+ │ └── cloud_agents.py # SOAP, summary, transcript repair, document analysis
61
  ├── transcription/
62
+ │ └── transcriber.py # faster-whisper batch transcription
63
+ ├── rag/
64
+ │ ├── retriever.py # ChromaDB + sentence-transformers RAG
65
+ │ └── data/ # ICD-10 codes + WHO essential medicines
66
  ├── database/
67
  │ └── db.py # SQLite helpers
68
+ ── requirements.txt
 
69
  ```
70
 
71
  ## Architecture
72
 
73
  ```
74
+ Browser Mic
75
+ └─► faster-whisper (CPU) → raw transcript
76
+ ─► Gemma 4 26B (API) cleaned transcript + speaker labels
77
+ ─► Gemma 4 function callingstructured symptom JSON
78
+ ─► ChromaDB RAG ICD-10 codes + drug dosages
79
+ └─► Gemma 4 reasoning mode SOAP note + patient summary
80
+ └─► SQLite → patient records
81
  ```
agents/symptom_agent.py CHANGED
@@ -1,81 +1,16 @@
1
- import os
2
- import json
3
- import ollama
4
  from agents.cloud_agents import extract_symptoms_cloud
5
 
6
- OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "gemma4:e2b")
7
-
8
- SYMPTOM_PROMPT = """You are a medical symptom extraction AI. Extract all clinical information from this transcript into valid JSON only.
9
-
10
- Return ONLY valid JSON — no markdown, no explanation, no code fences:
11
- {{
12
- "chief_complaint": "main reason for visit",
13
- "symptoms": ["list", "of", "symptoms"],
14
- "duration": "how long symptoms have been present",
15
- "severity": "mild | moderate | severe",
16
- "associated_symptoms": ["other symptoms"],
17
- "medications_mentioned": ["drugs or treatments mentioned"],
18
- "allergies": ["any allergies mentioned"],
19
- "vitals_mentioned": {{
20
- "temperature": null,
21
- "blood_pressure": null,
22
- "pulse": null,
23
- "weight": null
24
- }},
25
- "relevant_history": "past medical history",
26
- "follow_up_actions": ["follow-up steps, tests, referrals"]
27
- }}
28
-
29
- Transcript:
30
- {transcript}"""
31
-
32
-
33
- def _extract_via_ollama(transcript: str) -> dict:
34
- """Primary: local Gemma 4 E2B via Ollama."""
35
- response = ollama.chat(
36
- model=OLLAMA_MODEL,
37
- messages=[{"role": "user", "content": SYMPTOM_PROMPT.format(transcript=transcript)}],
38
- options={"temperature": 0.1},
39
- )
40
- raw = response["message"]["content"].strip()
41
-
42
- # strip markdown fences if present
43
- if "```" in raw:
44
- parts = raw.split("```")
45
- raw = parts[1] if len(parts) > 1 else parts[0]
46
- if raw.startswith("json"):
47
- raw = raw[4:]
48
- raw = raw.strip()
49
-
50
- result = json.loads(raw)
51
- # must be a dict with at least chief_complaint to be valid
52
- if not isinstance(result, dict) or "chief_complaint" not in result:
53
- raise ValueError("Invalid symptom structure from Ollama")
54
- return result
55
-
56
 
57
  def extract_symptoms(transcript: str) -> dict:
58
- """
59
- Extract structured symptoms from transcript.
60
- Tries local Gemma 4 E2B (Ollama) first — fast, private.
61
- Falls back to cloud Gemma 4 function calling on any failure — guaranteed valid schema.
62
- """
63
  if not transcript.strip():
64
  return {}
65
-
66
- try:
67
- result = _extract_via_ollama(transcript)
68
- print("[Symptoms] Extracted via local Gemma 4 E2B (Ollama)")
69
- return result
70
- except Exception as e:
71
- print(f"[Symptoms] Ollama failed ({e}), falling back to cloud function calling...")
72
-
73
  try:
74
  result = extract_symptoms_cloud(transcript)
75
  print("[Symptoms] Extracted via cloud Gemma 4 function calling")
76
  return result
77
  except Exception as e:
78
- print(f"[Symptoms] Cloud fallback also failed: {e}")
79
  return {"error": str(e)}
80
 
81
 
 
 
 
 
1
  from agents.cloud_agents import extract_symptoms_cloud
2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
 
4
  def extract_symptoms(transcript: str) -> dict:
5
+ """Extract structured symptoms via Gemma 4 function calling."""
 
 
 
 
6
  if not transcript.strip():
7
  return {}
 
 
 
 
 
 
 
 
8
  try:
9
  result = extract_symptoms_cloud(transcript)
10
  print("[Symptoms] Extracted via cloud Gemma 4 function calling")
11
  return result
12
  except Exception as e:
13
+ print(f"[Symptoms] Extraction failed: {e}")
14
  return {"error": str(e)}
15
 
16
 
app.py CHANGED
@@ -1,4 +1,3 @@
1
- import threading
2
  from dotenv import load_dotenv
3
 
4
  load_dotenv()
@@ -19,7 +18,7 @@ from database.db import (
19
  get_note_for_session,
20
  get_symptoms_for_session,
21
  )
22
- from transcription.transcriber import LiveTranscriber
23
  from agents.symptom_agent import extract_symptoms, format_symptoms_for_display
24
  from agents.cloud_agents import generate_soap_note, generate_patient_summary, clean_and_label_transcript, analyze_medical_document
25
  from rag.retriever import (
@@ -37,12 +36,9 @@ ensure_kb()
37
 
38
  # ── State ─────────────────────────────────────────────────────────────────────
39
 
40
- _transcriber: LiveTranscriber | None = None
41
- _transcript_parts: list[str] = []
42
  _labelled_transcript: str = ""
43
  _document_analysis: str = ""
44
  _current_session_id: int | None = None
45
- _transcript_lock = threading.Lock()
46
 
47
  # ── Helpers ───────────────────────────────────────────────────────────────────
48
 
@@ -55,11 +51,6 @@ def _parse_patient_choice(choice: str) -> int:
55
  return int(choice.split("—")[0].strip())
56
 
57
 
58
- def _full_transcript() -> str:
59
- with _transcript_lock:
60
- return " ".join(_transcript_parts)
61
-
62
-
63
  def _format_icd_panel(codes: list[dict]) -> str:
64
  if not codes:
65
  return "_No ICD-10 suggestions._"
@@ -94,61 +85,27 @@ def register_patient(name, dob, gender, phone):
94
  return gr.update(choices=choices, value=new_val), f"Patient '{name}' registered (ID {pid})."
95
 
96
 
97
- def start_consultation(patient_choice, doctor_name):
98
- global _transcriber, _transcript_parts, _current_session_id
 
99
 
100
  if not patient_choice:
101
- return "No patient selected.", "", gr.update(interactive=False), gr.update(interactive=True)
 
 
102
 
103
  pid = _parse_patient_choice(patient_choice)
104
  _current_session_id = create_session(pid, doctor_name or "Doctor")
105
-
106
- with _transcript_lock:
107
- _transcript_parts.clear()
108
- global _labelled_transcript, _document_analysis
109
  _labelled_transcript = ""
110
- _document_analysis = ""
111
-
112
- def on_text(text):
113
- with _transcript_lock:
114
- _transcript_parts.append(text)
115
- _transcriber = LiveTranscriber(on_text=on_text)
116
- _transcriber.start()
117
-
118
- return (
119
- "Recording... speak clearly.",
120
- "",
121
- gr.update(interactive=True),
122
- gr.update(interactive=False),
123
- )
124
-
125
-
126
- def poll_transcript():
127
- return _full_transcript()
128
-
129
-
130
- def stop_consultation():
131
- global _transcriber, _labelled_transcript
132
 
133
- if _transcriber:
134
- _transcriber.stop()
135
- _transcriber = None
136
-
137
- raw = _full_transcript()
138
  if not raw:
139
- return "No audio captured.", "", gr.update(interactive=False), gr.update(interactive=True)
140
-
141
- if _current_session_id:
142
- update_transcript(_current_session_id, raw)
143
 
 
144
  _labelled_transcript = clean_and_label_transcript(raw)
145
 
146
- return (
147
- "Consultation ended. Transcript cleaned ✓ Click 'Generate Notes' to proceed.",
148
- _labelled_transcript,
149
- gr.update(interactive=False),
150
- gr.update(interactive=True),
151
- )
152
 
153
 
154
  def upload_document(file):
@@ -168,20 +125,18 @@ def upload_document(file):
168
 
169
  def generate_notes():
170
  """RAG retrieval → cloud agents → save to DB."""
171
- # Use labelled transcript if available, fall back to raw
172
- transcript = _labelled_transcript or _full_transcript()
173
- if not transcript:
174
  return "No transcript available.", "No transcript available.", "_No symptoms._", "_No ICD codes._", "_No drug info._", "", ""
175
 
176
- # 1. Extract symptoms locally (Gemma 4 E2B via Ollama)
177
- symptoms = extract_symptoms(transcript)
178
  symptoms_md = format_symptoms_for_display(symptoms)
179
 
180
  # 2. RAG retrieval
181
  chief = symptoms.get("chief_complaint", "")
182
  sym_list = symptoms.get("symptoms", [])
183
  meds_list = symptoms.get("medications_mentioned", [])
184
- rag_query = f"{chief} {' '.join(sym_list)}".strip() or transcript[:300]
185
 
186
  icd_codes = retrieve_icd_codes(rag_query, n=5)
187
  drug_info = retrieve_drug_info(meds_list, n=3) if meds_list else []
@@ -195,12 +150,12 @@ def generate_notes():
195
 
196
  # 3. Cloud agents
197
  try:
198
- soap = generate_soap_note(transcript, rag_context=rag_context)
199
  except Exception as e:
200
  soap = f"_SOAP note generation failed: {e}_"
201
 
202
  try:
203
- summary_en = generate_patient_summary(transcript)
204
  except Exception as e:
205
  summary_en = f"_Summary generation failed: {e}_"
206
 
@@ -261,12 +216,10 @@ body, .gradio-container { font-family: 'Segoe UI', system-ui, sans-serif; }
261
  #header-banner h1 { margin: 0; font-size: 1.9rem; font-weight: 700; letter-spacing: -0.5px; }
262
  #header-banner p { margin: 5px 0 0; opacity: 0.85; font-size: 0.95rem; }
263
 
264
- /* Fix markdown panels — transparent so they inherit theme bg */
265
  .gr-markdown, .svelte-1ed2p3z, [data-testid="markdown"] {
266
  background: transparent !important;
267
  }
268
 
269
- /* Note cards — rendered SOAP/summary display */
270
  .note-card {
271
  border: 1px solid #2d4a6e;
272
  border-radius: 8px;
@@ -284,10 +237,8 @@ body, .gradio-container { font-family: 'Segoe UI', system-ui, sans-serif; }
284
  .note-card p { margin: 6px 0; }
285
  .note-card ul, .note-card ol { padding-left: 20px; margin: 4px 0; }
286
 
287
- /* Status bar */
288
  .status-bar p { font-weight: 600; color: #4a9eff; font-size: 1rem; }
289
 
290
- /* RAG accordion open panel */
291
  .rag-content {
292
  border-left: 3px solid #1a6eb5;
293
  padding: 10px 14px;
@@ -295,21 +246,16 @@ body, .gradio-container { font-family: 'Segoe UI', system-ui, sans-serif; }
295
  font-size: 0.9rem;
296
  }
297
 
298
- /* Tighten up accordion headers */
299
  .gr-accordion .label-wrap { font-weight: 600 !important; }
300
-
301
- /* Recording pulse indicator */
302
- @keyframes pulse { 0%,100%{opacity:1} 50%{opacity:0.4} }
303
- .recording p { animation: pulse 1.4s ease-in-out infinite; color: #ff4444 !important; font-weight: 700; }
304
  """
305
 
306
  # ── Layout ────────────────────────────────────────────────────────────────────
307
 
308
- with gr.Blocks(title="Hospital Copilot") as demo:
309
 
310
  gr.HTML("""
311
  <div id="header-banner">
312
- <h1>🏥 Hospital Copilot</h1>
313
  <p>AI-powered medical documentation &nbsp;·&nbsp; Gemma 4 &nbsp;·&nbsp; RAG-grounded &nbsp;·&nbsp; Ghana</p>
314
  </div>
315
  """)
@@ -343,20 +289,24 @@ with gr.Blocks(title="Hospital Copilot") as demo:
343
 
344
  # Right column — consultation
345
  with gr.Column(scale=3):
346
- status_txt = gr.Markdown("_Ready. Select a patient and click Start._", elem_classes=["status-bar"])
347
 
348
- with gr.Row():
349
- start_btn = gr.Button("▶ Start Consultation", variant="primary", scale=1)
350
- stop_btn = gr.Button(" End Consultation", variant="stop", scale=1, interactive=False)
 
 
 
 
 
 
351
 
352
  live_transcript = gr.Textbox(
353
- label="Transcript (cleaned & speaker-labelled after consultation ends)",
354
  lines=8, max_lines=16,
355
  interactive=False,
356
- placeholder="Transcript streams here as you speak. After you click End Consultation, Gemma 4 cleans and labels it automatically.",
357
  )
358
- timer = gr.Timer(value=2)
359
- timer.tick(poll_transcript, outputs=live_transcript)
360
 
361
  with gr.Accordion("🩺 Extracted Symptoms", open=False):
362
  symptoms_live = gr.Markdown("_Will populate after Generate Notes._")
@@ -380,7 +330,6 @@ with gr.Blocks(title="Hospital Copilot") as demo:
380
 
381
  generate_btn = gr.Button("⚡ Generate Notes from Transcript", variant="primary", size="lg")
382
 
383
- # RAG panels — inside accordions so they don't show as white boxes
384
  with gr.Row():
385
  with gr.Accordion("🏷️ ICD-10 Suggestions", open=True):
386
  icd_panel = gr.Markdown("_Click Generate Notes to see suggestions._")
@@ -407,14 +356,10 @@ with gr.Blocks(title="Hospital Copilot") as demo:
407
  with gr.Accordion("✏️ Edit Summary", open=False):
408
  summary_edit = gr.Textbox(lines=10, interactive=True, show_label=False)
409
 
410
- start_btn.click(
411
- start_consultation,
412
- inputs=[patient_dd, doctor_name],
413
- outputs=[status_txt, live_transcript, stop_btn, start_btn],
414
- )
415
- stop_btn.click(
416
- stop_consultation,
417
- outputs=[status_txt, live_transcript, stop_btn, start_btn],
418
  )
419
  generate_btn.click(
420
  generate_notes,
@@ -455,17 +400,18 @@ with gr.Blocks(title="Hospital Copilot") as demo:
455
  # ── Tab 3: About ──────────────────────────────────────────────────
456
  with gr.Tab("ℹ️ About"):
457
  gr.Markdown("""
458
- ## Hospital Copilot — Gemma 4 for Good
459
 
460
  **Reducing doctor burnout. Improving care quality. Built for Ghana.**
461
 
462
  ### How it works
463
- 1. **Live Transcription** — faster-whisper converts speech to text in real time on CPU
464
- 2. **Symptom Extraction** — Gemma 4 E2B (local, Ollama) extracts structured clinical JSON
465
- 3. **RAG Retrieval** — sentence-transformers + ChromaDB matches ICD-10 codes and drug dosages
466
- 4. **SOAP Note Generation** — Gemma 4 26B (cloud) writes a grounded, accurate medical note
467
- 5. **Patient Summary** — plain-language summary the patient can take home
468
- 6. **Structured Records** — everything saved to local SQLite
 
469
 
470
  ### RAG Knowledge Base
471
  | Collection | Entries | Source |
@@ -476,15 +422,15 @@ with gr.Blocks(title="Hospital Copilot") as demo:
476
  ### Technology Stack
477
  | Component | Model | Where |
478
  |---|---|---|
479
- | Speech-to-Text | faster-whisper (base) | Local CPU |
480
- | Symptom Extraction | Gemma 4 E2B (Q4_K_M) | Local CPU via Ollama |
481
- | Embeddings | all-MiniLM-L6-v2 | Local CPU |
482
  | Vector Store | ChromaDB | Local disk |
483
- | SOAP / Summary | Gemma 4 26B-IT | Google AI Studio API |
484
  | Storage | SQLite | Local |
485
- | UI | Gradio | Desktop |
486
  """)
487
 
488
 
489
  if __name__ == "__main__":
490
- demo.launch(server_name="0.0.0.0", server_port=7860, share=False, css=CSS)
 
 
1
  from dotenv import load_dotenv
2
 
3
  load_dotenv()
 
18
  get_note_for_session,
19
  get_symptoms_for_session,
20
  )
21
+ from transcription.transcriber import transcribe_file
22
  from agents.symptom_agent import extract_symptoms, format_symptoms_for_display
23
  from agents.cloud_agents import generate_soap_note, generate_patient_summary, clean_and_label_transcript, analyze_medical_document
24
  from rag.retriever import (
 
36
 
37
  # ── State ─────────────────────────────────────────────────────────────────────
38
 
 
 
39
  _labelled_transcript: str = ""
40
  _document_analysis: str = ""
41
  _current_session_id: int | None = None
 
42
 
43
  # ── Helpers ───────────────────────────────────────────────────────────────────
44
 
 
51
  return int(choice.split("—")[0].strip())
52
 
53
 
 
 
 
 
 
54
  def _format_icd_panel(codes: list[dict]) -> str:
55
  if not codes:
56
  return "_No ICD-10 suggestions._"
 
85
  return gr.update(choices=choices, value=new_val), f"Patient '{name}' registered (ID {pid})."
86
 
87
 
88
+ def transcribe_recording(patient_choice, doctor_name, audio_path):
89
+ """Called when the mic recording stops. Transcribes and cleans the audio."""
90
+ global _current_session_id, _labelled_transcript
91
 
92
  if not patient_choice:
93
+ return "No patient selected.", ""
94
+ if audio_path is None:
95
+ return "No audio recorded.", ""
96
 
97
  pid = _parse_patient_choice(patient_choice)
98
  _current_session_id = create_session(pid, doctor_name or "Doctor")
 
 
 
 
99
  _labelled_transcript = ""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
100
 
101
+ raw = transcribe_file(audio_path)
 
 
 
 
102
  if not raw:
103
+ return "No speech detected in the recording.", ""
 
 
 
104
 
105
+ update_transcript(_current_session_id, raw)
106
  _labelled_transcript = clean_and_label_transcript(raw)
107
 
108
+ return "Transcript ready ✓ Click 'Generate Notes' to proceed.", _labelled_transcript
 
 
 
 
 
109
 
110
 
111
  def upload_document(file):
 
125
 
126
  def generate_notes():
127
  """RAG retrieval → cloud agents → save to DB."""
128
+ if not _labelled_transcript:
 
 
129
  return "No transcript available.", "No transcript available.", "_No symptoms._", "_No ICD codes._", "_No drug info._", "", ""
130
 
131
+ # 1. Extract symptoms
132
+ symptoms = extract_symptoms(_labelled_transcript)
133
  symptoms_md = format_symptoms_for_display(symptoms)
134
 
135
  # 2. RAG retrieval
136
  chief = symptoms.get("chief_complaint", "")
137
  sym_list = symptoms.get("symptoms", [])
138
  meds_list = symptoms.get("medications_mentioned", [])
139
+ rag_query = f"{chief} {' '.join(sym_list)}".strip() or _labelled_transcript[:300]
140
 
141
  icd_codes = retrieve_icd_codes(rag_query, n=5)
142
  drug_info = retrieve_drug_info(meds_list, n=3) if meds_list else []
 
150
 
151
  # 3. Cloud agents
152
  try:
153
+ soap = generate_soap_note(_labelled_transcript, rag_context=rag_context)
154
  except Exception as e:
155
  soap = f"_SOAP note generation failed: {e}_"
156
 
157
  try:
158
+ summary_en = generate_patient_summary(_labelled_transcript)
159
  except Exception as e:
160
  summary_en = f"_Summary generation failed: {e}_"
161
 
 
216
  #header-banner h1 { margin: 0; font-size: 1.9rem; font-weight: 700; letter-spacing: -0.5px; }
217
  #header-banner p { margin: 5px 0 0; opacity: 0.85; font-size: 0.95rem; }
218
 
 
219
  .gr-markdown, .svelte-1ed2p3z, [data-testid="markdown"] {
220
  background: transparent !important;
221
  }
222
 
 
223
  .note-card {
224
  border: 1px solid #2d4a6e;
225
  border-radius: 8px;
 
237
  .note-card p { margin: 6px 0; }
238
  .note-card ul, .note-card ol { padding-left: 20px; margin: 4px 0; }
239
 
 
240
  .status-bar p { font-weight: 600; color: #4a9eff; font-size: 1rem; }
241
 
 
242
  .rag-content {
243
  border-left: 3px solid #1a6eb5;
244
  padding: 10px 14px;
 
246
  font-size: 0.9rem;
247
  }
248
 
 
249
  .gr-accordion .label-wrap { font-weight: 600 !important; }
 
 
 
 
250
  """
251
 
252
  # ── Layout ────────────────────────────────────────────────────────────────────
253
 
254
+ with gr.Blocks(title="Hospital Copilot", css=CSS) as demo:
255
 
256
  gr.HTML("""
257
  <div id="header-banner">
258
+ <h1>🏥 MediScribe AI</h1>
259
  <p>AI-powered medical documentation &nbsp;·&nbsp; Gemma 4 &nbsp;·&nbsp; RAG-grounded &nbsp;·&nbsp; Ghana</p>
260
  </div>
261
  """)
 
289
 
290
  # Right column — consultation
291
  with gr.Column(scale=3):
292
+ status_txt = gr.Markdown("_Ready. Select a patient, then record the consultation below._", elem_classes=["status-bar"])
293
 
294
+ gr.Markdown(
295
+ "**Record the consultation** click the mic to start, click stop when done. "
296
+ "The transcript will appear automatically."
297
+ )
298
+ mic_input = gr.Audio(
299
+ sources=["microphone"],
300
+ type="filepath",
301
+ label="Consultation Recording",
302
+ )
303
 
304
  live_transcript = gr.Textbox(
305
+ label="Transcript (cleaned & speaker-labelled)",
306
  lines=8, max_lines=16,
307
  interactive=False,
308
+ placeholder="Record the consultation above. The transcript will appear here after you stop recording.",
309
  )
 
 
310
 
311
  with gr.Accordion("🩺 Extracted Symptoms", open=False):
312
  symptoms_live = gr.Markdown("_Will populate after Generate Notes._")
 
330
 
331
  generate_btn = gr.Button("⚡ Generate Notes from Transcript", variant="primary", size="lg")
332
 
 
333
  with gr.Row():
334
  with gr.Accordion("🏷️ ICD-10 Suggestions", open=True):
335
  icd_panel = gr.Markdown("_Click Generate Notes to see suggestions._")
 
356
  with gr.Accordion("✏️ Edit Summary", open=False):
357
  summary_edit = gr.Textbox(lines=10, interactive=True, show_label=False)
358
 
359
+ mic_input.stop_recording(
360
+ transcribe_recording,
361
+ inputs=[patient_dd, doctor_name, mic_input],
362
+ outputs=[status_txt, live_transcript],
 
 
 
 
363
  )
364
  generate_btn.click(
365
  generate_notes,
 
400
  # ── Tab 3: About ──────────────────────────────────────────────────
401
  with gr.Tab("ℹ️ About"):
402
  gr.Markdown("""
403
+ ## MediScribe AI — Gemma 4 for Good
404
 
405
  **Reducing doctor burnout. Improving care quality. Built for Ghana.**
406
 
407
  ### How it works
408
+ 1. **Live Transcription** — Record the consultation via your browser mic. faster-whisper transcribes on CPU.
409
+ 2. **Transcript Repair** — Gemma 4 fixes ASR errors and labels each turn as Doctor or Patient.
410
+ 3. **Symptom Extraction** — Gemma 4 function calling extracts structured clinical JSON.
411
+ 4. **RAG Retrieval** — sentence-transformers + ChromaDB matches ICD-10 codes and drug dosages.
412
+ 5. **SOAP Note Generation** — Gemma 4 (reasoning mode) writes a grounded, accurate medical note.
413
+ 6. **Patient Summary** — plain-language summary the patient can take home.
414
+ 7. **Structured Records** — everything saved to local SQLite.
415
 
416
  ### RAG Knowledge Base
417
  | Collection | Entries | Source |
 
422
  ### Technology Stack
423
  | Component | Model | Where |
424
  |---|---|---|
425
+ | Speech-to-Text | faster-whisper (small) | CPU |
426
+ | Symptom Extraction | Gemma 4 26B function calling | Google AI Studio API |
427
+ | Embeddings | all-MiniLM-L6-v2 | CPU |
428
  | Vector Store | ChromaDB | Local disk |
429
+ | SOAP / Summary | Gemma 4 26B-IT — reasoning mode | Google AI Studio API |
430
  | Storage | SQLite | Local |
431
+ | UI | Gradio | Browser |
432
  """)
433
 
434
 
435
  if __name__ == "__main__":
436
+ demo.launch()
requirements.txt CHANGED
@@ -1,11 +1,7 @@
1
  gradio>=4.44.0
2
  faster-whisper>=1.0.0
3
- ollama>=0.3.0
4
  google-genai>=1.0.0
5
- sounddevice>=0.4.6
6
  numpy>=1.26.0
7
- scipy>=1.13.0
8
  python-dotenv>=1.0.0
9
- fpdf2>=2.7.9
10
  chromadb>=0.5.0
11
  sentence-transformers>=3.0.0
 
1
  gradio>=4.44.0
2
  faster-whisper>=1.0.0
 
3
  google-genai>=1.0.0
 
4
  numpy>=1.26.0
 
5
  python-dotenv>=1.0.0
 
6
  chromadb>=0.5.0
7
  sentence-transformers>=3.0.0
transcription/transcriber.py CHANGED
@@ -1,16 +1,6 @@
1
  import os
2
- import queue
3
- import tempfile
4
- import threading
5
- import wave
6
- import numpy as np
7
- import sounddevice as sd
8
  from faster_whisper import WhisperModel
9
 
10
- SAMPLE_RATE = 16000
11
- BLOCK_SECONDS = 3
12
- CHANNELS = 1
13
-
14
  _model: WhisperModel | None = None
15
 
16
 
@@ -22,104 +12,13 @@ def _load_model() -> WhisperModel:
22
  return _model
23
 
24
 
25
- class LiveTranscriber:
26
- """
27
- Streams microphone audio, transcribes in real time, and saves the full
28
- session to a WAV file for post-hoc speaker diarization.
29
- """
30
-
31
- def __init__(self, on_text):
32
- self.on_text = on_text
33
- self._audio_q: queue.Queue = queue.Queue()
34
- self._stop_event = threading.Event()
35
- self._thread: threading.Thread | None = None
36
- self._stream: sd.InputStream | None = None
37
-
38
- # accumulate all raw audio for diarization
39
- self._all_audio: list[np.ndarray] = []
40
- self._audio_lock = threading.Lock()
41
-
42
- # path to saved WAV after stop()
43
- self.wav_path: str | None = None
44
-
45
- def _audio_callback(self, indata, frames, time_info, status):
46
- chunk = indata.copy()
47
- self._audio_q.put(chunk)
48
- with self._audio_lock:
49
- self._all_audio.append(chunk.flatten())
50
-
51
- def _process_loop(self):
52
- model = _load_model()
53
- buffer = np.empty((0,), dtype=np.float32)
54
- chunk_size = SAMPLE_RATE * BLOCK_SECONDS
55
-
56
- while not self._stop_event.is_set():
57
- try:
58
- chunk = self._audio_q.get(timeout=0.5)
59
- buffer = np.concatenate([buffer, chunk.flatten()])
60
- except queue.Empty:
61
- continue
62
-
63
- if len(buffer) >= chunk_size:
64
- audio_chunk = buffer[:chunk_size].astype(np.float32)
65
- buffer = buffer[chunk_size:]
66
- segments, _ = model.transcribe(
67
- audio_chunk,
68
- language="en",
69
- vad_filter=True,
70
- vad_parameters={"min_silence_duration_ms": 300},
71
- )
72
- text = " ".join(s.text for s in segments).strip()
73
- if text:
74
- self.on_text(text)
75
-
76
- # flush remaining audio
77
- if len(buffer) > SAMPLE_RATE:
78
- segments, _ = _load_model().transcribe(
79
- buffer.astype(np.float32), language="en", vad_filter=True
80
- )
81
- text = " ".join(s.text for s in segments).strip()
82
- if text:
83
- self.on_text(text)
84
-
85
- def start(self):
86
- self._stop_event.clear()
87
- self._all_audio.clear()
88
- self._stream = sd.InputStream(
89
- samplerate=SAMPLE_RATE,
90
- channels=CHANNELS,
91
- dtype="float32",
92
- blocksize=SAMPLE_RATE,
93
- callback=self._audio_callback,
94
- )
95
- self._stream.start()
96
- self._thread = threading.Thread(target=self._process_loop, daemon=True)
97
- self._thread.start()
98
-
99
- def stop(self) -> str | None:
100
- """Stop recording and save full audio to a WAV file. Returns the WAV path."""
101
- self._stop_event.set()
102
- if self._stream:
103
- self._stream.stop()
104
- self._stream.close()
105
- if self._thread:
106
- self._thread.join(timeout=5)
107
-
108
- with self._audio_lock:
109
- all_audio = list(self._all_audio)
110
-
111
- if not all_audio:
112
- return None
113
-
114
- full_audio = np.concatenate(all_audio).astype(np.float32)
115
- pcm = (full_audio * 32767).astype(np.int16)
116
-
117
- tmp = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
118
- with wave.open(tmp.name, "wb") as wf:
119
- wf.setnchannels(CHANNELS)
120
- wf.setsampwidth(2)
121
- wf.setframerate(SAMPLE_RATE)
122
- wf.writeframes(pcm.tobytes())
123
-
124
- self.wav_path = tmp.name
125
- return tmp.name
 
1
  import os
 
 
 
 
 
 
2
  from faster_whisper import WhisperModel
3
 
 
 
 
 
4
  _model: WhisperModel | None = None
5
 
6
 
 
12
  return _model
13
 
14
 
15
+ def transcribe_file(file_path: str) -> str:
16
+ """Transcribe an audio file and return the full text."""
17
+ model = _load_model()
18
+ segments, _ = model.transcribe(
19
+ file_path,
20
+ language="en",
21
+ vad_filter=True,
22
+ vad_parameters={"min_silence_duration_ms": 300},
23
+ )
24
+ return " ".join(s.text for s in segments).strip()