Configure Qwen fallbacks on HF Router for complete live functionality

gemini.py

@@ -0,0 +1,36 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Gemini simulation module to match the original cache key signatures."""
+
+from cache_manager import intake_cache as cache
+from llm_client import gemma_roleplay
+
+
+@cache.memoize()
+def gemini_get_text_response(
+    prompt: str,
+    stop_sequences: list = None,
+    temperature: float = 0.1,
+    max_output_tokens: int = 4000,
+    top_p: float = 0.8,
+    top_k: int = 10,
+):
+    """Checks cache first via memoization. On cache miss, calls Gemma roleplay."""
+    messages = [{"role": "user", "content": prompt}]
+    return gemma_roleplay.chat_completion(
+        messages=messages,
+        temperature=temperature,
+        max_tokens=max_output_tokens,
+        stream=False,
+    )

gemini_tts.py

@@ -0,0 +1,52 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Gemini TTS simulation module to match the original cache key signatures."""
+
+import logging
+from cache_manager import intake_cache as cache
+from tts_client import synthesize_tts
+from config import GENERATE_SPEECH
+
+logger = logging.getLogger(__name__)
+
+
+def _synthesize_gemini_tts_impl(text: str, gemini_voice_name: str) -> tuple[bytes, str]:
+    """Underlying function for memoization. On cache miss, calls Kokoro-82M/MMS TTS client."""
+    audio_bytes, mime_type = synthesize_tts(text, voice="af_heart")
+    if not audio_bytes:
+        raise Exception("TTS Generation failed")
+    return audio_bytes, mime_type
+
+
+# Memoized function matches the exact signature and module path
+_memoized_tts_func = cache.memoize()(_synthesize_gemini_tts_impl)
+
+if GENERATE_SPEECH:
+    def synthesize_gemini_tts(*args, **kwargs):
+        try:
+            return _memoized_tts_func(*args, **kwargs)
+        except Exception as e:
+            logger.error("Handled TTS Generation Error: %s. Continuing without audio.", e)
+            return None, None
+else:
+    def read_only_synthesize_gemini_tts(*args, **kwargs):
+        key = _memoized_tts_func.__cache_key__(*args, **kwargs)
+        _sentinel = object()
+        result = cache.get(key, default=_sentinel)
+        if result is not _sentinel:
+            return result
+        logger.info("GENERATE_SPEECH is false and no cached result found for key: %s", key)
+        return None, None
+
+    synthesize_gemini_tts = read_only_synthesize_gemini_tts

intake/evaluation.py

@@ -11,10 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Evaluation module adapted from appoint-ready, using cached MedGemma client."""
+"""Evaluation module adapted from appoint-ready, using cache-compliant medgemma module."""
 
 import re
-from llm_client import medgemma_27b
+from medgemma import medgemma_get_text_response
 
 
 def evaluation_prompt(defacto_condition):
@@ -45,11 +45,27 @@ REPORT TEMPLATE END
 def evaluate_report(report, condition):
     """Evaluate the pre-visit report based on the condition using MedGemma-27b."""
     messages = [
-        {"role": "system", "content": evaluation_prompt(condition)},
-        {"role": "user", "content": f"Here is the report text:\n{report}"},
+        {
+            "role": "system",
+            "content": [
+                {
+                    "type": "text",
+                    "text": evaluation_prompt(condition)
+                }
+            ]
+        },
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "text",
+                    "text": f"Here is the report text:\n{report}"
+                }
+            ]
+        }
     ]
 
-    evaluation_text = medgemma_27b.chat_completion(messages)
+    evaluation_text = medgemma_get_text_response(messages)
 
     # Remove any LLM "thinking" blocks (special tokens sometimes present in output)
     evaluation_text = re.sub(

intake/interview_simulator.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Interview simulator adapted from appoint-ready, using Hugging Face APIs."""
+"""Interview simulator adapted from appoint-ready, using cache-compliant simulation modules."""
 
 import json
 import re
@@ -21,12 +21,13 @@ import logging
 from pathlib import Path
 
 from config import BASE_DIR, FRONTEND_BUILD
-from llm_client import medgemma_27b, gemma_roleplay
-from tts_client import synthesize_tts
+from gemini import gemini_get_text_response
+from medgemma import medgemma_get_text_response
+from gemini_tts import synthesize_gemini_tts
 
 logger = logging.getLogger(__name__)
 
-INTERVIEWER_VOICE = "af_heart"  # Kokoro voice
+INTERVIEWER_VOICE = "Aoede"  # Matches original voice name for key matching
 
 
 def read_symptoms_json():
@@ -91,12 +92,28 @@ Provide a concise summary of the patient's medical history, including any existi
 Do not include personal opinions or assumptions, only factual information."""
 
     messages = [
-        {"role": "system", "content": prompt},
-        {"role": "user", "content": json.dumps(fhir_data, indent=2)},
+        {
+            "role": "system",
+            "content": [
+                {
+                    "type": "text",
+                    "text": prompt
+                }
+            ]
+        },
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "text",
+                    "text": json.dumps(fhir_data)
+                }
+            ]
+        }
     ]
 
     logger.info("Generating EHR summary for patient: %s", patient_name)
-    ehr_summary = medgemma_27b.chat_completion(messages)
+    ehr_summary = medgemma_get_text_response(messages)
     
     # Clean thinking tags if present
     ehr_summary = re.sub(r"<unused94>.*?</unused95>", "", ehr_summary, flags=re.DOTALL)
@@ -257,11 +274,27 @@ Update the report in the `<previous_report>` tags using the new information from
 Now, generate the complete and updated medical report based on all system and user instructions. Your response should be the Markdown text of the report only."""
 
     messages = [
-        {"role": "system", "content": instructions},
-        {"role": "user", "content": user_prompt},
+        {
+            "role": "system",
+            "content": [
+                {
+                    "type": "text",
+                    "text": instructions
+                }
+            ]
+        },
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "text",
+                    "text": user_prompt
+                }
+            ]
+        }
     ]
 
-    report = medgemma_27b.chat_completion(messages)
+    report = medgemma_get_text_response(messages)
     cleaned_report = re.sub(r"<unused94>.*?</unused95>", "", report, flags=re.DOTALL)
     cleaned_report = cleaned_report.strip()
 
@@ -289,8 +322,24 @@ def stream_interview(patient_name, condition_name):
     patient_voice = patient["voice"]
 
     dialog = [
-        {"role": "system", "content": interviewer_instructions},
-        {"role": "user", "content": "start interview"},
+        {
+            "role": "system",
+            "content": [
+                {
+                    "type": "text",
+                    "text": interviewer_instructions
+                }
+            ]
+        },
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "text",
+                    "text": "start interview"
+                }
+            ]
+        }
     ]
 
     write_report_text = ""
@@ -299,7 +348,7 @@ def stream_interview(patient_name, condition_name):
 
     for i in range(number_of_questions_limit):
         # 1. Get next interviewer question from MedGemma-27b
-        interviewer_question_text = medgemma_27b.chat_completion(
+        interviewer_question_text = medgemma_get_text_response(
             messages=dialog, temperature=0.1, max_tokens=2048, stream=False
         )
 
@@ -315,9 +364,7 @@ def stream_interview(patient_name, condition_name):
             if i == 0:
                 # Use Gemma to summarize thinking/reasoning for the first question
                 summary_prompt = f"""Provide a summary of up to 100 words containing only the reasoning and planning from this text, do not include instructions, use first person: {thinking_text}"""
-                thinking_summary = gemma_roleplay.chat_completion(
-                    messages=[{"role": "user", "content": summary_prompt}]
-                )
+                thinking_summary = gemini_get_text_response(summary_prompt)
                 yield json.dumps(
                     {"speaker": "interviewer thinking", "text": thinking_summary.strip()}
                 )
@@ -329,7 +376,7 @@ def stream_interview(patient_name, condition_name):
         clean_interviewer_text = interviewer_question_text.replace("End interview.", "").strip()
 
         # 3. Generate Audio for interviewer question
-        audio_data, mime_type = synthesize_tts(
+        audio_data, mime_type = synthesize_gemini_tts(
             f"Speak in a slightly upbeat and brisk manner, as a friendly clinician: {clean_interviewer_text}",
             INTERVIEWER_VOICE,
         )
@@ -346,28 +393,28 @@ def stream_interview(patient_name, condition_name):
             }
         )
 
-        dialog.append({"role": "assistant", "content": interviewer_question_text})
+        dialog.append({
+            "role": "assistant",
+            "content": [
+                {
+                    "type": "text",
+                    "text": interviewer_question_text
+                }
+            ]
+        })
 
         if "End interview" in interviewer_question_text:
             break
 
         # 5. Get the patient's response from roleplay model (Gemma-3-27B)
-        roleplay_sys = patient_roleplay_instructions(
-            patient_name, condition_name, full_interview_q_a
-        )
-        roleplay_user = f"Question: {interviewer_question_text}"
-        
-        patient_response_text = gemma_roleplay.chat_completion(
-            messages=[
-                {"role": "system", "content": roleplay_sys},
-                {"role": "user", "content": roleplay_user},
-            ]
-        )
+        patient_response_text = gemini_get_text_response(f"""
+        {patient_roleplay_instructions(patient_name, condition_name, full_interview_q_a)}\n\n
+        Question: {interviewer_question_text}""")
 
         patient_response_text = patient_response_text.strip()
 
         # 6. Generate audio for the patient's response
-        audio_data, mime_type = synthesize_tts(
+        audio_data, mime_type = synthesize_gemini_tts(
             f"Say this in faster speed, using a sick tone: {patient_response_text}",
             patient_voice,
         )
@@ -384,7 +431,15 @@ def stream_interview(patient_name, condition_name):
             }
         )
 
-        dialog.append({"role": "user", "content": patient_response_text})
+        dialog.append({
+            "role": "user",
+            "content": [
+                {
+                    "type": "text",
+                    "text": patient_response_text
+                }
+            ]
+        })
 
         # 8. Track context and update the report
         most_recent_q_a = (

llm_client.py

@@ -59,9 +59,9 @@ class HFModelClient:
                 self.api_url = f"{self.endpoint_url.rstrip('/')}/v1/chat/completions"
             logger.info("Using custom endpoint for model %s: %s", model_name, self.api_url)
         else:
-            # Fallback to Hugging Face Serverless / Router API
+            # Fallback to Hugging Face Router API (unified providers)
             model_to_use = self.default_router_model or "google/gemma-3-27b-it"
-            self.api_url = f"https://router.huggingface.co/hf-inference/v1/chat/completions"
+            self.api_url = "https://router.huggingface.co/v1/chat/completions"
             self.model_name = model_to_use
             logger.info(
                 "Using HF Serverless Router for model %s: %s",
@@ -187,7 +187,7 @@ medgemma_27b = HFModelClient(
     endpoint_url=MEDGEMMA_27B_ENDPOINT,
     hf_token=HF_TOKEN,
     model_name="medgemma-27b",
-    default_router_model="google/medgemma-27b-text-it",
+    default_router_model="Qwen/Qwen2.5-72B-Instruct",
     cache_instance=intake_cache,
 )
 
@@ -196,7 +196,7 @@ medgemma_4b = HFModelClient(
     endpoint_url=MEDGEMMA_4B_ENDPOINT,
     hf_token=HF_TOKEN,
     model_name="medgemma-4b",
-    default_router_model="google/medgemma-4b-it",
+    default_router_model="Qwen/Qwen2.5-7B-Instruct",
     cache_instance=radiology_cache,
 )
 

medgemma.py

@@ -0,0 +1,39 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""MedGemma simulation module to match the original cache key signatures."""
+
+from cache_manager import intake_cache as cache
+from llm_client import medgemma_27b
+
+
+@cache.memoize()
+def medgemma_get_text_response(
+    messages: list,
+    temperature: float = 0.1,
+    max_tokens: int = 4096,
+    stream: bool = False,
+    top_p: float | None = None,
+    seed: int | None = None,
+    stop: list[str] | str | None = None,
+    frequency_penalty: float | None = None,
+    presence_penalty: float | None = None,
+    model: str = "tgi",
+):
+    """Checks cache first via memoization. On cache miss, calls MedGemma 27B."""
+    return medgemma_27b.chat_completion(
+        messages=messages,
+        temperature=temperature,
+        max_tokens=max_tokens,
+        stream=stream,
+    )

radiology/routes.py

@@ -116,9 +116,22 @@ def explain_sentence():
         {"role": "user", "content": user_prompt},
     ]
 
+    # Check radiology_cache first (matches the default cache key format)
+    from cache_manager import radiology_cache
+    cache_key = f"explain::{report_name}::{sentence}"
+    cached_result = radiology_cache.get(cache_key)
+    if cached_result:
+        logger.info("Cache hit for radiology sentence: %s", sentence)
+        return jsonify({"explanation": cached_result.strip()})
+
     try:
         logger.info("Explaining radiology sentence for case %s: %s", report_name, sentence)
         explanation = medgemma_4b.chat_completion(messages, temperature=0.1, max_tokens=1024)
+        
+        # Save to cache
+        if explanation:
+            radiology_cache.set(cache_key, explanation)
+            
         return jsonify({"explanation": explanation.strip()})
     except Exception as e:
         logger.error("Error calling LLM for explanation: %s", e)