Enhance patient analysis form with dynamic site-specific fields and support for lab image uploads

app.py

@@ -12,6 +12,7 @@ import streamlit as st
 PROJECT_ROOT = Path(__file__).parent
 sys.path.insert(0, str(PROJECT_ROOT))
 
+from src.form_config import CREATININE_PROMINENT_SITES, SITE_SPECIFIC_FIELDS, SUSPECTED_SOURCE_OPTIONS
 from src.tools import (
     calculate_mic_trend,
     get_empirical_therapy_guidance,
@@ -380,13 +381,75 @@ def page_patient_analysis():
             height = st.number_input("Height (cm)", 50.0, 250.0, 170.0, step=0.5)
         with c2:
             sex = st.selectbox("Biological sex", ["male", "female"])
-            creatinine = st.number_input("Serum Creatinine (mg/dL)", 0.1, 20.0, 1.2, step=0.1)
+            # Infection site is needed to decide creatinine visibility, so render it first
+            # (Streamlit reruns top-to-bottom, but c3 renders in the same pass, so we
+            #  read infection_site from session state on the *next* rerun.  We default
+            #  to the current widget value via a placeholder key.)
+            infection_site = st.session_state.get("_infection_site_val", "urinary")
+            if infection_site in CREATININE_PROMINENT_SITES:
+                creatinine = st.number_input("Serum Creatinine (mg/dL)", 0.1, 20.0, 1.2, step=0.1,
+                                             help="Required for CrCl-based dose adjustment")
+            else:
+                renal_flag = st.checkbox("Known renal impairment / CKD?",
+                                         help="Check to enter serum creatinine for dose adjustment")
+                creatinine = (
+                    st.number_input("Serum Creatinine (mg/dL)", 0.1, 20.0, 1.2, step=0.1)
+                    if renal_flag else None
+                )
         with c3:
             infection_site = st.selectbox(
                 "Primary infection site",
                 ["urinary", "respiratory", "bloodstream", "skin", "intra-abdominal", "CNS", "other"],
+                key="_infection_site_val",
             )
-            suspected_source = st.text_input("Suspected source", placeholder="e.g., community-acquired UTI")
+            source_options = SUSPECTED_SOURCE_OPTIONS.get(infection_site, [])
+            if source_options:
+                suspected_source = st.selectbox("Suspected source", source_options)
+                if suspected_source == "Other":
+                    suspected_source = st.text_input(
+                        "Specify source", placeholder="Describe the suspected source"
+                    )
+            else:
+                suspected_source = st.text_input(
+                    "Suspected source", placeholder="e.g., community-acquired infection"
+                )
+
+    # ── Site-specific assessment (dynamic per infection site) ──
+    site_vitals: dict[str, str] = {}
+    site_fields = SITE_SPECIFIC_FIELDS.get(infection_site, [])
+    if site_fields:
+        with st.expander(f"Site-Specific Assessment — {infection_site.title()}", expanded=True):
+            cols = st.columns(2)
+            for i, field in enumerate(site_fields):
+                col = cols[i % 2]
+                with col:
+                    fkey = f"site_{field['key']}"
+                    ftype = field["type"]
+                    if ftype == "selectbox":
+                        val = st.selectbox(field["label"], field["options"], key=fkey)
+                    elif ftype == "multiselect":
+                        val = st.multiselect(field["label"], field["options"], key=fkey)
+                        val = ", ".join(val) if val else ""
+                    elif ftype == "number_input":
+                        val = st.number_input(
+                            field["label"],
+                            min_value=field.get("min", 0.0),
+                            max_value=field.get("max", 999.0),
+                            value=field.get("default", 0.0),
+                            step=field.get("step", 1.0),
+                            key=fkey,
+                        )
+                        val = str(val)
+                    elif ftype == "checkbox":
+                        val = st.checkbox(
+                            field["label"], value=field.get("default", False), key=fkey
+                        )
+                        val = "Yes" if val else "No"
+                    elif ftype == "text_input":
+                        val = st.text_input(field["label"], key=fkey)
+                    else:
+                        continue
+                    site_vitals[field["key"]] = str(val)
 
     with st.expander("Medical History"):
         c1, c2 = st.columns(2)
@@ -404,9 +467,56 @@ def page_patient_analysis():
             )
 
     with st.expander("Lab / Culture Results  (optional — triggers targeted pathway)"):
-        method = st.radio("Input method", ["None — empirical pathway only", "Paste lab text"], horizontal=True)
+        method = st.radio(
+            "Input method",
+            ["None — empirical pathway only", "Upload file (PDF / image)", "Paste lab text"],
+            horizontal=True,
+        )
         labs_raw_text = None
-        if method == "Paste lab text":
+        labs_image_bytes = None
+
+        if method == "Upload file (PDF / image)":
+            uploaded = st.file_uploader(
+                "Lab report file",
+                type=["pdf", "png", "jpg", "jpeg", "tiff", "tif", "bmp"],
+                help="Upload a culture & sensitivity report, antibiogram, or any lab document.",
+            )
+            if uploaded is not None:
+                file_bytes = uploaded.read()
+                ext = uploaded.name.rsplit(".", 1)[-1].lower()
+                if ext == "pdf":
+                    # Extract text from PDF using pypdf
+                    import pypdf
+                    from io import BytesIO
+                    try:
+                        reader = pypdf.PdfReader(BytesIO(file_bytes))
+                        extracted = "\n".join(
+                            page.extract_text() or "" for page in reader.pages
+                        ).strip()
+                        if extracted:
+                            labs_raw_text = extracted
+                            st.success(f"PDF parsed — {len(reader.pages)} page(s), {len(extracted)} characters extracted.")
+                        else:
+                            st.warning(
+                                "PDF text extraction returned empty content (scanned PDF?). "
+                                "The file will be processed as an image by the vision model."
+                            )
+                            # Convert first page to image fallback via pillow (requires pypdf extras)
+                            labs_image_bytes = file_bytes
+                    except Exception as e:
+                        st.error(f"PDF parsing failed: {e}")
+                else:
+                    # Image file — pass directly to the multimodal model
+                    labs_image_bytes = file_bytes
+                    from PIL import Image as _PILImage
+                    from io import BytesIO as _BytesIO
+                    try:
+                        thumb = _PILImage.open(_BytesIO(file_bytes))
+                        st.image(thumb, caption=f"Uploaded: {uploaded.name}", width=320)
+                    except Exception:
+                        st.info(f"Image uploaded: {uploaded.name}")
+
+        elif method == "Paste lab text":
             labs_raw_text = st.text_area(
                 "Lab report",
                 placeholder=(
@@ -422,6 +532,7 @@ def page_patient_analysis():
     run_btn = st.button("Run Agent Pipeline", type="primary", use_container_width=False)
 
     if run_btn:
+        has_lab_input = bool(labs_raw_text or labs_image_bytes)
         patient_data = {
             "age_years": age,
             "weight_kg": weight,
@@ -433,11 +544,13 @@ def page_patient_analysis():
             "medications": [m.strip() for m in medications.split("\n") if m.strip()],
             "allergies": [a.strip() for a in allergies.split("\n") if a.strip()],
             "comorbidities": list(comorbidities) + list(risk_factors),
+            "vitals": site_vitals,
+            "labs_image_bytes": labs_image_bytes,
         }
 
         stages = (
             ["Intake Historian", "Vision Specialist", "Trend Analyst", "Clinical Pharmacologist"]
-            if labs_raw_text
+            if has_lab_input
             else ["Intake Historian", "Clinical Pharmacologist"]
         )
 
@@ -449,7 +562,7 @@ def page_patient_analysis():
             from src.graph import run_pipeline
             result = run_pipeline(patient_data, labs_raw_text)
         except Exception:
-            result = _demo_result(patient_data, labs_raw_text)
+            result = _demo_result(patient_data, labs_raw_text or bool(labs_image_bytes))
 
         prog.progress(100, text="Complete")
         st.session_state.pipeline_result = result

src/agents.py

@@ -12,7 +12,7 @@ import logging
 from typing import Optional
 
 from .config import get_settings
-from .loader import run_inference, TextModelName
+from .loader import run_inference, run_inference_with_image, TextModelName
 from .prompts import (
     INTAKE_HISTORIAN_SYSTEM,
     INTAKE_HISTORIAN_PROMPT,
@@ -66,12 +66,17 @@ def run_intake_historian(state: InfectionState) -> InfectionState:
         patient_context={"pathogen_type": state.get("suspected_source")},
     )
 
+    site_vitals_str = "\n".join(
+        f"- {k.replace('_', ' ').title()}: {v}" for k, v in state.get("vitals", {}).items()
+    ) or "None provided"
+
     prompt = f"{INTAKE_HISTORIAN_SYSTEM}\n\n{INTAKE_HISTORIAN_PROMPT.format(
         patient_data=patient_data,
         medications=', '.join(state.get('medications', [])) or 'None reported',
         allergies=', '.join(state.get('allergies', [])) or 'No known allergies',
         infection_site=state.get('infection_site', 'Unknown'),
         suspected_source=state.get('suspected_source', 'Unknown'),
+        site_vitals=site_vitals_str,
         rag_context=rag_context,
     )}"
 
@@ -105,14 +110,24 @@ def run_vision_specialist(state: InfectionState) -> InfectionState:
     logger.info("Running Vision Specialist agent...")
 
     labs_raw = state.get("labs_raw_text", "")
-    if not labs_raw:
+    labs_image_bytes = state.get("labs_image_bytes")
+
+    if not labs_raw and not labs_image_bytes:
         logger.info("No lab data to process, skipping Vision Specialist")
         state["vision_notes"] = "No lab data provided"
         state["route_to_trend_analyst"] = False
         return state
 
-    # Language detection is not implemented; we assume English or instruct the model to translate
-    language = "English (assumed)"
+    # Determine input modality and prepare prompt content description
+    if labs_image_bytes:
+        source_format = "image"
+        language = "Auto-detected"
+        report_content = "See attached image — extract all lab data visible in the image."
+    else:
+        source_format = "text"
+        language = "English (assumed)"
+        report_content = labs_raw
+
     rag_context = get_context_for_agent(
         agent_name="vision_specialist",
         query="culture sensitivity susceptibility interpretation",
@@ -120,13 +135,22 @@ def run_vision_specialist(state: InfectionState) -> InfectionState:
     )
 
     prompt = f"{VISION_SPECIALIST_SYSTEM}\n\n{VISION_SPECIALIST_PROMPT.format(
-        report_content=labs_raw,
-        source_format='text',
+        report_content=report_content,
+        source_format=source_format,
         language=language,
     )}"
 
     try:
-        response = run_inference(prompt=prompt, model_name="medgemma_4b", max_new_tokens=2048, temperature=0.1)
+        if labs_image_bytes:
+            from io import BytesIO
+            from PIL import Image as PILImage
+            image = PILImage.open(BytesIO(labs_image_bytes)).convert("RGB")
+            logger.info(f"Running vision inference on uploaded image ({image.size})")
+            response = run_inference_with_image(
+                prompt=prompt, image=image, model_name="medgemma_4b", max_new_tokens=2048, temperature=0.1
+            )
+        else:
+            response = run_inference(prompt=prompt, model_name="medgemma_4b", max_new_tokens=2048, temperature=0.1)
         parsed = safe_json_parse(response)
         if parsed:
             state["vision_notes"] = json.dumps(parsed, indent=2)
@@ -256,6 +280,10 @@ def run_clinical_pharmacologist(state: InfectionState) -> InfectionState:
         patient_context={"proposed_antibiotic": None},
     )
 
+    site_vitals_str = "\n".join(
+        f"- {k.replace('_', ' ').title()}: {v}" for k, v in state.get("vitals", {}).items()
+    ) or "None provided"
+
     prompt = f"{CLINICAL_PHARMACOLOGIST_SYSTEM}\n\n{CLINICAL_PHARMACOLOGIST_PROMPT.format(
         intake_summary=intake_summary,
         lab_results=lab_results,
@@ -268,6 +296,7 @@ def run_clinical_pharmacologist(state: InfectionState) -> InfectionState:
         infection_site=state.get('infection_site', 'Unknown'),
         suspected_source=state.get('suspected_source', 'Unknown'),
         severity=state.get('intake_notes', {}).get('infection_severity', 'Unknown') if isinstance(state.get('intake_notes'), dict) else 'Unknown',
+        site_vitals=site_vitals_str,
         rag_context=rag_context,
     )}"
 
@@ -355,8 +384,10 @@ def _format_patient_data(state: InfectionState) -> str:
         lines.append(f"Comorbidities: {', '.join(state['comorbidities'])}")
 
     if state.get("vitals"):
-        vitals_str = ", ".join(f"{k}: {v}" for k, v in state["vitals"].items())
-        lines.append(f"Vitals: {vitals_str}")
+        lines.append("Site-Specific Assessment:")
+        for k, v in state["vitals"].items():
+            label = k.replace("_", " ").title()
+            lines.append(f"  - {label}: {v}")
 
     return "\n".join(lines) if lines else "No patient data available"
 

src/form_config.py

@@ -0,0 +1,327 @@
+"""
+Declarative field definitions for the dynamic Patient Analysis form.
+
+Each infection site maps to a list of site-specific fields and contextual
+suspected-source options. Universal fields (age, sex, weight, height,
+creatinine, medications, allergies, comorbidities, risk factors) are always
+shown and are NOT listed here.
+"""
+
+SITE_SPECIFIC_FIELDS: dict[str, list[dict]] = {
+    "urinary": [
+        {
+            "key": "catheter_status",
+            "label": "Catheter status",
+            "type": "selectbox",
+            "options": [
+                "No catheter",
+                "Indwelling (Foley)",
+                "Intermittent",
+                "Suprapubic",
+                "Recently removed (<48 h)",
+            ],
+        },
+        {
+            "key": "urinary_symptoms",
+            "label": "Urinary symptoms",
+            "type": "multiselect",
+            "options": [
+                "Dysuria",
+                "Frequency",
+                "Urgency",
+                "Hematuria",
+                "Suprapubic pain",
+                "Flank pain",
+                "Fever / chills",
+            ],
+        },
+        {
+            "key": "urine_appearance",
+            "label": "Urine appearance",
+            "type": "selectbox",
+            "options": ["Clear", "Cloudy", "Turbid", "Malodorous", "Hematuria"],
+        },
+    ],
+    "respiratory": [
+        {
+            "key": "o2_saturation",
+            "label": "O\u2082 Saturation (%)",
+            "type": "number_input",
+            "min": 50.0,
+            "max": 100.0,
+            "default": 97.0,
+            "step": 0.5,
+        },
+        {
+            "key": "ventilation_status",
+            "label": "Ventilation status",
+            "type": "selectbox",
+            "options": [
+                "Room air",
+                "Supplemental O\u2082 (nasal cannula)",
+                "Supplemental O\u2082 (mask)",
+                "Non-invasive (BiPAP / CPAP)",
+                "Mechanical ventilation",
+            ],
+        },
+        {
+            "key": "cough_type",
+            "label": "Cough type",
+            "type": "selectbox",
+            "options": ["None", "Dry", "Productive", "Hemoptysis"],
+        },
+        {
+            "key": "sputum_character",
+            "label": "Sputum character",
+            "type": "selectbox",
+            "options": [
+                "None",
+                "Clear / white",
+                "Yellow",
+                "Green / purulent",
+                "Rust-colored",
+                "Blood-tinged",
+            ],
+        },
+    ],
+    "bloodstream": [
+        {
+            "key": "central_line_present",
+            "label": "Central line present",
+            "type": "checkbox",
+            "default": False,
+        },
+        {
+            "key": "temperature_c",
+            "label": "Temperature (\u00b0C)",
+            "type": "number_input",
+            "min": 34.0,
+            "max": 43.0,
+            "default": 38.5,
+            "step": 0.1,
+        },
+        {
+            "key": "heart_rate_bpm",
+            "label": "Heart rate (bpm)",
+            "type": "number_input",
+            "min": 30,
+            "max": 250,
+            "default": 90,
+            "step": 1,
+        },
+        {
+            "key": "respiratory_rate",
+            "label": "Respiratory rate (/min)",
+            "type": "number_input",
+            "min": 5,
+            "max": 60,
+            "default": 18,
+            "step": 1,
+        },
+        {
+            "key": "wbc_count",
+            "label": "WBC count (\u00d710\u2079/L)",
+            "type": "number_input",
+            "min": 0.0,
+            "max": 100.0,
+            "default": 12.0,
+            "step": 0.1,
+        },
+        {
+            "key": "lactate_mmol",
+            "label": "Lactate (mmol/L)",
+            "type": "number_input",
+            "min": 0.0,
+            "max": 30.0,
+            "default": 1.0,
+            "step": 0.1,
+        },
+        {
+            "key": "shock_status",
+            "label": "Shock status",
+            "type": "selectbox",
+            "options": [
+                "No shock",
+                "Compensated (SBP > 90, tachycardia)",
+                "Septic shock (vasopressors required)",
+            ],
+        },
+    ],
+    "skin": [
+        {
+            "key": "wound_type",
+            "label": "Wound type",
+            "type": "selectbox",
+            "options": [
+                "Laceration",
+                "Ulcer (diabetic / pressure)",
+                "Bite (animal / human)",
+                "Surgical site",
+                "Burn",
+                "Abscess",
+                "Cellulitis (no wound)",
+            ],
+        },
+        {
+            "key": "cellulitis_extent",
+            "label": "Cellulitis extent",
+            "type": "selectbox",
+            "options": [
+                "None",
+                "Localized (< 5 cm)",
+                "Moderate (5\u201310 cm)",
+                "Extensive (> 10 cm)",
+                "Rapidly spreading",
+            ],
+        },
+        {
+            "key": "abscess_present",
+            "label": "Abscess present",
+            "type": "checkbox",
+            "default": False,
+        },
+        {
+            "key": "foreign_body",
+            "label": "Foreign body / implant",
+            "type": "checkbox",
+            "default": False,
+        },
+    ],
+    "intra-abdominal": [
+        {
+            "key": "abdominal_pain_location",
+            "label": "Pain location",
+            "type": "selectbox",
+            "options": [
+                "Diffuse",
+                "RUQ",
+                "LUQ",
+                "RLQ",
+                "LLQ",
+                "Epigastric",
+                "Periumbilical",
+            ],
+        },
+        {
+            "key": "peritonitis_signs",
+            "label": "Peritonitis signs",
+            "type": "multiselect",
+            "options": [
+                "Guarding",
+                "Rebound tenderness",
+                "Rigidity",
+                "Absent bowel sounds",
+            ],
+        },
+        {
+            "key": "perforation_suspected",
+            "label": "Perforation suspected",
+            "type": "checkbox",
+            "default": False,
+        },
+        {
+            "key": "ascites",
+            "label": "Ascites present",
+            "type": "checkbox",
+            "default": False,
+        },
+    ],
+    "CNS": [
+        {
+            "key": "csf_obtained",
+            "label": "CSF obtained",
+            "type": "checkbox",
+            "default": False,
+        },
+        {
+            "key": "neuro_symptoms",
+            "label": "Neurological symptoms",
+            "type": "multiselect",
+            "options": [
+                "Headache",
+                "Neck stiffness",
+                "Photophobia",
+                "Altered mental status",
+                "Seizures",
+                "Focal deficits",
+            ],
+        },
+        {
+            "key": "recent_neurosurgery",
+            "label": "Recent neurosurgery",
+            "type": "checkbox",
+            "default": False,
+        },
+        {
+            "key": "gcs_score",
+            "label": "GCS score",
+            "type": "number_input",
+            "min": 3,
+            "max": 15,
+            "default": 15,
+            "step": 1,
+        },
+    ],
+    "other": [],
+}
+
+
+# Sites where serum creatinine is shown prominently in demographics.
+# For all other sites a "renal impairment?" toggle is shown instead.
+CREATININE_PROMINENT_SITES: frozenset[str] = frozenset(
+    {"urinary", "bloodstream", "CNS", "respiratory"}
+)
+
+SUSPECTED_SOURCE_OPTIONS: dict[str, list[str]] = {
+    "urinary": [
+        "Community-acquired UTI",
+        "Catheter-associated UTI (CAUTI)",
+        "Complicated UTI",
+        "Pyelonephritis",
+        "Urosepsis",
+        "Other",
+    ],
+    "respiratory": [
+        "Community-acquired pneumonia (CAP)",
+        "Hospital-acquired pneumonia (HAP)",
+        "Ventilator-associated pneumonia (VAP)",
+        "Aspiration pneumonia",
+        "Lung abscess",
+        "Empyema",
+        "Other",
+    ],
+    "bloodstream": [
+        "Primary bacteremia",
+        "Catheter-related BSI (CRBSI)",
+        "Secondary bacteremia (from known source)",
+        "Endocarditis",
+        "Unknown source",
+        "Other",
+    ],
+    "skin": [
+        "Cellulitis",
+        "Surgical site infection",
+        "Diabetic foot infection",
+        "Bite wound infection",
+        "Necrotizing fasciitis",
+        "Abscess",
+        "Other",
+    ],
+    "intra-abdominal": [
+        "Appendicitis",
+        "Cholecystitis / cholangitis",
+        "Diverticulitis",
+        "Peritonitis (SBP)",
+        "Post-surgical",
+        "Liver abscess",
+        "Other",
+    ],
+    "CNS": [
+        "Community-acquired meningitis",
+        "Post-neurosurgical meningitis",
+        "Healthcare-associated ventriculitis",
+        "Brain abscess",
+        "Other",
+    ],
+    "other": [],
+}

src/graph.py

@@ -77,6 +77,9 @@ def run_pipeline(patient_data: dict, labs_raw_text: str | None = None) -> Infect
     Pass labs_raw_text to trigger the targeted (Stage 2) pathway.
     Without it, only the empirical (Stage 1) pathway runs.
     """
+    labs_image_bytes: bytes | None = patient_data.get("labs_image_bytes")
+    has_lab_input = bool(labs_raw_text or labs_image_bytes)
+
     initial_state: InfectionState = {
         "age_years": patient_data.get("age_years"),
         "weight_kg": patient_data.get("weight_kg"),
@@ -90,15 +93,17 @@ def run_pipeline(patient_data: dict, labs_raw_text: str | None = None) -> Infect
         "suspected_source": patient_data.get("suspected_source"),
         "country_or_region": patient_data.get("country_or_region"),
         "vitals": patient_data.get("vitals", {}),
-        "stage": "targeted" if labs_raw_text else "empirical",
+        "stage": "targeted" if has_lab_input else "empirical",
         "errors": [],
         "safety_warnings": [],
     }
 
     if labs_raw_text:
         initial_state["labs_raw_text"] = labs_raw_text
+    if labs_image_bytes:
+        initial_state["labs_image_bytes"] = labs_image_bytes
 
-    logger.info(f"Starting pipeline (stage: {initial_state['stage']})")
+    logger.info(f"Starting pipeline (stage: {initial_state['stage']}, lab_text={bool(labs_raw_text)}, lab_image={bool(labs_image_bytes)})")
     logger.info(f"Patient: {patient_data.get('age_years')}y, {patient_data.get('sex')}, infection: {patient_data.get('infection_site')}")
     
     try:

src/loader.py

@@ -58,9 +58,20 @@ def _get_local_multimodal(model_name: TextModelName):
     model = AutoModelForImageTextToText.from_pretrained(model_path, **load_kwargs)
     logger.info(f"Model loaded successfully: {model_path}")
 
-    def _call(prompt: str, max_new_tokens: int = 512, temperature: float = 0.2, **generate_kwargs: Any) -> str:
-        # Build a chat-style input for text-only queries
-        messages = [{"role": "user", "content": [{"type": "text", "text": prompt}]}]
+    def _call(
+        prompt: str,
+        max_new_tokens: int = 512,
+        temperature: float = 0.2,
+        image=None,  # optional PIL.Image.Image for vision-language inference
+        **generate_kwargs: Any,
+    ) -> str:
+        # Build chat content; prepend image token when an image is provided
+        content = []
+        if image is not None:
+            content.append({"type": "image", "image": image})
+        content.append({"type": "text", "text": prompt})
+        messages = [{"role": "user", "content": content}]
+
         inputs = processor.apply_chat_template(
             messages, add_generation_prompt=True, tokenize=True,
             return_dict=True, return_tensors="pt",
@@ -155,3 +166,38 @@ def run_inference(
     except Exception as e:
         logger.error(f"Inference failed for {model_name}: {e}", exc_info=True)
         raise
+
+
+def run_inference_with_image(
+    prompt: str,
+    image: Any,  # PIL.Image.Image
+    model_name: TextModelName = "medgemma_4b",
+    max_new_tokens: int = 1024,
+    temperature: float = 0.1,
+    **kwargs: Any,
+) -> str:
+    """
+    Run vision-language inference passing a PIL image alongside the text prompt.
+
+    Falls back to text-only inference if the resolved model is not multimodal
+    (e.g. when medgemma_4b is remapped to a text-only model in the env config).
+    """
+    logger.info(f"Running vision inference with {model_name}, max_tokens={max_new_tokens}")
+    try:
+        model_path = _get_model_path(model_name)
+        if not _is_multimodal(model_path):
+            logger.warning(
+                f"{model_name} ({model_path}) is not a multimodal model; "
+                "falling back to text-only inference."
+            )
+            return run_inference(prompt, model_name, max_new_tokens, temperature, **kwargs)
+
+        model_fn = _get_local_multimodal(model_name)
+        result = model_fn(
+            prompt, max_new_tokens=max_new_tokens, temperature=temperature, image=image, **kwargs
+        )
+        logger.info(f"Vision inference complete, response length: {len(result)} chars")
+        return result
+    except Exception as e:
+        logger.error(f"Vision inference failed for {model_name}: {e}", exc_info=True)
+        raise

src/prompts.py

@@ -49,6 +49,9 @@ CLINICAL CONTEXT:
 - Suspected infection site: {infection_site}
 - Suspected source: {suspected_source}
 
+SITE-SPECIFIC ASSESSMENT:
+{site_vitals}
+
 RAG CONTEXT (Relevant Guidelines):
 {rag_context}
 
@@ -267,6 +270,9 @@ INFECTION CONTEXT:
 - Source: {suspected_source}
 - Severity: {severity}
 
+SITE-SPECIFIC ASSESSMENT:
+{site_vitals}
+
 RAG CONTEXT (Guidelines & Safety Data):
 {rag_context}
 

src/state.py

@@ -64,7 +64,8 @@ class InfectionState(TypedDict, total=False):
     vitals: NotRequired[Dict[str, str]]  # flexible key/value, e.g. {"BP": "120/80"}
 
     # Lab data & MICs
-    labs_raw_text: NotRequired[Optional[str]]  # raw OCR or PDF text
+    labs_raw_text: NotRequired[Optional[str]]   # raw OCR or pasted text
+    labs_image_bytes: NotRequired[Optional[bytes]]  # uploaded image (PNG/JPG/TIFF) for vision model
     labs_parsed: NotRequired[List[LabResult]]
     mic_data: NotRequired[List[MICDatum]]
     mic_trend_summary: NotRequired[Optional[str]]