Deploy build-99deee4

.dockerignore

@@ -0,0 +1,42 @@
+# Git metadata
+.git/
+.gitignore
+
+# Local environments and caches
+.codex
+.venv/
+venv/
+env/
+.codex/
+.cache/
+.huggingface-spaces/
+data/hf_cache/
+__pycache__/
+**/__pycache__/
+*.py[cod]
+
+# Local docs and screenshots not needed in the image build context
+Screenshot_*.png
+
+# Development-only files
+docker-compose.yml
+
+# Large generated artifacts that should not bloat the Docker build context
+data/*.npz
+models/realworld_densenet121/
+models/realworld_efficientnet_b0/
+models/plots/
+models/*.csv
+
+# Keep core project files available
+!backend/
+!frontend/
+!scripts/
+!models/
+!data/
+!docs/
+!README.md
+!requirements.txt
+!Dockerfile
+!.python-version
+!.github/

.github/workflows/deploy.yml

@@ -0,0 +1,135 @@
+name: CI / Lint + Deploy to Hugging Face Spaces
+
+on:
+  push:
+    branches: ["main", "master"]
+  pull_request:
+    branches: ["main", "master"]
+  workflow_dispatch:
+
+jobs:
+  # ─────────────────────────────────────────────────────────────────────────────
+  # 1. CI — syntax + import smoke test
+  # ─────────────────────────────────────────────────────────────────────────────
+  ci:
+    name: Syntax & Import Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          # Install CPU-only torch to keep CI fast
+          pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
+          pip install -r requirements.txt
+
+      - name: Compile-check all Python modules
+        run: |
+          python -m py_compile \
+            scripts/train_chestmnist.py \
+            backend/main.py \
+            frontend/app.py \
+            model_utils.py
+
+      - name: Import smoke test (no model files needed)
+        run: |
+          python - <<'PY'
+          import ast, pathlib
+          for f in ["scripts/train_chestmnist.py", "backend/main.py", "frontend/app.py", "model_utils.py"]:
+              ast.parse(pathlib.Path(f).read_text())
+              print(f"  {f} OK")
+          print("All imports parseable.")
+          PY
+
+  # ─────────────────────────────────────────────────────────────────────────────
+  # 2. Deploy — push code to Hugging Face Space (on main branch only)
+  # ─────────────────────────────────────────────────────────────────────────────
+  deploy:
+    name: Deploy to Hugging Face Spaces
+    if: (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master') && github.event_name == 'push'
+    needs: ci
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install huggingface_hub
+        run: pip install huggingface_hub
+
+      - name: Push Space to Hugging Face Hub
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          HF_SPACE_REPO: ${{ secrets.HF_SPACE_REPO }}
+          GITHUB_SHA: ${{ github.sha }}
+        run: |
+          python - <<'PY'
+          import os
+          from huggingface_hub import HfApi
+
+          token      = os.environ["HF_TOKEN"]
+          space_repo = os.environ["HF_SPACE_REPO"]   # e.g. "your-username/pneumoops"
+          sha_tag    = f"build-{os.environ['GITHUB_SHA'][:7]}"
+
+          api = HfApi(token=token)
+          api.create_repo(repo_id=space_repo, repo_type="space", space_sdk="docker", exist_ok=True)
+
+          api.upload_folder(
+              repo_id=space_repo,
+              repo_type="space",
+              folder_path=".",
+              commit_message=f"Deploy {sha_tag}",
+              ignore_patterns=[
+                  ".git/*", "__pycache__/*", "*.pyc",
+                  ".venv/*", "venv/*",
+                  "data/hf_cache/*",
+              ],
+          )
+
+          try:
+              api.create_tag(repo_id=space_repo, repo_type="space", tag=sha_tag)
+          except Exception:
+              pass   # tag already exists
+
+          print(f"Deployed {sha_tag} → https://huggingface.co/spaces/{space_repo}")
+          PY
+
+      - name: Upload model artifacts to HF Model Hub
+        # Only runs if HF_MODEL_REPO secret is set
+        if: ${{ env.HF_MODEL_REPO != '' }}
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          HF_MODEL_REPO: ${{ secrets.HF_MODEL_REPO }}
+          GITHUB_SHA: ${{ github.sha }}
+        run: |
+          python - <<'PY'
+          import os
+          from huggingface_hub import HfApi
+          from pathlib import Path
+
+          token      = os.environ["HF_TOKEN"]
+          model_repo = os.environ["HF_MODEL_REPO"]
+          sha_tag    = f"build-{os.environ['GITHUB_SHA'][:7]}"
+
+          api = HfApi(token=token)
+          api.create_repo(repo_id=model_repo, repo_type="model", exist_ok=True)
+
+          model_dir = Path("models/chestmnist_mobilenetv3")
+          if model_dir.exists():
+              api.upload_folder(
+                  repo_id=model_repo,
+                  repo_type="model",
+                  folder_path=str(model_dir),
+                  commit_message=f"Upload model {sha_tag}",
+              )
+              print(f"Model artifacts uploaded → https://huggingface.co/models/{model_repo}")
+          PY

.gitignore

@@ -0,0 +1,56 @@
+# Python
+__pycache__/
+*.py[cod]
+*.so
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+dist/
+build/
+*.egg-info/
+
+# Virtual environments
+.venv/
+venv/
+env/
+
+# Editor / OS
+.DS_Store
+.idea/
+.vscode/
+.codex
+.codex/
+
+# Secrets
+.env
+.env.*
+*.local
+
+# Logs and temp
+*.log
+*.tmp
+
+# Dataset cache (auto-downloaded — not committed)
+data/hf_cache/
+data/*.npz
+data/*.json
+.cache/
+
+# Screenshots
+Screenshot_*.png
+
+# ─── Model artifacts (large binaries — stored on HF Hub, not in Git) ─────────
+models/*.pth
+models/*.onnx
+models/*.json
+models/*.csv
+models/chestmnist_mobilenetv3/*.pth
+models/chestmnist_mobilenetv3/*.onnx
+models/chestmnist_mobilenetv3/*.npz
+models/chestmnist_mobilenetv3/plots/
+
+# Keep directory structure in Git
+!models/.gitkeep
+!data/.gitkeep

ASSIGNMENT_TARGET.md

@@ -0,0 +1,37 @@
+# PneumoOps Assignment Target
+
+## Title
+PneumoOps: Continuous MLOps Pipeline with A/B Testing & Drift Monitoring for Multi-Class Thoracic Diagnostics
+
+## Project Task
+Build an advanced, continuous ML/DL Ops pipeline for 14-class chest X-ray classification. The system will deploy containerized models via Docker on Hugging Face Spaces, featuring live A/B testing between a standard and an optimized (ONNX) model. With a strict focus on real-world maintenance, the backend includes real-time inference latency tracking and a data drift monitoring system that triggers automated alerts for model retraining when production data shifts.
+
+## Idea Explanation in Detail
+This project upgrades a multi-label classification task into a complete, production-ready MLOps system, addressing real-world "Lab-to-Production" challenges and ongoing maintenance. In a real clinical setting, models degrade over time as scanner hardware changes or patient demographics shift. Instead of deploying a single static model, we upload two versions to the Hugging Face Hub: a baseline PyTorch model and an ONNX-optimized model.
+The Dockerized FastAPI backend performs A/B testing by randomly routing user uploads to either model, allowing us to compare real-world inference latency and performance. Crucially for deployment maintenance, the backend includes a Data Drift Monitor that compares the incoming image's statistical distribution against the training dataset. If an out-of-distribution image is uploaded, the system flags it as "Drift Detected," simulating the exact trigger required for automated retraining cycles in enterprise MLOps.
+
+## Datasets and Models to be used
+
+- **Dataset**: ChestMNIST (14-class multi-label dataset containing Atelectasis, Cardiomegaly, Consolidation, Edema, Effusion, Emphysema, Fibrosis, Hernia, Infiltration, Mass, Nodule, Pleural Thickening, Pneumonia, Pneumothorax). It is lightweight enough for rapid CI/CD iterations without wasting hours on training.
+- **Model A (Baseline)**: MobileNetV3-small (Standard PyTorch format adapted for 14 output nodes).
+- **Model B (Optimized)**: MobileNetV3-small (Converted to ONNX format for faster client-level inference).
+
+## Task 1: The Gradio Web UI
+The UI needs to have an image upload component for a chest X-ray. When the user clicks submit, the app should send the image via a `requests.post` call to a local FastAPI backend (assume the URL is `http://127.0.0.1:8000/predict`).
+
+The UI needs to display the following output fields clearly:
+- **Predictions**: A bar chart or clean list showing the top 3 highest-probability conditions out of the 14 thoracic disease classes, e.g., Cardiomegaly, Effusion, Mass
+- **Model Used**: This will show either 'Baseline PyTorch' or 'Optimized ONNX' for our A/B testing
+- **Inference Latency**: (ms)
+- **Data Drift Alert**: Will show 'Normal' or 'Drift Detected' with a warning color to simulate production maintenance alerts
+
+*Please make the UI look clean, professional, and add a nice title and description at the top explaining that this is an MLOps A/B Testing Pipeline focused on real-world deployment and reliability.*
+
+## Task 2: The Poster Text
+Draft the text content for an academic project poster. Title: PneumoOps: Continuous MLOps Pipeline for 14-Class Thoracic Diagnostics. Generate professional, concise text for the following sections:
+- **Project Task**
+- **Focus on Deployment & Maintenance**
+- **Libraries / APIs**
+- **Deployment Architecture**
+- **Datasets and Models**
+*Keep the text punchy and readable for a visual poster presentation.*

Dockerfile

@@ -0,0 +1,42 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Apply latest system cybersecurity patches and install minimal dependencies
+RUN apt-get update && apt-get upgrade -y && \
+    apt-get install -y --no-install-recommends \
+    curl \
+    libgomp1 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Security: Create an unprivileged user for executing the application
+RUN groupadd -r pneumoops && useradd -r -g pneumoops appuser
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+# Copy application code and ensure secure permissions
+COPY --chown=appuser:pneumoops backend  ./backend
+COPY --chown=appuser:pneumoops frontend ./frontend
+COPY --chown=appuser:pneumoops scripts  ./scripts
+COPY --chown=appuser:pneumoops models   ./models
+COPY --chown=appuser:pneumoops data     ./data
+COPY --chown=appuser:pneumoops model_utils.py ./model_utils.py
+COPY --chown=appuser:pneumoops README.md ./README.md
+
+ENV PYTHONPATH=/app
+ENV PORT=7860
+ENV MPLCONFIGDIR=/tmp/matplotlib
+# Default to the ChestMNIST + MobileNetV3-small profile (assignment target)
+ENV PNEUMOOPS_PROFILE=chestmnist
+
+EXPOSE 7860
+
+HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \
+    CMD curl -f http://127.0.0.1:7860/health || exit 1
+
+# Security: Drop root privileges before running the application
+USER appuser
+
+CMD ["python", "-m", "uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "7860"]

POSTER_CONTENT.md

@@ -0,0 +1,40 @@
+# 🫁 PneumoOps: Continuous MLOps Pipeline for 14-Class Thoracic Diagnostics
+
+---
+
+### 🎯 Project Task
+An advanced, continuous ML/DL Ops pipeline designed for multi-label chest X-ray classification. The system containerizes models via **Docker** and deploys them to **Hugging Face Spaces**. It features live **A/B testing** between standard and optimized (ONNX) deployments, paired with robust **data drift monitoring** for 14 thoracic diseases, bridging the gap between lab research and continuous production.
+
+---
+
+### 🛠️ Focus on Deployment & Maintenance
+Traditional clinical models natively degrade over time as scanner hardware and patient demographics shift. PneumoOps addresses this "Lab-to-Production" friction through proactive maintenance:
+* **Real-time Latency Tracking:** Enables active comparison of model serving speeds, ensuring responsiveness on varying clinical hardware profiles in production environments.
+* **Data Drift Detection:** Continuously compares incoming clinical image distributions against the original training baseline. Discovering out-of-distribution uploads instantly triggers a `Drift Detected` alert—providing the automated, programmatic trigger needed for model retraining cycles in true enterprise MLOps.
+
+---
+
+### 💻 Libraries / APIs
+The pipeline is powered by a modern, full-stack open source MLOps ecosystem:
+* **PyTorch** (Model Training & Baseline Backend)
+* **ONNX Runtime** (Model Optimization & Accelerated Inference)
+* **FastAPI** (High-performance API Routing & Drift Computation)
+* **Gradio** (Interactive Maintenance Console & Dashboard)
+* **Docker** (Environment Standardization & Containerization)
+* **Hugging Face Hub/Spaces** (Cloud Hosting & Artifact Registry)
+
+---
+
+### 🏗️ Deployment Architecture
+A streamlined, automated flow from data to deployment:
+1. **Train** → Train Baseline MobileNetV3-small & Export to ONNX.
+2. **Push to HF Hub** → Push version-controlled weights, training metrics, and code.
+3. **Dockerized FastAPI Router** → Routes live traffic dynamically (50/50) for rapid A/B testing and computes drift statistics on the fly.
+4. **Gradio UI Dashboard** → Empowers clinical data teams with an interactive maintenance dashboard showing top-3 predictions, model-arm comparisons, and system data-drift alerts.
+
+---
+
+### 📊 Datasets and Models
+* **Dataset:** **ChestMNIST** — A highly efficient, 14-class multi-label dataset. Supports rapid CI/CD iteration loops while capturing complex thoracic conditions (e.g., Cardiomegaly, Pneumonia, Consolidation, Effusion, Mass).
+* **Model A (Baseline):** **MobileNetV3-small** in standard PyTorch format, tailored for 14 output nodes.
+* **Model B (Optimized):** **MobileNetV3-small** converted explicitly to ONNX architecture for faster execution speeds and reduced client hardware strain.

README.md

@@ -1,10 +1,210 @@
 ---
-title: Pneumoops
-emoji: 👀
-colorFrom: purple
+title: PneumoOps
+emoji: 🫁
+colorFrom: blue
 colorTo: indigo
 sdk: docker
-pinned: false
+app_port: 7860
+pinned: true
+license: mit
+short_description: MLOps A/B testing & drift monitoring
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# 🫁 PneumoOps
+
+**Continuous MLOps Pipeline with A/B Testing & Data Drift Monitoring for 14-Class Thoracic Disease Detection**
+
+[![CI](https://github.com/Prakhar54-byte/PneumoOps/actions/workflows/deploy.yml/badge.svg)](https://github.com/Prakhar54-byte/PneumoOps/actions)
+[![Python 3.11](https://img.shields.io/badge/python-3.11-blue)](https://www.python.org/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-green)](#)
+
+---
+
+## What This Is
+
+PneumoOps is a production-style MLOps system for **multi-label chest X-ray classification**. It demonstrates real-world deployment challenges:
+
+- **A/B Testing** — every inference request is randomly routed to either *Model A (PyTorch)* or *Model B (ONNX)*, letting you measure real-world latency differences between serving backends.
+- **Data Drift Monitoring** — statistical pixel-distribution comparison (KS-test) against the training baseline. When distribution shifts, the system flags `DRIFT_DETECTED` — the trigger for automated retraining in production.
+- **Prometheus Observability** — request counters, latency histograms, per-disease prediction rates, and drift alert counters are all scraped at `/metrics`.
+- **Dockerized Deployment** — the entire stack runs in containers, deployable to Hugging Face Spaces via a single `git push`.
+
+---
+
+## Architecture
+
+```
+         Train (ChestMNIST + MobileNetV3-small)
+                    │
+          ┌─────────┴──────────┐
+    Model A (.pth)       Model B (.onnx)
+    PyTorch serving      ONNX Runtime serving
+          └─────────┬──────────┘
+                    │
+        ┌───────────┴───────────┐
+        │  FastAPI Backend       │
+        │  ├─ A/B Router (60/40) │
+        │  ├─ Drift Monitor (KS) │
+        │  ├─ Prometheus /metrics│
+        │  └─ /health /history   │
+        └───────────┬───────────┘
+                    │
+          ┌─────────┴──────────┐
+          │   Gradio UI         │
+          │  Top-3 predictions  │
+          │  Model arm used     │
+          │  Latency (ms)       │
+          │  Drift alert badge  │
+          └─────────────────────┘
+```
+
+---
+
+## Dataset & Model
+
+| Property | Value |
+|---|---|
+| Dataset | [ChestMNIST](https://medmnist.com/) — 14-class multi-label chest X-ray |
+| Classes | Atelectasis, Cardiomegaly, Effusion, Infiltration, Mass, Nodule, Pneumonia, Pneumothorax, Consolidation, Edema, Emphysema, Fibrosis, Pleural Thickening, Hernia |
+| Model A | MobileNetV3-small (PyTorch `.pth`) |
+| Model B | MobileNetV3-small (ONNX Runtime `.onnx`) |
+| Training | 5 epochs, AdamW, BCEWithLogitsLoss, per-class threshold tuning |
+| Macro AUROC | 0.686 (5-epoch, 5k samples — improves with full dataset) |
+
+---
+
+## Project Structure
+
+```
+pneumo_ops/
+├── backend/
+│   └── main.py              # FastAPI: A/B routing, drift monitor, Prometheus
+├── frontend/
+│   └── app.py               # Gradio UI: top-3 chart, drift badge, latency
+├── scripts/
+│   └── train_chestmnist.py  # Training: ChestMNIST → MobileNetV3 → ONNX export
+├── models/
+│   └── chestmnist_mobilenetv3/
+│       ├── mobilenetv3_chestmnist.pth    # Model A (PyTorch)
+│       ├── mobilenetv3_chestmnist.onnx   # Model B (ONNX)
+│       ├── training_metrics.json
+│       └── baseline_stats.json           # Pixel stats for drift reference
+├── model_utils.py           # CalibratedModel + temperature scaling util
+├── Dockerfile               # Single-container build
+├── docker-compose.yml       # backend + frontend services
+├── requirements.txt
+└── .github/workflows/
+    └── deploy.yml           # CI (lint/import check) + HF Spaces deploy
+```
+
+---
+
+## Quick Start
+
+### 1. Install
+
+```bash
+git clone https://github.com/Prakhar54-byte/PneumoOps
+cd pneumo_ops
+python3 -m venv .venv && source .venv/bin/activate
+pip install -r requirements.txt
+```
+
+### 2. Train the model
+
+```bash
+# Quick run (5k samples, ~1 min on GPU)
+python3 scripts/train_chestmnist.py --epochs 5 --batch-size 32 --max-train-samples 5000
+
+# Full dataset
+python3 scripts/train_chestmnist.py --epochs 15 --batch-size 64
+```
+
+Outputs saved to `models/chestmnist_mobilenetv3/`:
+- `mobilenetv3_chestmnist.pth` — PyTorch checkpoint
+- `mobilenetv3_chestmnist.onnx` — ONNX export
+- `training_metrics.json` — AUROC, AUPRC, F1, thresholds
+- `baseline_stats.json` — pixel reference for drift detection
+
+### 3. Run the backend
+
+```bash
+PNEUMOOPS_PROFILE=chestmnist python3 -m uvicorn backend.main:app --port 7860
+```
+
+Key endpoints:
+
+| Endpoint | Description |
+|---|---|
+| `POST /predict` | Run inference (A/B routed) |
+| `GET /health` | System status + model metadata |
+| `GET /metrics` | Prometheus scrape endpoint |
+| `GET /history` | Last 20 requests |
+| `GET /metrics/class-rates` | Per-class prediction rates |
+| `GET /metrics/calibration` | AUROC / AUPRC / Brier per class |
+
+### 4. Run the UI
+
+```bash
+BACKEND_PREDICT_URL=http://127.0.0.1:7860/predict python3 frontend/app.py
+# Open http://localhost:7861
+```
+
+### 5. Docker (full stack)
+
+```bash
+docker compose up --build
+# Backend → http://localhost:7860
+# Frontend → http://localhost:7861
+```
+
+---
+
+## Deployment — Hugging Face Spaces
+
+### Manual push
+
+```bash
+# Add HF remote
+git remote add space https://huggingface.co/spaces/Prakhar54-byte/PneumoOps
+
+# Push (Spaces will build the Docker image automatically)
+git push space main
+```
+
+### Automated (GitHub Actions)
+
+Set these repository secrets on GitHub:
+
+| Secret | Description |
+|---|---|
+| `HF_TOKEN` | Hugging Face access token (write permission) |
+| `HF_SPACE_REPO` | e.g. `your-username/pneumoops` |
+| `HF_MODEL_REPO` | *(optional)* e.g. `your-username/pneumoops-models` |
+
+Every push to `main` triggers CI checks then deploys to your Space automatically.
+
+---
+
+## Real-World MLOps Challenges Addressed
+
+| Challenge | Solution |
+|---|---|
+| Model degradation over time | Drift Monitor (KS-test on pixel distribution) |
+| Serving latency variance | A/B routing between PyTorch and ONNX, latency tracked per arm |
+| Class imbalance (rare diseases) | Per-class threshold tuning on val set + AUPRC tracking |
+| Missed diagnoses | Per-class recall monitored at `/metrics/class-rates` |
+| Production observability | Prometheus metrics — latency histograms, per-disease counters, drift alerts |
+| Automated retraining signals | `DRIFT_DETECTED` flag logged + exposed via Prometheus counter |
+
+---
+
+## Libraries
+
+`PyTorch` · `ONNX Runtime` · `FastAPI` · `Gradio` · `Docker` · `Hugging Face Hub/Spaces` · `scikit-learn` · `Prometheus` · `MedMNIST` · `SciPy`
+
+---
+
+## License
+
+MIT

backend/main.py

@@ -0,0 +1,675 @@
+import asyncio
+import io
+import json
+import logging
+import os
+import random
+import sys
+import time
+from collections import deque
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import gradio as gr
+import numpy as np
+import onnxruntime as ort
+import torch
+from fastapi import FastAPI, File, HTTPException, Request, Response, UploadFile
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import PlainTextResponse
+from PIL import Image
+from prometheus_client import CONTENT_TYPE_LATEST, Counter, Histogram, generate_latest
+from scipy.stats import ks_2samp
+from torchvision import transforms
+from torchvision.models import mobilenet_v3_small, efficientnet_b0
+
+from model_utils import CalibratedModel
+
+
+BASE_DIR = Path(__file__).resolve().parents[1]
+LEGACY_MODEL_DIR = BASE_DIR / "models"
+REALWORLD_MODEL_DIR = BASE_DIR / "models" / "realworld_efficientnet_b0"
+CHESTMNIST_MODEL_DIR = BASE_DIR / "models" / "chestmnist_mobilenetv3"
+
+PROFILE = os.getenv("PNEUMOOPS_PROFILE", "realworld").lower()
+_DEFAULT_MODEL_DIR = {
+    "realworld": REALWORLD_MODEL_DIR,
+    "chestmnist": CHESTMNIST_MODEL_DIR,
+}.get(PROFILE, LEGACY_MODEL_DIR)
+MODEL_DIR = Path(os.getenv("PNEUMOOPS_MODEL_DIR", str(_DEFAULT_MODEL_DIR)))
+REQUEST_LOG_HISTORY = deque(maxlen=20)
+
+API_KEY = os.getenv("PNEUMOOPS_API_KEY")
+ALLOWED_ORIGINS = [origin.strip() for origin in os.getenv("PNEUMOOPS_ALLOWED_ORIGINS", "*").split(",") if origin.strip()]
+TRAFFIC_WEIGHTS = {"pytorch": 60, "onnx": 40}
+LOW_CONFIDENCE_THRESHOLD = float(os.getenv("PNEUMOOPS_LOW_CONFIDENCE_THRESHOLD", "0.60"))
+MIN_UPLOAD_EDGE = int(os.getenv("PNEUMOOPS_MIN_UPLOAD_EDGE", "96"))
+MAX_CHANNEL_DELTA = float(os.getenv("PNEUMOOPS_MAX_CHANNEL_DELTA", "0.08"))
+MIN_ASPECT_RATIO = float(os.getenv("PNEUMOOPS_MIN_ASPECT_RATIO", "0.6"))
+MAX_ASPECT_RATIO = float(os.getenv("PNEUMOOPS_MAX_ASPECT_RATIO", "1.6"))
+
+REQUEST_COUNTER = Counter(
+    "pneumoops_requests_total",
+    "Total inference requests served by PneumoOps.",
+    ["model", "status"],
+)
+LATENCY_HISTOGRAM = Histogram(
+    "pneumoops_inference_latency_ms",
+    "Inference latency per model in milliseconds.",
+    ["model"],
+    buckets=(5, 10, 25, 50, 100, 250, 500, 1000),
+)
+DRIFT_COUNTER = Counter(
+    "pneumoops_drift_alerts_total",
+    "Number of drift alerts emitted by PneumoOps.",
+    ["status"],
+)
+DISEASE_PREDICTION_COUNTER = Counter(
+    "pneumoops_disease_predictions_total",
+    "Per-disease prediction counts for production monitoring.",
+    ["disease", "model"],
+)
+
+logger = logging.getLogger("pneumoops")
+if not logger.handlers:
+    logging.basicConfig(level=logging.INFO, format="%(message)s")
+
+
+def resolve_path(candidates: list[Path]) -> Path | None:
+    for candidate in candidates:
+        if candidate.exists():
+            return candidate
+    return None
+
+
+def load_json(path: Path | None, fallback: dict | None = None) -> dict:
+    fallback_dict = {} if fallback is None else fallback
+    if path and path.exists():
+        try:
+            return json.loads(path.read_text(encoding="utf-8"))
+        except json.JSONDecodeError:
+            logger.warning(f"Failed to parse JSON file {path}, using fallback.")
+            return fallback_dict
+    return fallback_dict
+
+
+def resolve_runtime_paths(model_dir: Path) -> dict[str, Path | None]:
+    checkpoint_path = resolve_path(
+        [
+            model_dir / "mobilenetv3_chestmnist.pth",   # chestmnist profile
+            model_dir / "realworld_efficientnet_b0.pth",
+            model_dir / "pneumo_model.pth",
+        ]
+    )
+    onnx_report_path = resolve_path([model_dir / "onnx_export_report.json", LEGACY_MODEL_DIR / "onnx_export_report.json"])
+    onnx_report = load_json(onnx_report_path)
+
+    base_onnx = onnx_report.get("base_onnx")
+    optimized_onnx = onnx_report.get("optimized_onnx")
+    serving_onnx = onnx_report.get("serving_onnx")
+
+    onnx_candidates = []
+    if serving_onnx:
+        onnx_candidates.append(model_dir / serving_onnx)
+    if optimized_onnx:
+        onnx_candidates.append(model_dir / optimized_onnx)
+    if base_onnx:
+        onnx_candidates.append(model_dir / base_onnx)
+    onnx_candidates.extend(
+        [
+            # chestmnist profile
+            model_dir / "mobilenetv3_chestmnist.onnx",
+            # realworld profile
+            model_dir / "realworld_efficientnet_b0_quantized.onnx",
+            model_dir / "realworld_efficientnet_b0_optimized.onnx",
+            model_dir / "realworld_efficientnet_b0.onnx",
+            model_dir / "pneumo_model_quantized.onnx",
+            model_dir / "pneumo_model_optimized.onnx",
+            model_dir / "pneumo_model.onnx",
+        ]
+    )
+
+    return {
+        "checkpoint": checkpoint_path,
+        "onnx": resolve_path(onnx_candidates),
+        "training_metrics": resolve_path([model_dir / "training_metrics.json", LEGACY_MODEL_DIR / "training_metrics.json"]),
+        "baseline_stats": resolve_path([model_dir / "baseline_stats.json", LEGACY_MODEL_DIR / "baseline_stats.json"]),
+        "onnx_export_report": onnx_report_path,
+    }
+
+
+RUNTIME_PATHS = resolve_runtime_paths(MODEL_DIR)
+
+
+def load_checkpoint_metadata(checkpoint_path: Path | None) -> dict[str, Any]:
+    if checkpoint_path is None or not checkpoint_path.exists():
+        return {
+            "architecture": "mobilenet_v3_small",
+            "class_names": ["Normal", "Pneumonia"],
+            "image_size": 224,
+            "normalize_mean": [0.485, 0.456, 0.406],
+            "normalize_std": [0.229, 0.224, 0.225],
+            "thresholds": [0.5, 0.5],
+            "logit_temperature": 1.0,
+            "multi_label": False,
+        }
+
+    payload = torch.load(checkpoint_path, map_location="cpu")
+    if isinstance(payload, dict) and "model_state_dict" in payload:
+        return {
+            "state_dict": payload["model_state_dict"],
+            "architecture": payload.get("architecture", "efficientnet_b0"),
+            "class_names": payload.get("class_names", ["No Finding"]),
+            "image_size": int(payload.get("image_size", 320)),
+            "normalize_mean": payload.get("normalize_mean", [0.485, 0.456, 0.406]),
+            "normalize_std": payload.get("normalize_std", [0.229, 0.224, 0.225]),
+            "thresholds": payload.get("thresholds", [0.5] * len(payload.get("class_names", ["No Finding"]))),
+            "logit_temperature": float(payload.get("logit_temperature", 1.0)),
+            "multi_label": bool(payload.get("multi_label", True)),
+        }
+
+    # Plain state_dict (e.g. from train_chestmnist.py) — pull metadata from training_metrics.json
+    tm_path = MODEL_DIR / "training_metrics.json"
+    tm = load_json(tm_path)
+    class_names = tm.get("class_names", ["Normal", "Pneumonia"])
+    n = len(class_names)
+    return {
+        "state_dict": payload,
+        "architecture": tm.get("architecture", "mobilenet_v3_small"),
+        "class_names": class_names,
+        "image_size": int(tm.get("image_size", 224)),
+        "normalize_mean": [0.5, 0.5, 0.5],
+        "normalize_std": [0.5, 0.5, 0.5],
+        "thresholds": tm.get("thresholds", [0.5] * n),
+        "logit_temperature": 1.0,
+        "multi_label": bool(tm.get("multi_label", True)),
+    }
+
+
+
+MODEL_METADATA = load_checkpoint_metadata(RUNTIME_PATHS["checkpoint"])
+CLASS_NAMES = MODEL_METADATA["class_names"]
+MULTI_LABEL = bool(MODEL_METADATA["multi_label"])
+IMAGE_SIZE = int(MODEL_METADATA["image_size"])
+THRESHOLDS = np.asarray(MODEL_METADATA["thresholds"], dtype=np.float32)
+LOGIT_TEMPERATURE = float(MODEL_METADATA.get("logit_temperature", 1.0))
+
+TRANSFORM = transforms.Compose(
+    [
+        transforms.Resize((IMAGE_SIZE, IMAGE_SIZE)),
+        transforms.Grayscale(num_output_channels=3),
+        transforms.ToTensor(),
+        transforms.Normalize(
+            mean=MODEL_METADATA["normalize_mean"],
+            std=MODEL_METADATA["normalize_std"],
+        ),
+    ]
+)
+
+BASELINE_STATS = load_json(
+    RUNTIME_PATHS["baseline_stats"],
+    fallback={
+        "pixel_mean_mean": 0.5,
+        "pixel_mean_std": 0.1,
+        "pixel_std_mean": 0.2,
+        "pixel_std_std": 0.05,
+        "histogram_bins": 32,
+        "histogram_mean": [1.0 / 32.0] * 32,
+        "pixel_reference_sample": [],
+        "drift_threshold": 1.2,
+        "drift_ks_pvalue_threshold": 0.05,
+        "class_names": CLASS_NAMES,
+    },
+)
+
+
+def build_model() -> torch.nn.Module | None:
+    checkpoint_path = RUNTIME_PATHS["checkpoint"]
+    if checkpoint_path is None or not checkpoint_path.exists():
+        return None
+
+    architecture = MODEL_METADATA["architecture"]
+    num_outputs = len(CLASS_NAMES)
+    if architecture == "efficientnet_b0":
+        base_model = efficientnet_b0(weights=None)
+        base_model.classifier[1] = torch.nn.Linear(base_model.classifier[1].in_features, num_outputs)
+    else:
+        base_model = mobilenet_v3_small(weights=None)
+        base_model.classifier[3] = torch.nn.Linear(base_model.classifier[3].in_features, num_outputs)
+
+    base_model.load_state_dict(MODEL_METADATA["state_dict"])
+    model = CalibratedModel(base_model, LOGIT_TEMPERATURE)
+    model.to(torch.device("cuda" if torch.cuda.is_available() else "cpu"))
+    model.eval()
+    return model
+
+
+def build_onnx_session():
+    onnx_path = RUNTIME_PATHS["onnx"]
+    if onnx_path is None or not onnx_path.exists():
+        return None, None
+    session = ort.InferenceSession(str(onnx_path), providers=["CPUExecutionProvider"])
+    return session, onnx_path.name
+
+
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+PYTORCH_MODEL = build_model()
+ONNX_SESSION, ACTIVE_ONNX_MODEL_NAME = build_onnx_session()
+
+app = FastAPI(
+    title="PneumoOps API",
+    description="Production-style MLOps pipeline for medical image classification with A/B routing and drift monitoring.",
+)
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=ALLOWED_ORIGINS,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.middleware("http")
+async def auth_middleware(request: Request, call_next):
+    if API_KEY and request.url.path not in {"/health", "/metrics", "/"}:
+        provided = request.headers.get("x-api-key")
+        if provided != API_KEY:
+            return Response(content="Unauthorized", status_code=401)
+    return await call_next(request)
+
+
+def load_image_from_upload(upload: UploadFile) -> Image.Image:
+    try:
+        content = upload.file.read()
+        return Image.open(io.BytesIO(content)).convert("RGB")
+    except Exception as exc:
+        raise HTTPException(status_code=400, detail="Uploaded file is not a valid image.") from exc
+
+
+def summarize_image(image: Image.Image) -> dict[str, Any]:
+    gray = np.asarray(image.convert("L"), dtype=np.float32) / 255.0
+    width, height = image.size
+    rgb = np.asarray(image.convert("RGB"), dtype=np.float32) / 255.0
+    channel_delta = float(
+        np.mean(np.abs(rgb[:, :, 0] - rgb[:, :, 1]))
+        + np.mean(np.abs(rgb[:, :, 1] - rgb[:, :, 2]))
+        + np.mean(np.abs(rgb[:, :, 0] - rgb[:, :, 2]))
+    ) / 3.0
+    return {
+        "width": width,
+        "height": height,
+        "aspect_ratio": round(width / max(height, 1), 4),
+        "pixel_mean": round(float(np.mean(gray)), 6),
+        "pixel_std": round(float(np.std(gray)), 6),
+        "pixel_min": round(float(np.min(gray)), 6),
+        "pixel_max": round(float(np.max(gray)), 6),
+        "channel_delta": round(channel_delta, 6),
+    }
+
+
+def validate_image(image: Image.Image, summary: dict[str, Any]) -> None:
+    if min(summary["width"], summary["height"]) < MIN_UPLOAD_EDGE:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Input is too small for robust X-ray screening. Minimum edge is {MIN_UPLOAD_EDGE}px.",
+        )
+    if not (MIN_ASPECT_RATIO <= summary["aspect_ratio"] <= MAX_ASPECT_RATIO):
+        raise HTTPException(
+            status_code=400,
+            detail="Input aspect ratio is outside the expected chest X-ray range.",
+        )
+    if summary["channel_delta"] > MAX_CHANNEL_DELTA:
+        raise HTTPException(
+            status_code=400,
+            detail="Input appears to be a color image instead of a grayscale-style radiograph.",
+        )
+
+
+def calculate_drift(image: Image.Image) -> dict[str, Any]:
+    gray = np.asarray(image.convert("L"), dtype=np.float32) / 255.0
+    image_mean = float(np.mean(gray))
+    image_std = float(np.std(gray))
+    hist_bins = int(BASELINE_STATS.get("histogram_bins", 32))
+    hist, _ = np.histogram(gray, bins=hist_bins, range=(0.0, 1.0), density=True)
+    baseline_hist = np.asarray(BASELINE_STATS.get("histogram_mean", [1.0 / hist_bins] * hist_bins), dtype=np.float32)
+
+    mean_z = abs(image_mean - BASELINE_STATS.get("pixel_mean_mean", 0.5)) / max(BASELINE_STATS.get("pixel_mean_std", 0.1), 1e-6)
+    std_z = abs(image_std - BASELINE_STATS.get("pixel_std_mean", 0.2)) / max(BASELINE_STATS.get("pixel_std_std", 0.05), 1e-6)
+    hist_distance = float(np.mean(np.abs(hist - baseline_hist)))
+    drift_score = round(0.35 * mean_z + 0.35 * std_z + 0.30 * hist_distance, 6)
+
+    reference_sample = np.asarray(BASELINE_STATS.get("pixel_reference_sample", []), dtype=np.float32)
+    incoming_sample = gray.reshape(-1)
+    if reference_sample.size > 0:
+        sample_size = min(len(incoming_sample), len(reference_sample), 4096)
+        incoming_idx = np.random.choice(len(incoming_sample), size=sample_size, replace=False)
+        reference_idx = np.random.choice(len(reference_sample), size=sample_size, replace=False)
+        _, ks_pvalue = ks_2samp(incoming_sample[incoming_idx], reference_sample[reference_idx])
+        ks_pvalue = float(ks_pvalue)
+    else:
+        ks_pvalue = 1.0
+
+    drift_detected = ks_pvalue < float(BASELINE_STATS.get("drift_ks_pvalue_threshold", 0.05)) or drift_score > float(
+        BASELINE_STATS.get("drift_threshold", 1.2)
+    )
+    return {
+        "drift_alert": "DRIFT_DETECTED" if drift_detected else "NORMAL",
+        "drift_score": drift_score,
+        "ks_pvalue": round(ks_pvalue, 6),
+        "mean_z": round(float(mean_z), 6),
+        "std_z": round(float(std_z), 6),
+        "histogram_distance": round(hist_distance, 6),
+    }
+
+
+def postprocess_probabilities(probabilities: np.ndarray) -> dict[str, Any]:
+    probabilities = probabilities.astype(np.float32)
+    if MULTI_LABEL:
+        predicted_indices = [index for index, value in enumerate(probabilities) if value >= THRESHOLDS[index]]
+        if not predicted_indices:
+            predicted_indices = [int(np.argmax(probabilities))]
+        predicted_labels = [CLASS_NAMES[index] for index in predicted_indices]
+    else:
+        predicted_index = int(np.argmax(probabilities))
+        predicted_indices = [predicted_index]
+        predicted_labels = [CLASS_NAMES[predicted_index]]
+
+    sorted_pairs = sorted(
+        [
+            {
+                "label": CLASS_NAMES[index],
+                "confidence": round(float(probabilities[index]) * 100, 2),
+                "threshold": round(float(THRESHOLDS[index]) * 100, 2) if index < len(THRESHOLDS) else 50.0,
+            }
+            for index in range(len(CLASS_NAMES))
+        ],
+        key=lambda item: item["confidence"],
+        reverse=True,
+    )
+    top_confidence = sorted_pairs[0]["confidence"] if sorted_pairs else 0.0
+    return {
+        "predicted_labels": predicted_labels,
+        "top_predictions": sorted_pairs[: min(5, len(sorted_pairs))],
+        "max_confidence": top_confidence,
+        "low_confidence": top_confidence < (LOW_CONFIDENCE_THRESHOLD * 100.0),
+    }
+
+
+def run_pytorch_inference(image: Image.Image) -> dict[str, Any]:
+    if PYTORCH_MODEL is None:
+        raise RuntimeError("PyTorch checkpoint is missing.")
+
+    tensor = TRANSFORM(image).unsqueeze(0).to(DEVICE)
+    start = time.perf_counter()
+    with torch.no_grad():
+        logits = PYTORCH_MODEL(tensor)
+        probabilities = torch.sigmoid(logits).squeeze(0).cpu().numpy() if MULTI_LABEL else torch.softmax(logits, dim=1).squeeze(0).cpu().numpy()
+    latency_ms = round((time.perf_counter() - start) * 1000, 2)
+    return {
+        "model_key": "pytorch",
+        "model_used": "Baseline PyTorch",
+        "latency_ms": latency_ms,
+        "probabilities": probabilities.tolist(),
+        **postprocess_probabilities(probabilities),
+    }
+
+
+def run_onnx_inference(image: Image.Image) -> dict[str, Any]:
+    if ONNX_SESSION is None:
+        raise RuntimeError("ONNX artifact is missing.")
+
+    tensor = TRANSFORM(image).unsqueeze(0).numpy().astype(np.float32)
+    input_name = ONNX_SESSION.get_inputs()[0].name
+    start = time.perf_counter()
+    outputs = ONNX_SESSION.run(None, {input_name: tensor})
+    latency_ms = round((time.perf_counter() - start) * 1000, 2)
+    logits = torch.from_numpy(outputs[0]).squeeze(0)
+    probabilities = torch.sigmoid(logits).numpy() if MULTI_LABEL else torch.softmax(logits, dim=0).numpy()
+    return {
+        "model_key": "onnx",
+        "model_used": "Optimized ONNX",
+        "latency_ms": latency_ms,
+        "probabilities": probabilities.tolist(),
+        **postprocess_probabilities(probabilities),
+    }
+
+
+async def benchmark_both_models(image: Image.Image) -> dict[str, Any]:
+    async def safe_call(model_name: str, fn):
+        try:
+            result = await asyncio.to_thread(fn, image)
+            LATENCY_HISTOGRAM.labels(model=model_name).observe(result["latency_ms"])
+            return result
+        except Exception as exc:
+            return {"model_key": model_name, "error": str(exc)}
+
+    pytorch_result, onnx_result = await asyncio.gather(
+        safe_call("pytorch", run_pytorch_inference),
+        safe_call("onnx", run_onnx_inference),
+    )
+    return {"pytorch": pytorch_result, "onnx": onnx_result}
+
+
+def build_recommendation(selected_result: dict[str, Any], drift_result: dict[str, Any], dual_results: dict[str, Any]) -> str:
+    if drift_result["drift_alert"] == "DRIFT_DETECTED":
+        return "Input distribution differs from the stored training baseline. Manual review is recommended before trusting this result."
+    if selected_result["low_confidence"]:
+        return "Prediction confidence is below the review threshold. Treat this as low confidence and escalate for human review."
+
+    other_key = "onnx" if selected_result["model_key"] == "pytorch" else "pytorch"
+    other_result = dual_results.get(other_key, {})
+    if other_result.get("predicted_labels") and other_result["predicted_labels"] != selected_result["predicted_labels"]:
+        return "The two serving paths disagree on the predicted findings. Use this as a monitoring alert and fall back to manual review."
+
+    return "Use this output as a triage aid only. PneumoOps monitors latency and drift, but it is not a clinical decision-maker."
+
+
+def append_history(entry: dict[str, Any]) -> None:
+    REQUEST_LOG_HISTORY.append(entry)
+
+
+def emit_structured_log(payload: dict[str, Any]) -> None:
+    logger.info(json.dumps(payload, ensure_ascii=True))
+
+
+@app.get("/health")
+def health():
+    return {
+        "status": "ok",
+        "profile": PROFILE,
+        "model_dir": str(MODEL_DIR),
+        "pytorch_model_loaded": PYTORCH_MODEL is not None,
+        "onnx_model_loaded": ONNX_SESSION is not None,
+        "active_onnx_model": ACTIVE_ONNX_MODEL_NAME,
+        "class_count": len(CLASS_NAMES),
+        "class_names": CLASS_NAMES,
+        "multi_label": MULTI_LABEL,
+        "logit_temperature": LOGIT_TEMPERATURE,
+        "training_metrics": load_json(RUNTIME_PATHS["training_metrics"]),
+        "onnx_export_report": load_json(RUNTIME_PATHS["onnx_export_report"]),
+        "recent_requests": list(REQUEST_LOG_HISTORY),
+    }
+
+
+@app.get("/metrics")
+def metrics():
+    return PlainTextResponse(generate_latest(), media_type=CONTENT_TYPE_LATEST)
+
+
+@app.get("/history")
+def history():
+    return {"recent_requests": list(REQUEST_LOG_HISTORY)}
+
+
+@app.get("/metrics/class-rates")
+def class_prediction_rates():
+    """Return per-class prediction rates from the rolling request history."""
+    history_list = list(REQUEST_LOG_HISTORY)
+    total = max(len(history_list), 1)
+    rates: dict[str, float] = {label: 0.0 for label in CLASS_NAMES}
+    avg_confidence: dict[str, list] = {label: [] for label in CLASS_NAMES}
+
+    for entry in history_list:
+        per_class = entry.get("per_class_predictions", {})
+        per_class_probs = entry.get("per_class_probabilities", {})
+        for label in CLASS_NAMES:
+            if per_class.get(label, False):
+                rates[label] = rates[label] + 1
+            if label in per_class_probs:
+                avg_confidence[label].append(per_class_probs[label])
+
+    return {
+        "window_size": total,
+        "per_class_prediction_rate": {
+            label: round(count / total, 4)
+            for label, count in rates.items()
+        },
+        "per_class_avg_confidence": {
+            label: round(float(sum(vals) / len(vals)), 4) if vals else None
+            for label, vals in avg_confidence.items()
+        },
+        "drift_rate": round(
+            sum(1 for e in history_list if e.get("drift_alert") == "DRIFT_DETECTED") / total, 4
+        ),
+    }
+
+
+@app.get("/metrics/calibration")
+def calibration_summary():
+    """Return per-class calibration (Brier scores) from training metrics."""
+    tm = load_json(RUNTIME_PATHS["training_metrics"])
+    return {
+        "test_macro_brier": tm.get("test_macro_brier"),
+        "test_macro_auprc": tm.get("test_macro_auprc"),
+        "test_macro_roc_auc": tm.get("test_macro_roc_auc"),
+        "per_class_brier": tm.get("per_class_brier", {}),
+        "per_class_auprc": tm.get("per_class_auprc", {}),
+        "per_class_roc_auc": tm.get("per_class_roc_auc", {}),
+        "per_class_recall": tm.get("per_class_recall", {}),
+        "threshold_details": tm.get("threshold_details", {}),
+        "class_names": tm.get("class_names", CLASS_NAMES),
+    }
+
+
+@app.post("/predict")
+async def predict(request: Request, file: UploadFile = File(...)):
+    image = load_image_from_upload(file)
+    input_summary = summarize_image(image)
+    validate_image(image, input_summary)
+
+    start = time.perf_counter()
+    dual_results = await benchmark_both_models(image)
+    selected_key = random.choices(["pytorch", "onnx"], weights=[TRAFFIC_WEIGHTS["pytorch"], TRAFFIC_WEIGHTS["onnx"]], k=1)[0]
+    selected_result = dual_results[selected_key]
+    warning_flags = []
+
+    if "error" in selected_result:
+        fallback_result = dual_results["pytorch"]
+        if "error" in fallback_result:
+            REQUEST_COUNTER.labels(model=selected_key, status="failure").inc()
+            raise HTTPException(status_code=503, detail=f"Both inference backends failed: {dual_results}")
+        selected_result = fallback_result
+        warning_flags.append(f"{selected_key.upper()} failed, fallback to PyTorch.")
+
+    # Increment per-class disease prediction counters
+    for label in selected_result["predicted_labels"]:
+        DISEASE_PREDICTION_COUNTER.labels(disease=label, model=selected_result["model_key"]).inc()
+
+    drift_result = calculate_drift(image)
+    DRIFT_COUNTER.labels(status=drift_result["drift_alert"]).inc()
+    REQUEST_COUNTER.labels(model=selected_result["model_key"], status="success").inc()
+
+    pytorch_latency = dual_results["pytorch"].get("latency_ms") if "error" not in dual_results["pytorch"] else None
+    onnx_latency = dual_results["onnx"].get("latency_ms") if "error" not in dual_results["onnx"] else None
+    latency_delta = None
+    if pytorch_latency is not None and onnx_latency is not None:
+        latency_delta = round(float(onnx_latency) - float(pytorch_latency), 2)
+
+    timestamp = datetime.now(timezone.utc).isoformat()
+    response_payload = {
+        "timestamp": timestamp,
+        "selected_model": selected_result["model_used"],
+        "selected_arm": "A" if selected_result["model_key"] == "pytorch" else "B",
+        "weighted_traffic_split": TRAFFIC_WEIGHTS,
+        "predicted_labels": selected_result["predicted_labels"],
+        "top_predictions": selected_result["top_predictions"],
+        "confidence": selected_result["max_confidence"],
+        "low_confidence": selected_result["low_confidence"],
+        "confidence_review_threshold": LOW_CONFIDENCE_THRESHOLD * 100.0,
+        "pytorch_latency_ms": pytorch_latency,
+        "onnx_latency_ms": onnx_latency,
+        "latency_delta_ms": latency_delta,
+        "drift": drift_result,
+        "input_summary": input_summary,
+        "warning_flags": warning_flags,
+        "recommendation": build_recommendation(selected_result, drift_result, dual_results),
+        "active_onnx_model": ACTIVE_ONNX_MODEL_NAME,
+        "recent_history": list(REQUEST_LOG_HISTORY),
+    }
+
+    history_entry = {
+        "timestamp": timestamp,
+        "model": selected_result["model_used"],
+        "latency_ms": selected_result["latency_ms"],
+        "drift_alert": drift_result["drift_alert"],
+        "drift_score": drift_result["drift_score"],
+        "confidence": selected_result["max_confidence"],
+        "labels": selected_result["predicted_labels"],
+        "per_class_probabilities": {
+            CLASS_NAMES[i]: round(float(selected_result["probabilities"][i]), 4)
+            for i in range(len(CLASS_NAMES))
+            if i < len(selected_result.get("probabilities", []))
+        },
+        "per_class_predictions": {
+            label: (label in selected_result["predicted_labels"])
+            for label in CLASS_NAMES
+        },
+    }
+    append_history(history_entry)
+    response_payload["recent_history"] = list(REQUEST_LOG_HISTORY)
+
+    emit_structured_log(
+        {
+            "event": "predict",
+            "timestamp": timestamp,
+            "model_used": selected_result["model_used"],
+            "latency_ms": selected_result["latency_ms"],
+            "confidence": selected_result["max_confidence"],
+            "drift_status": drift_result["drift_alert"],
+            "pathologies": selected_result["predicted_labels"],
+            "client": request.client.host if request.client else None,
+        }
+    )
+
+    response_payload["request_latency_ms"] = round((time.perf_counter() - start) * 1000, 2)
+    return response_payload
+
+
+# ─── Mount Gradio UI into FastAPI (single-port for HF Spaces) ───────────────
+# This allows the entire app (API + UI) to run on one port (7860).
+# - FastAPI REST endpoints remain at /predict, /health, /metrics, etc.
+# - Gradio UI is served at / (root)
+
+try:
+    # Dynamically add frontend dir to path so app.py can be imported
+    _frontend_dir = str(BASE_DIR / "frontend")
+    if _frontend_dir not in sys.path:
+        sys.path.insert(0, _frontend_dir)
+
+    import gradio as gr
+    from frontend.app import demo as gradio_demo  # the gr.Blocks() object
+
+    # Mount Gradio at root; FastAPI routes take priority because they're
+    # registered first via @app.get / @app.post decorators.
+    app = gr.mount_gradio_app(app, gradio_demo, path="/ui")
+    logger.info("Gradio UI mounted at /ui — full app on single port.")
+except Exception as _e:
+    logger.warning(f"Gradio mount skipped ({_e}). API-only mode active.")
+
+
+if __name__ == "__main__":
+    import uvicorn
+    port = int(os.getenv("PORT", "7860"))
+    uvicorn.run(app, host="0.0.0.0", port=port)
+

docker-compose.yml

@@ -0,0 +1,30 @@
+version: "3.9"
+
+services:
+  # ─── Single container: FastAPI + Gradio on port 7860 ─────────────────────
+  # This mirrors the HF Spaces deployment (single exposed port).
+  # Access:
+  #   UI  →  http://localhost:7860/ui
+  #   API →  http://localhost:7860/predict
+  #   Health → http://localhost:7860/health
+  app:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    ports:
+      - "7860:7860"
+    volumes:
+      - ./models:/app/models   # hot-swap model files without rebuilding
+    environment:
+      - PYTHONPATH=/app
+      - PNEUMOOPS_PROFILE=chestmnist
+      - PORT=7860
+      # Optional: set PNEUMOOPS_API_KEY to protect /predict in production
+      # - PNEUMOOPS_API_KEY=change-me
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://127.0.0.1:7860/health"]
+      interval: 30s
+      timeout: 5s
+      start_period: 40s
+      retries: 3

frontend/app.py

@@ -0,0 +1,235 @@
+"""
+PneumoOps — Simple Assignment Demo UI (Task 1 Spec)
+====================================================
+Matches the exact Gradio UI requirements from ASSIGNMENT_TARGET.md:
+  1. Image upload
+  2. Top-3 predictions — bar chart
+  3. Model Used — "Baseline PyTorch" or "Optimized ONNX"
+  4. Inference Latency (ms)
+  5. Drift Alert — "Normal" or "Drift Detected" (color badge)
+
+Start independently of the main dashboard:
+  python3 frontend/app_simple.py
+"""
+
+import io
+import os
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import matplotlib.patches as mpatches
+import gradio as gr
+import numpy as np
+import requests
+from PIL import Image
+
+# When running embedded inside FastAPI (HF Spaces), the backend is on the same process.
+# When running standalone via docker-compose, override with BACKEND_PREDICT_URL env var.
+BACKEND_URL = os.getenv("BACKEND_PREDICT_URL", "http://127.0.0.1:7860/predict")
+
+CHESTMNIST_CLASSES = [
+    "Atelectasis", "Cardiomegaly", "Effusion", "Infiltration",
+    "Mass", "Nodule", "Pneumonia", "Pneumothorax",
+    "Consolidation", "Edema", "Emphysema", "Fibrosis",
+    "Pleural_Thickening", "Hernia",
+]
+
+
+# ---------------------------------------------------------------------------
+# Backend call
+# ---------------------------------------------------------------------------
+
+def call_backend(image: Image.Image, api_url: str) -> dict:
+    buf = io.BytesIO()
+    image.convert("RGB").save(buf, format="PNG")
+    buf.seek(0)
+    resp = requests.post(
+        api_url,
+        files={"file": ("xray.png", buf.getvalue(), "image/png")},
+        timeout=60,
+    )
+    resp.raise_for_status()
+    return resp.json()
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+_TIER_COLORS = {
+    "Pneumonia": "#dc2626",
+    "Pneumothorax": "#dc2626",
+    "Mass": "#dc2626",
+    "Nodule": "#dc2626",
+    "Effusion": "#f59e0b",
+    "Cardiomegaly": "#f59e0b",
+    "Consolidation": "#f59e0b",
+}
+
+def _bar_color(label: str) -> str:
+    return _TIER_COLORS.get(label, "#6366f1")
+
+
+def build_top3_chart(top_predictions: list[dict]) -> object:
+    """Horizontal bar chart of top-3 predicted pathologies."""
+    top3 = sorted(top_predictions, key=lambda x: x["confidence"], reverse=True)[:3]
+    if not top3:
+        fig, ax = plt.subplots(figsize=(7, 2))
+        ax.set_title("No pathologies detected above threshold")
+        ax.axis("off")
+        return fig
+
+    labels = [p["label"] for p in top3]
+    values = [p["confidence"] / 100.0 for p in top3]
+    colors = [_bar_color(l) for l in labels]
+
+    fig, ax = plt.subplots(figsize=(8, 3.2))
+    y = np.arange(len(labels))
+    ax.barh(y, values, color=colors, height=0.55, zorder=3)
+    ax.set_yticks(y)
+    ax.set_yticklabels(labels, fontsize=13, fontweight="bold")
+    ax.set_xlim(0, 1.15)
+    ax.set_xlabel("Confidence Score", fontsize=11)
+    ax.set_title("Top-3 Predicted Pathologies", fontsize=13, fontweight="bold", pad=10)
+    ax.grid(axis="x", alpha=0.25, zorder=0)
+
+    for i, v in enumerate(values):
+        ax.text(v + 0.02, i, f"{v:.0%}", va="center", fontsize=11, fontweight="bold")
+
+    legend_patches = [
+        mpatches.Patch(color="#dc2626", label="Critical"),
+        mpatches.Patch(color="#f59e0b", label="Significant"),
+        mpatches.Patch(color="#6366f1", label="Standard"),
+    ]
+    ax.legend(handles=legend_patches, loc="lower right", fontsize=9)
+
+    fig.patch.set_facecolor("#f8fafc")
+    ax.set_facecolor("#f1f5f9")
+    plt.tight_layout()
+    return fig
+
+
+def drift_badge(status: str) -> str:
+    if status == "DRIFT_DETECTED":
+        return (
+            "<div style='padding:0.8rem 1.2rem;border-radius:12px;font-size:1rem;"
+            "font-weight:700;background:#fef3f2;color:#b42318;border:2px solid #fca5a5;'>"
+            "⚠️ DRIFT DETECTED — Out-of-distribution input. Consider retraining.</div>"
+        )
+    return (
+        "<div style='padding:0.8rem 1.2rem;border-radius:12px;font-size:1rem;"
+        "font-weight:700;background:#ecfdf5;color:#065f46;border:2px solid #6ee7b7;'>"
+        "✅ NORMAL — Input distribution matches training baseline.</div>"
+    )
+
+
+# ---------------------------------------------------------------------------
+# Main predict function
+# ---------------------------------------------------------------------------
+
+def predict(image: Image.Image, api_url: str):
+    if image is None:
+        raise gr.Error("Please upload a chest X-ray image.")
+
+    try:
+        payload = call_backend(image, api_url)
+    except requests.HTTPError as exc:
+        detail = exc.response.text if exc.response is not None else str(exc)
+        raise gr.Error(f"Backend error: {detail}") from exc
+    except requests.RequestException as exc:
+        raise gr.Error(f"Cannot reach backend at {api_url} — is the FastAPI server running?") from exc
+
+    top_preds   = payload.get("top_predictions", [])
+    model_key   = payload.get("selected_arm", payload.get("selected_model", "onnx"))
+    latency_ms  = payload.get("latency_delta_ms") or payload.get("request_latency_ms", "n/a")
+    drift_status = payload.get("drift", {}).get("drift_alert", "UNKNOWN")
+
+    model_label = "Optimized ONNX" if "onnx" in str(model_key).lower() else "Baseline PyTorch"
+
+    chart = build_top3_chart(top_preds)
+    findings_list = ", ".join(payload.get("predicted_labels", [])) or "No findings above threshold"
+
+    return (
+        chart,
+        findings_list,
+        model_label,
+        f"{latency_ms} ms",
+        drift_badge(drift_status),
+    )
+
+
+# ---------------------------------------------------------------------------
+# Gradio UI
+# ---------------------------------------------------------------------------
+
+CSS = """
+.hero {
+    background: linear-gradient(135deg, #0f172a 0%, #1e1b4b 60%, #2563eb 100%);
+    padding: 1.5rem 2rem;
+    border-radius: 18px;
+    color: white;
+    margin-bottom: 1rem;
+}
+.hero h1 { margin: 0 0 0.4rem; font-size: 2rem; letter-spacing: -1px; }
+.hero p  { margin: 0; opacity: 0.85; font-size: 0.95rem; line-height: 1.6; }
+"""
+
+with gr.Blocks(
+    theme=gr.themes.Soft(),
+    css=CSS,
+    title="PneumoOps — A/B Testing MLOps Pipeline",
+) as demo:
+
+    gr.HTML("""
+        <div class="hero">
+          <h1>🫁 PneumoOps</h1>
+          <p>
+            <strong>MLOps A/B Testing Pipeline</strong> for 14-class thoracic disease screening.<br/>
+            Every request is randomly routed to <strong>Model A (Baseline PyTorch)</strong> or
+            <strong>Model B (Optimized ONNX)</strong>, enabling real-world latency benchmarking.<br/>
+            A built-in <strong>Data Drift Monitor</strong> flags out-of-distribution inputs —
+            simulating automated retraining triggers in production.
+          </p>
+        </div>
+    """)
+
+    with gr.Accordion("⚙️ Backend Settings", open=False):
+        api_url = gr.Textbox(label="Backend URL", value=BACKEND_URL)
+
+    with gr.Row():
+        image_input = gr.Image(type="pil", label="Upload Chest X-Ray", height=380)
+
+    submit_btn = gr.Button("🔬 Run Screening", variant="primary", size="lg")
+
+    gr.Markdown("---")
+    gr.Markdown("## Results")
+
+    with gr.Row():
+        model_used   = gr.Textbox(label="Model Used (A/B Arm)", interactive=False, scale=1)
+        latency_out  = gr.Textbox(label="Inference Latency",     interactive=False, scale=1)
+
+    findings_out = gr.Textbox(label="All Findings Detected", interactive=False)
+    drift_out    = gr.HTML(label="Data Drift Alert")
+    top3_chart   = gr.Plot(label="Top-3 Predicted Pathologies")
+
+    gr.Markdown("""
+    ---
+    **14 Classes:** Atelectasis · Cardiomegaly · Effusion · Infiltration · Mass · Nodule ·
+    Pneumonia · Pneumothorax · Consolidation · Edema · Emphysema · Fibrosis ·
+    Pleural Thickening · Hernia
+
+    🔴 Critical · 🟠 Significant · 🟣 Standard
+    """)
+
+    submit_btn.click(
+        fn=predict,
+        inputs=[image_input, api_url],
+        outputs=[top3_chart, findings_out, model_used, latency_out, drift_out],
+    )
+
+
+if __name__ == "__main__":
+    port = int(os.getenv("GRADIO_PORT", "7860"))
+    demo.launch(server_name="0.0.0.0", server_port=port)
+

model_utils.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+import torch
+import torch.nn as nn
+
+
+def apply_temperature_to_logits(logits: torch.Tensor, temperature: float | torch.Tensor) -> torch.Tensor:
+    if isinstance(temperature, torch.Tensor):
+        temperature_tensor = temperature.to(device=logits.device, dtype=logits.dtype)
+    else:
+        temperature_tensor = torch.tensor(float(temperature), device=logits.device, dtype=logits.dtype)
+    return logits / torch.clamp(temperature_tensor, min=1e-3)
+
+
+class CalibratedModel(nn.Module):
+    def __init__(self, base_model: nn.Module, temperature: float):
+        super().__init__()
+        self.base_model = base_model
+        self.register_buffer("temperature", torch.tensor(float(temperature), dtype=torch.float32))
+
+    def forward(self, inputs: torch.Tensor) -> torch.Tensor:
+        logits = self.base_model(inputs)
+        return apply_temperature_to_logits(logits, self.temperature)

models/chestmnist_mobilenetv3/README.md

@@ -0,0 +1,47 @@
+---
+license: mit
+tags:
+  - medical
+  - image-classification
+  - chest-xray
+  - mlops
+  - onnx
+  - pytorch
+  - mobilenetv3
+datasets:
+  - medmnist/chestmnist
+metrics:
+  - roc_auc
+---
+
+# PneumoOps — ChestMNIST MobileNetV3-small
+
+This repository contains **two model versions** for the PneumoOps MLOps pipeline:
+- **Model A (Baseline):** `mobilenetv3_chestmnist.pth` — Standard PyTorch checkpoint
+- **Model B (Optimized):** `mobilenetv3_chestmnist.onnx` — ONNX-exported for faster inference
+
+Both models are identical in architecture (MobileNetV3-small) and weights.
+The ONNX version is used for inference time optimization in A/B testing.
+
+## Dataset
+**ChestMNIST** — 14-class multi-label chest X-ray classification  
+78,468 training images, 224×224 pixels, grayscale (converted to 3-channel).
+
+## Classes (14)
+Atelectasis, Cardiomegaly, Effusion, Infiltration, Mass, Nodule, Pneumonia,
+Pneumothorax, Consolidation, Edema, Emphysema, Fibrosis, Pleural Thickening, Hernia
+
+## Performance (Test Set)
+| Metric | Score |
+|--------|-------|
+| Macro AUROC | **0.808** |
+| Macro AUPRC | 0.210 |
+| Micro F1 | 0.343 |
+
+## Usage in PneumoOps
+These artifacts are loaded by the FastAPI backend and selected via a weighted A/B router:
+- 60% of requests → PyTorch model
+- 40% of requests → ONNX model
+
+The backend also computes a drift score using `baseline_stats.json` to detect
+out-of-distribution inputs in real time.

models/chestmnist_mobilenetv3/baseline_stats.json

@@ -0,0 +1,15 @@
+{
+  "pixel_mean": -0.010379456914961338,
+  "pixel_std": 0.5114685297012329,
+  "channel_means": [
+    -0.010379449464380741,
+    -0.010379449464380741,
+    -0.010379449464380741
+  ],
+  "channel_stds": [
+    0.5114684700965881,
+    0.5114684700965881,
+    0.5114684700965881
+  ],
+  "n_samples": 640
+}

models/chestmnist_mobilenetv3/onnx_export_report.json

@@ -0,0 +1,11 @@
+{
+  "base_onnx": "mobilenetv3_chestmnist.onnx",
+  "optimized_onnx": "mobilenetv3_chestmnist.onnx",
+  "serving_onnx": "mobilenetv3_chestmnist.onnx",
+  "input_shape": [
+    1,
+    3,
+    224,
+    224
+  ]
+}

models/chestmnist_mobilenetv3/training_metrics.json

@@ -0,0 +1,172 @@
+{
+  "architecture": "MobileNetV3-small",
+  "dataset": "ChestMNIST",
+  "class_names": [
+    "Atelectasis",
+    "Cardiomegaly",
+    "Effusion",
+    "Infiltration",
+    "Mass",
+    "Nodule",
+    "Pneumonia",
+    "Pneumothorax",
+    "Consolidation",
+    "Edema",
+    "Emphysema",
+    "Fibrosis",
+    "Pleural_Thickening",
+    "Hernia"
+  ],
+  "num_classes": 14,
+  "epochs": 15,
+  "batch_size": 32,
+  "learning_rate": 0.001,
+  "image_size": 224,
+  "thresholds": [
+    0.2,
+    0.15,
+    0.25,
+    0.2,
+    0.15,
+    0.1,
+    0.5,
+    0.15,
+    0.1,
+    0.1,
+    0.15,
+    0.1,
+    0.1,
+    0.5
+  ],
+  "multi_label": true,
+  "best_val_loss": 0.1511,
+  "history": [
+    {
+      "epoch": 1,
+      "train_loss": 0.1781,
+      "val_loss": 0.1591
+    },
+    {
+      "epoch": 2,
+      "train_loss": 0.1586,
+      "val_loss": 0.1547
+    },
+    {
+      "epoch": 3,
+      "train_loss": 0.1545,
+      "val_loss": 0.153
+    },
+    {
+      "epoch": 4,
+      "train_loss": 0.1517,
+      "val_loss": 0.1519
+    },
+    {
+      "epoch": 5,
+      "train_loss": 0.1496,
+      "val_loss": 0.1519
+    },
+    {
+      "epoch": 6,
+      "train_loss": 0.1478,
+      "val_loss": 0.1511
+    },
+    {
+      "epoch": 7,
+      "train_loss": 0.146,
+      "val_loss": 0.1512
+    },
+    {
+      "epoch": 8,
+      "train_loss": 0.1443,
+      "val_loss": 0.152
+    },
+    {
+      "epoch": 9,
+      "train_loss": 0.1428,
+      "val_loss": 0.152
+    },
+    {
+      "epoch": 10,
+      "train_loss": 0.1413,
+      "val_loss": 0.152
+    },
+    {
+      "epoch": 11,
+      "train_loss": 0.1399,
+      "val_loss": 0.1526
+    },
+    {
+      "epoch": 12,
+      "train_loss": 0.1386,
+      "val_loss": 0.1544
+    },
+    {
+      "epoch": 13,
+      "train_loss": 0.137,
+      "val_loss": 0.1536
+    },
+    {
+      "epoch": 14,
+      "train_loss": 0.1356,
+      "val_loss": 0.1547
+    },
+    {
+      "epoch": 15,
+      "train_loss": 0.134,
+      "val_loss": 0.1559
+    }
+  ],
+  "per_class_auroc": {
+    "Atelectasis": 0.787,
+    "Cardiomegaly": 0.894,
+    "Effusion": 0.8679,
+    "Infiltration": 0.6993,
+    "Mass": 0.7819,
+    "Nodule": 0.7006,
+    "Pneumonia": 0.7558,
+    "Pneumothorax": 0.8535,
+    "Consolidation": 0.7868,
+    "Edema": 0.8848,
+    "Emphysema": 0.873,
+    "Fibrosis": 0.8002,
+    "Pleural_Thickening": 0.7577,
+    "Hernia": 0.8757
+  },
+  "per_class_auprc": {
+    "Atelectasis": 0.308,
+    "Cardiomegaly": 0.2844,
+    "Effusion": 0.5018,
+    "Infiltration": 0.3297,
+    "Mass": 0.2325,
+    "Nodule": 0.1687,
+    "Pneumonia": 0.0425,
+    "Pneumothorax": 0.2811,
+    "Consolidation": 0.1412,
+    "Edema": 0.1476,
+    "Emphysema": 0.2651,
+    "Fibrosis": 0.0952,
+    "Pleural_Thickening": 0.1069,
+    "Hernia": 0.0419
+  },
+  "per_class_f1": {
+    "Atelectasis": 0.3587,
+    "Cardiomegaly": 0.3368,
+    "Effusion": 0.5178,
+    "Infiltration": 0.3906,
+    "Mass": 0.2883,
+    "Nodule": 0.2232,
+    "Pneumonia": 0.0,
+    "Pneumothorax": 0.3755,
+    "Consolidation": 0.2189,
+    "Edema": 0.215,
+    "Emphysema": 0.317,
+    "Fibrosis": 0.1239,
+    "Pleural_Thickening": 0.1788,
+    "Hernia": 0.0
+  },
+  "test_macro_roc_auc": 0.8084,
+  "test_macro_auprc": 0.2105,
+  "test_micro_f1": 0.3425,
+  "test_macro_f1": 0.2532
+}

requirements.txt

@@ -0,0 +1,19 @@
+datasets==3.6.0
+fastapi==0.115.12
+gradio==5.25.2
+huggingface_hub==0.31.2
+matplotlib==3.10.1
+medmnist==3.0.2
+numpy==1.26.4
+onnx==1.17.0
+onnxruntime==1.21.0
+Pillow==11.1.0
+prometheus-client==0.21.1
+python-multipart==0.0.20
+requests==2.32.3
+scikit-learn==1.6.1
+scipy==1.15.2
+seaborn==0.13.2
+torch==2.6.0
+torchvision==0.21.0
+uvicorn[standard]==0.34.1

scripts/train_chestmnist.py

@@ -0,0 +1,384 @@
+"""
+Train MobileNetV3-small on ChestMNIST (14-class multi-label).
+Outputs:
+  models/chestmnist_mobilenetv3/
+    mobilenetv3_chestmnist.pth      – PyTorch checkpoint (Model A)
+    mobilenetv3_chestmnist.onnx     – ONNX export       (Model B)
+    training_metrics.json
+    baseline_stats.json             – pixel stats for drift detection
+
+Usage (quick, shared-server-safe):
+  python3 scripts/train_chestmnist.py --epochs 5 --batch-size 32
+"""
+
+import argparse
+import json
+import logging
+import time
+from pathlib import Path
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+
+import numpy as np
+import onnx
+import torch
+import torch.nn as nn
+import torch.onnx
+from PIL import Image
+from sklearn.metrics import (
+    average_precision_score,
+    f1_score,
+    roc_auc_score,
+)
+from torch.utils.data import DataLoader
+from torchvision import transforms
+from torchvision.models import MobileNet_V3_Small_Weights, mobilenet_v3_small
+
+try:
+    import medmnist
+    from medmnist import ChestMNIST, INFO
+except ImportError as exc:
+    raise SystemExit("medmnist not installed — run: pip install medmnist") from exc
+
+BASE_DIR = Path(__file__).resolve().parents[1]
+OUTPUT_DIR = BASE_DIR / "models" / "chestmnist_mobilenetv3"
+
+CHESTMNIST_CLASSES = [
+    "Atelectasis", "Cardiomegaly", "Effusion", "Infiltration",
+    "Mass", "Nodule", "Pneumonia", "Pneumothorax",
+    "Consolidation", "Edema", "Emphysema", "Fibrosis",
+    "Pleural_Thickening", "Hernia",
+]
+NUM_CLASSES = 14
+
+
+def get_transforms(image_size: int = 224):
+    train_tf = transforms.Compose([
+        transforms.Resize((image_size, image_size)),
+        transforms.RandomHorizontalFlip(),
+        transforms.ColorJitter(brightness=0.2, contrast=0.2),
+        transforms.ToTensor(),
+        transforms.Normalize([0.5] * 3, [0.5] * 3),
+    ])
+    val_tf = transforms.Compose([
+        transforms.Resize((image_size, image_size)),
+        transforms.ToTensor(),
+        transforms.Normalize([0.5] * 3, [0.5] * 3),
+    ])
+    return train_tf, val_tf
+
+
+def build_model(num_classes: int = NUM_CLASSES) -> nn.Module:
+    model = mobilenet_v3_small(weights=MobileNet_V3_Small_Weights.IMAGENET1K_V1)
+    in_features = model.classifier[3].in_features
+    model.classifier[3] = nn.Linear(in_features, num_classes)
+    return model
+
+
+def load_chestmnist(split: str, transform, download: bool, size: int = 224,
+                    max_samples: int | None = None):
+    ds = ChestMNIST(split=split, transform=transform, download=download, size=size, as_rgb=True)
+    if max_samples and len(ds) > max_samples:
+        indices = list(range(max_samples))
+        from torch.utils.data import Subset
+        ds = Subset(ds, indices)
+    return ds
+
+
+def compute_baseline_stats(loader: DataLoader) -> dict:
+    """Compute pixel mean/std from training set for drift detection."""
+    pixels = []
+    for images, _ in loader:
+        pixels.append(images.numpy())
+        if len(pixels) >= 20:  # Sample first 20 batches
+            break
+    arr = np.concatenate(pixels, axis=0)  # (N, C, H, W)
+    flat = arr.reshape(arr.shape[0], -1)
+    return {
+        "pixel_mean": float(flat.mean()),
+        "pixel_std": float(flat.std()),
+        "channel_means": arr.mean(axis=(0, 2, 3)).tolist(),
+        "channel_stds": arr.std(axis=(0, 2, 3)).tolist(),
+        "n_samples": int(arr.shape[0]),
+    }
+
+
+def tune_thresholds(model: nn.Module, loader: DataLoader, device: torch.device) -> list[float]:
+    """Best-F1 threshold per class on validation set."""
+    model.eval()
+    all_probs, all_labels = [], []
+    with torch.no_grad():
+        for images, labels in loader:
+            images = images.to(device)
+            logits = model(images)
+            probs = torch.sigmoid(logits).cpu().numpy()
+            all_probs.append(probs)
+            all_labels.append(labels.numpy().astype(float))
+
+    probs = np.vstack(all_probs)
+    labels = np.vstack(all_labels)
+    thresholds = []
+    for i in range(NUM_CLASSES):
+        best_t, best_f1 = 0.5, 0.0
+        for t in np.arange(0.10, 0.90, 0.05):
+            preds = (probs[:, i] >= t).astype(int)
+            f = f1_score(labels[:, i], preds, zero_division=0)
+            if f > best_f1:
+                best_f1, best_t = f, t
+        thresholds.append(round(float(best_t), 3))
+    return thresholds
+
+
+def evaluate(model: nn.Module, loader: DataLoader, device: torch.device,
+             thresholds: list[float]) -> dict:
+    model.eval()
+    all_probs, all_labels = [], []
+    with torch.no_grad():
+        for images, labels in loader:
+            images = images.to(device)
+            logits = model(images)
+            probs = torch.sigmoid(logits).cpu().numpy()
+            all_probs.append(probs)
+            all_labels.append(labels.numpy().astype(float))
+
+    probs = np.vstack(all_probs)
+    labels = np.vstack(all_labels)
+
+    thr_arr = np.array(thresholds)
+    preds = (probs >= thr_arr).astype(int)
+
+    per_class_auroc, per_class_auprc, per_class_f1 = {}, {}, {}
+    for i, cls in enumerate(CHESTMNIST_CLASSES):
+        if labels[:, i].sum() > 0:
+            per_class_auroc[cls] = round(float(roc_auc_score(labels[:, i], probs[:, i])), 4)
+            per_class_auprc[cls] = round(float(average_precision_score(labels[:, i], probs[:, i])), 4)
+        else:
+            per_class_auroc[cls] = None
+            per_class_auprc[cls] = None
+        per_class_f1[cls] = round(float(f1_score(labels[:, i], preds[:, i], zero_division=0)), 4)
+
+    macro_auroc_vals = [v for v in per_class_auroc.values() if v is not None]
+    macro_auprc_vals = [v for v in per_class_auprc.values() if v is not None]
+
+    return {
+        "per_class_auroc": per_class_auroc,
+        "per_class_auprc": per_class_auprc,
+        "per_class_f1": per_class_f1,
+        "test_macro_roc_auc": round(float(np.mean(macro_auroc_vals)), 4) if macro_auroc_vals else None,
+        "test_macro_auprc": round(float(np.mean(macro_auprc_vals)), 4) if macro_auprc_vals else None,
+        "test_micro_f1": round(float(f1_score(labels, preds, average="micro", zero_division=0)), 4),
+        "test_macro_f1": round(float(f1_score(labels, preds, average="macro", zero_division=0)), 4),
+    }
+
+
+def export_onnx(model: nn.Module, output_path: Path, image_size: int, device: torch.device):
+    model.eval()
+    dummy = torch.randn(1, 3, image_size, image_size).to(device)
+    torch.onnx.export(
+        model, dummy, str(output_path),
+        input_names=["input"], output_names=["logits"],
+        dynamic_axes={"input": {0: "batch_size"}, "logits": {0: "batch_size"}},
+        opset_version=17,
+    )
+    onnx.checker.check_model(str(output_path))
+    print(f"  ONNX saved → {output_path}")
+
+
+def train_epoch(model: nn.Module, loader: DataLoader,
+                optimizer: torch.optim.Optimizer,
+                criterion: nn.Module, device: torch.device) -> float:
+    model.train()
+    total_loss = 0.0
+    for images, labels in loader:
+        images, labels = images.to(device), labels.float().to(device)
+        optimizer.zero_grad()
+        loss = criterion(model(images), labels)
+        loss.backward()
+        optimizer.step()
+        total_loss += loss.item()
+    return total_loss / max(len(loader), 1)
+
+
+def save_training_plots(history: list[dict], test_metrics: dict, output_dir: Path):
+    plots_dir = output_dir / "plots"
+    plots_dir.mkdir(parents=True, exist_ok=True)
+    
+    # 1. Learning Curve
+    epochs = [h["epoch"] for h in history]
+    train_loss = [h["train_loss"] for h in history]
+    val_loss = [h["val_loss"] for h in history]
+    
+    plt.figure(figsize=(8, 5))
+    plt.plot(epochs, train_loss, label="Train Loss", marker="o", color="#2563eb")
+    plt.plot(epochs, val_loss, label="Val Loss", marker="o", color="#dc2626")
+    plt.title("Training & Validation Loss Curve")
+    plt.xlabel("Epoch")
+    plt.ylabel("BCE Loss")
+    plt.grid(True, alpha=0.3)
+    plt.legend()
+    plt.tight_layout()
+    plt.savefig(plots_dir / "loss_curve.png", dpi=150)
+    plt.close()
+
+    # 2. Per-class metrics bar chart
+    auroc = test_metrics["per_class_auroc"]
+    auprc = test_metrics["per_class_auprc"]
+    labels = [k for k in auroc.keys() if auroc[k] is not None]
+    auroc_vals = [auroc[k] for k in labels]
+    auprc_vals = [auprc[k] for k in labels]
+    
+    y = np.arange(len(labels))
+    fig, ax = plt.subplots(figsize=(10, 8))
+    ax.barh(y - 0.2, auroc_vals, height=0.4, label="AUROC", color="#3b82f6")
+    ax.barh(y + 0.2, auprc_vals, height=0.4, label="AUPRC", color="#10b981")
+    ax.set_yticks(y)
+    ax.set_yticklabels(labels, fontweight="bold")
+    ax.set_xlim(0, 1.05)
+    ax.set_title("Per-Class AUROC & AUPRC", fontweight="bold")
+    ax.grid(axis="x", alpha=0.3)
+    ax.legend()
+    plt.tight_layout()
+    plt.savefig(plots_dir / "per_class_metrics.png", dpi=150)
+    plt.close()
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Train MobileNetV3-small on ChestMNIST (14-class multi-label)")
+    parser.add_argument("--epochs", type=int, default=15)
+    parser.add_argument("--batch-size", type=int, default=32)
+    parser.add_argument("--lr", type=float, default=1e-3)
+    parser.add_argument("--image-size", type=int, default=224)
+    parser.add_argument("--workers", type=int, default=2)
+    parser.add_argument("--max-train-samples", type=int, default=None)
+    parser.add_argument("--max-val-samples", type=int, default=None)
+    parser.add_argument("--max-test-samples", type=int, default=None)
+    parser.add_argument("--output-dir", type=Path, default=OUTPUT_DIR)
+    parser.add_argument("--no-download", action="store_true")
+    args = parser.parse_args()
+
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    print(f"Device: {device}")
+    print(f"Output: {args.output_dir}")
+
+    train_tf, val_tf = get_transforms(args.image_size)
+    download = not args.no_download
+
+    print("Loading ChestMNIST…")
+    train_ds = load_chestmnist("train", train_tf, download, args.image_size, args.max_train_samples)
+    val_ds   = load_chestmnist("val",   val_tf,   download, args.image_size, args.max_val_samples)
+    test_ds  = load_chestmnist("test",  val_tf,   download, args.image_size, args.max_test_samples)
+    print(f"  Train: {len(train_ds)}  Val: {len(val_ds)}  Test: {len(test_ds)}")
+
+    train_loader = DataLoader(train_ds, batch_size=args.batch_size, shuffle=True,
+                              num_workers=args.workers, pin_memory=True)
+    val_loader   = DataLoader(val_ds,   batch_size=64, shuffle=False, num_workers=args.workers)
+    test_loader  = DataLoader(test_ds,  batch_size=64, shuffle=False, num_workers=args.workers)
+
+    # Baseline pixel stats for drift detection
+    print("Computing baseline stats…")
+    baseline_stats = compute_baseline_stats(train_loader)
+    (args.output_dir / "baseline_stats.json").write_text(
+        json.dumps(baseline_stats, indent=2), encoding="utf-8"
+    )
+    print(f"  Mean={baseline_stats['pixel_mean']:.4f}  Std={baseline_stats['pixel_std']:.4f}")
+
+    model = build_model(NUM_CLASSES).to(device)
+    criterion = nn.BCEWithLogitsLoss()
+    optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=1e-4)
+    scheduler = torch.optim.lr_scheduler.OneCycleLR(
+        optimizer, max_lr=args.lr,
+        steps_per_epoch=len(train_loader), epochs=args.epochs,
+    )
+
+    best_val_loss = float("inf")
+    history = []
+
+    print(f"\nTraining {args.epochs} epochs…")
+    for epoch in range(1, args.epochs + 1):
+        t0 = time.time()
+        train_loss = train_epoch(model, train_loader, optimizer, criterion, device)
+        scheduler.step()
+
+        # Quick val loss
+        model.eval()
+        val_loss = 0.0
+        with torch.no_grad():
+            for imgs, lbls in val_loader:
+                imgs, lbls = imgs.to(device), lbls.float().to(device)
+                val_loss += criterion(model(imgs), lbls).item()
+        val_loss /= max(len(val_loader), 1)
+
+        elapsed = time.time() - t0
+        print(f"  Epoch {epoch}/{args.epochs} — train_loss={train_loss:.4f}  val_loss={val_loss:.4f}  ({elapsed:.1f}s)")
+        history.append({"epoch": epoch, "train_loss": round(train_loss, 4), "val_loss": round(val_loss, 4)})
+
+        if val_loss < best_val_loss:
+            best_val_loss = val_loss
+            torch.save(model.state_dict(), args.output_dir / "mobilenetv3_chestmnist.pth")
+            print("    ✓ checkpoint saved")
+
+    # Reload best checkpoint
+    model.load_state_dict(torch.load(args.output_dir / "mobilenetv3_chestmnist.pth", map_location=device))
+
+    # Threshold tuning on val set
+    print("\nTuning thresholds on validation set…")
+    thresholds = tune_thresholds(model, val_loader, device)
+    print(f"  Thresholds: {thresholds}")
+
+    # Final evaluation on test set
+    print("\nEvaluating on test set…")
+    test_metrics = evaluate(model, test_loader, device, thresholds)
+
+    # Save training_metrics.json
+    training_metrics = {
+        "architecture": "MobileNetV3-small",
+        "dataset": "ChestMNIST",
+        "class_names": CHESTMNIST_CLASSES,
+        "num_classes": NUM_CLASSES,
+        "epochs": args.epochs,
+        "batch_size": args.batch_size,
+        "learning_rate": args.lr,
+        "image_size": args.image_size,
+        "thresholds": thresholds,
+        "multi_label": True,
+        "best_val_loss": round(best_val_loss, 4),
+        "history": history,
+        **test_metrics,
+    }
+    (args.output_dir / "training_metrics.json").write_text(
+        json.dumps(training_metrics, indent=2), encoding="utf-8"
+    )
+    
+    # Save Professional Plots
+    print("\nGenerating training & evaluation plots…")
+    save_training_plots(history, test_metrics, args.output_dir)
+    print(f"\n  Macro AUROC : {test_metrics['test_macro_roc_auc']}")
+    print(f"  Macro AUPRC : {test_metrics['test_macro_auprc']}")
+    print(f"  Micro  F1   : {test_metrics['test_micro_f1']}")
+
+    # ONNX Export
+    onnx_path = args.output_dir / "mobilenetv3_chestmnist.onnx"
+    print(f"\nExporting ONNX → {onnx_path}")
+    export_onnx(model, onnx_path, args.image_size, device)
+
+    # ONNX export report (for backend resolver)
+    onnx_report = {
+        "base_onnx": str(onnx_path.name),
+        "optimized_onnx": str(onnx_path.name),
+        "serving_onnx": str(onnx_path.name),
+        "input_shape": [1, 3, args.image_size, args.image_size],
+    }
+    (args.output_dir / "onnx_export_report.json").write_text(
+        json.dumps(onnx_report, indent=2), encoding="utf-8"
+    )
+
+    print(f"\n✅ All artifacts saved to {args.output_dir}")
+    print("   Model A (PyTorch) : mobilenetv3_chestmnist.pth")
+    print("   Model B (ONNX)    : mobilenetv3_chestmnist.onnx")
+
+
+if __name__ == "__main__":
+    main()

scripts/upload_to_hf.py

@@ -0,0 +1,143 @@
+"""
+PneumoOps — Hugging Face Model Hub Upload Script
+=================================================
+Uploads both model artifacts to the HF Model Hub for versioning.
+Run this once after training, and again whenever you retrain.
+
+Usage:
+    huggingface-cli login   # one-time login
+    python scripts/upload_to_hf.py
+
+Environment variables (override defaults):
+    HF_MODEL_REPO   your-username/pneumoops-chestmnist
+    HF_TOKEN        your write token (if not logged in via CLI)
+"""
+
+import os
+import json
+from pathlib import Path
+from huggingface_hub import HfApi, upload_file, create_repo
+
+# ─── Config ───────────────────────────────────────────────────────────────────
+ROOT = Path(__file__).resolve().parents[1]
+MODEL_DIR = ROOT / "models" / "chestmnist_mobilenetv3"
+
+HF_TOKEN = os.getenv("HF_TOKEN")  # optional if already logged in via CLI
+MODEL_REPO = os.getenv("HF_MODEL_REPO", "")  # e.g. "your-username/pneumoops-chestmnist"
+
+if not MODEL_REPO:
+    print("\n⚠️  Please set HF_MODEL_REPO environment variable, e.g.:")
+    print('   export HF_MODEL_REPO="your-hf-username/pneumoops-chestmnist"')
+    raise SystemExit(1)
+
+# ─── Files to upload ──────────────────────────────────────────────────────────
+ARTIFACTS = [
+    ("mobilenetv3_chestmnist.pth",    "Model A — Baseline PyTorch checkpoint"),
+    ("mobilenetv3_chestmnist.onnx",   "Model B — Optimized ONNX artifact"),
+    ("training_metrics.json",          "Training + evaluation metrics (AUROC, AUPRC, F1)"),
+    ("baseline_stats.json",            "Pixel distribution stats for drift monitoring"),
+    ("onnx_export_report.json",        "ONNX export configuration"),
+]
+
+# ─── Model Card ───────────────────────────────────────────────────────────────
+MODEL_CARD = """---
+license: mit
+tags:
+  - medical
+  - image-classification
+  - chest-xray
+  - mlops
+  - onnx
+  - pytorch
+  - mobilenetv3
+datasets:
+  - medmnist/chestmnist
+metrics:
+  - roc_auc
+---
+
+# PneumoOps — ChestMNIST MobileNetV3-small
+
+This repository contains **two model versions** for the PneumoOps MLOps pipeline:
+- **Model A (Baseline):** `mobilenetv3_chestmnist.pth` — Standard PyTorch checkpoint
+- **Model B (Optimized):** `mobilenetv3_chestmnist.onnx` — ONNX-exported for faster inference
+
+Both models are identical in architecture (MobileNetV3-small) and weights.
+The ONNX version is used for inference time optimization in A/B testing.
+
+## Dataset
+**ChestMNIST** — 14-class multi-label chest X-ray classification  
+78,468 training images, 224×224 pixels, grayscale (converted to 3-channel).
+
+## Classes (14)
+Atelectasis, Cardiomegaly, Effusion, Infiltration, Mass, Nodule, Pneumonia,
+Pneumothorax, Consolidation, Edema, Emphysema, Fibrosis, Pleural Thickening, Hernia
+
+## Performance (Test Set)
+| Metric | Score |
+|--------|-------|
+| Macro AUROC | **0.808** |
+| Macro AUPRC | 0.210 |
+| Micro F1 | 0.343 |
+
+## Usage in PneumoOps
+These artifacts are loaded by the FastAPI backend and selected via a weighted A/B router:
+- 60% of requests → PyTorch model
+- 40% of requests → ONNX model
+
+The backend also computes a drift score using `baseline_stats.json` to detect
+out-of-distribution inputs in real time.
+"""
+
+# ─── Main ─────────────────────────────────────────────────────────────────────
+
+def main():
+    api = HfApi(token=HF_TOKEN)
+
+    print(f"\n📦 Creating/verifying model repository: {MODEL_REPO}")
+    create_repo(
+        repo_id=MODEL_REPO,
+        repo_type="model",
+        exist_ok=True,
+        token=HF_TOKEN,
+    )
+
+    # Write model card
+    card_path = MODEL_DIR / "README.md"
+    card_path.write_text(MODEL_CARD, encoding="utf-8")
+    print("   Model card written.")
+
+    # Upload model card first
+    print(f"\n⬆️  Uploading artifacts to https://huggingface.co/{MODEL_REPO}")
+    upload_file(
+        path_or_fileobj=str(card_path),
+        path_in_repo="README.md",
+        repo_id=MODEL_REPO,
+        repo_type="model",
+        commit_message="Add model card",
+        token=HF_TOKEN,
+    )
+
+    # Upload all artifacts
+    for filename, description in ARTIFACTS:
+        local_path = MODEL_DIR / filename
+        if not local_path.exists():
+            print(f"   ⚠️  Skipping {filename} — file not found")
+            continue
+        size_mb = local_path.stat().st_size / (1024 * 1024)
+        print(f"   Uploading {filename} ({size_mb:.1f} MB) — {description} ...")
+        upload_file(
+            path_or_fileobj=str(local_path),
+            path_in_repo=filename,
+            repo_id=MODEL_REPO,
+            repo_type="model",
+            commit_message=f"Upload {filename}",
+            token=HF_TOKEN,
+        )
+
+    print(f"\n✅ All artifacts uploaded!")
+    print(f"   View at: https://huggingface.co/{MODEL_REPO}")
+
+
+if __name__ == "__main__":
+    main()