Upload 14 files

.gitattributes

@@ -1,35 +1,35 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.python_version

@@ -0,0 +1 @@
+3.10

README.md

@@ -1,16 +1,15 @@
----
-title: Diabetes Readmissions
-emoji: 💬
-colorFrom: yellow
-colorTo: purple
-sdk: gradio
-sdk_version: 5.42.0
-app_file: app.py
-pinned: false
-hf_oauth: true
-hf_oauth_scopes:
-- inference-api
-short_description: This interactive Hugging Face Space provides a complete Diab
----
-
-An example chatbot using [Gradio](https://gradio.app), [`huggingface_hub`](https://huggingface.co/docs/huggingface_hub/v0.22.2/en/index), and the [Hugging Face Inference API](https://huggingface.co/docs/api-inference/index).
+---
+title: Diabetes Readmission
+emoji: 💬
+colorFrom: yellow
+colorTo: purple
+sdk: gradio
+sdk_version: 5.42.0
+app_file: app.py
+pinned: false
+hf_oauth: true
+hf_oauth_scopes:
+- inference-api
+---
+
+An example chatbot using [Gradio](https://gradio.app), [`huggingface_hub`](https://huggingface.co/docs/huggingface_hub/v0.22.2/en/index), and the [Hugging Face Inference API](https://huggingface.co/docs/api-inference/index).

chatbot/chatbot.py

@@ -0,0 +1,25 @@
+from chatbot.intents import detect_intent
+from models.bart_explainer import generate_explanation
+
+def chatbot_response(question, note, risk_score):
+    intent = detect_intent(question)
+
+    if intent == "appointment":
+        if "follow" in note.lower():
+            return "Your discharge summary mentions a follow-up. Please consult your provider for scheduling."
+        return "No specific follow-up appointment is mentioned in the discharge summary."
+
+    if intent == "severity":
+        if risk_score > 0.6:
+            level = "high"
+        elif risk_score > 0.3:
+            level = "moderate"
+        else:
+            level = "low"
+
+        return f"Your readmission risk is considered {level}."
+
+    if intent == "explanation":
+        return generate_explanation(note, risk_score)
+
+    return "I can help explain risk, follow-up, or discharge information. I cannot provide medical advice."

chatbot/intents.py

@@ -0,0 +1,11 @@
+def detect_intent(question: str) -> str:
+    q = question.lower()
+
+    if "appointment" in q or "follow up" in q:
+        return "appointment"
+    if "how serious" in q or "severity" in q or "risk" in q:
+        return "severity"
+    if "why" in q or "explain" in q:
+        return "explanation"
+
+    return "unknown"

data/load_data.py

@@ -0,0 +1,15 @@
+from datasets import load_dataset
+
+def load_discharge_notes():
+    notes = load_dataset(
+        "ntphuc149/MIMIC-III-Clinical-Database",
+        "NOTEEVENTS",
+        split="train"
+    )
+
+    # Filter only discharge summaries
+    notes = notes.filter(
+        lambda x: x["CATEGORY"] == "Discharge summary"
+    )
+
+    return notes

data/preprocess.py

@@ -0,0 +1,7 @@
+import re
+
+def clean_text(text: str) -> str:
+    text = text.lower()
+    text = re.sub(r"\[\*\*.*?\*\*\]", "", text)  # remove PHI
+    text = re.sub(r"\n+", " ", text)
+    return text.strip()

intents.py

@@ -0,0 +1,11 @@
+def detect_intent(question: str) -> str:
+    q = question.lower()
+
+    if "appointment" in q or "follow up" in q:
+        return "appointment"
+    if "how serious" in q or "severity" in q or "risk" in q:
+        return "severity"
+    if "why" in q or "explain" in q:
+        return "explanation"
+
+    return "unknown"

models/bart_explainer.py

@@ -0,0 +1,32 @@
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
+
+MODEL_NAME = "dmacres/bart-large-mimiciii-v2"
+
+tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME)
+
+def generate_explanation(note: str, risk_score: float) -> str:
+    prompt = f"""
+Discharge summary:
+{note}
+
+Predicted readmission risk: {risk_score:.2f}
+
+Explain the key clinical reasons for readmission risk.
+"""
+
+    inputs = tokenizer(
+        prompt,
+        return_tensors="pt",
+        truncation=True,
+        max_length=1024
+    )
+
+    outputs = model.generate(
+        **inputs,
+        max_length=200,
+        num_beams=4,
+        early_stopping=True
+    )
+
+    return tokenizer.decode(outputs[0], skip_special_tokens=True)

models/baseline_tfidf.py

@@ -0,0 +1,17 @@
+from sklearn.pipeline import Pipeline
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.linear_model import LogisticRegression
+
+def build_tfidf_model():
+    model = Pipeline([
+        ("tfidf", TfidfVectorizer(
+            max_features=20000,
+            ngram_range=(1, 2),
+            stop_words="english"
+        )),
+        ("clf", LogisticRegression(
+            max_iter=1000,
+            class_weight="balanced"
+        ))
+    ])
+    return model

notebooks/01_exploration.ipynb

@@ -0,0 +1,54 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "27d3ad11",
+   "metadata": {
+    "vscode": {
+     "languageId": "plaintext"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Install dependencies\n",
+    "!pip install datasets pandas\n",
+    "\n",
+    "from datasets import load_dataset\n",
+    "import pandas as pd\n",
+    "\n",
+    "# Load NOTEEVENTS directly from Hugging Face (no manual download)\n",
+    "notes = load_dataset(\n",
+    "    \"ntphuc149/MIMIC-III-Clinical-Database\",\n",
+    "    \"NOTEEVENTS\",\n",
+    "    split=\"train\"\n",
+    ")\n",
+    "\n",
+    "# Inspect columns\n",
+    "print(\"Columns:\", notes.column_names)\n",
+    "\n",
+    "# Filter discharge summaries\n",
+    "discharge_notes = notes.filter(\n",
+    "    lambda x: x[\"CATEGORY\"] == \"Discharge summary\"\n",
+    ")\n",
+    "\n",
+    "print(\"Total discharge summaries:\", len(discharge_notes))\n",
+    "\n",
+    "# Convert small sample to pandas for inspection\n",
+    "df = discharge_notes.select(range(5)).to_pandas()\n",
+    "\n",
+    "# Print sample text\n",
+    "for i, text in enumerate(df[\"TEXT\"]):\n",
+    "    print(f\"\\n--- Discharge Summary {i+1} ---\\n\")\n",
+    "    print(text[:2000])\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/02_baseline_nlp.ipynb

@@ -0,0 +1,95 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1d9c9ac7",
+   "metadata": {
+    "vscode": {
+     "languageId": "plaintext"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Install dependencies\n",
+    "!pip install datasets scikit-learn pandas numpy\n",
+    "\n",
+    "from datasets import load_dataset\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import re\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.pipeline import Pipeline\n",
+    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
+    "from sklearn.linear_model import LogisticRegression\n",
+    "from sklearn.metrics import classification_report, roc_auc_score\n",
+    "\n",
+    "# Load discharge summaries\n",
+    "notes = load_dataset(\n",
+    "    \"ntphuc149/MIMIC-III-Clinical-Database\",\n",
+    "    \"NOTEEVENTS\",\n",
+    "    split=\"train\"\n",
+    ")\n",
+    "\n",
+    "notes = notes.filter(lambda x: x[\"CATEGORY\"] == \"Discharge summary\")\n",
+    "\n",
+    "# Minimal clinical-safe cleaning\n",
+    "def clean_text(text):\n",
+    "    text = text.lower()\n",
+    "    text = re.sub(r\"\\[\\*\\*.*?\\*\\*\\]\", \"\", text)\n",
+    "    text = re.sub(r\"\\n+\", \" \", text)\n",
+    "    return text.strip()\n",
+    "\n",
+    "notes = notes.map(lambda x: {\"clean_text\": clean_text(x[\"TEXT\"])})\n",
+    "\n",
+    "# TEMPORARY labels (replace with READMITTED_30D later)\n",
+    "np.random.seed(42)\n",
+    "labels = np.random.binomial(1, 0.35, size=len(notes))\n",
+    "\n",
+    "df = pd.DataFrame({\n",
+    "    \"text\": notes[\"clean_text\"],\n",
+    "    \"label\": labels\n",
+    "})\n",
+    "\n",
+    "# Train-test split\n",
+    "X_train, X_test, y_train, y_test = train_test_split(\n",
+    "    df[\"text\"],\n",
+    "    df[\"label\"],\n",
+    "    test_size=0.2,\n",
+    "    stratify=df[\"label\"],\n",
+    "    random_state=42\n",
+    ")\n",
+    "\n",
+    "# TF-IDF baseline model\n",
+    "model = Pipeline([\n",
+    "    (\"tfidf\", TfidfVectorizer(\n",
+    "        max_features=20000,\n",
+    "        ngram_range=(1,2),\n",
+    "        stop_words=\"english\"\n",
+    "    )),\n",
+    "    (\"clf\", LogisticRegression(\n",
+    "        max_iter=1000,\n",
+    "        class_weight=\"balanced\"\n",
+    "    ))\n",
+    "])\n",
+    "\n",
+    "# Train\n",
+    "model.fit(X_train, y_train)\n",
+    "\n",
+    "# Evaluate\n",
+    "y_pred = model.predict(X_test)\n",
+    "y_prob = model.predict_proba(X_test)[:,1]\n",
+    "\n",
+    "print(classification_report(y_test, y_pred))\n",
+    "print(\"ROC-AUC:\", roc_auc_score(y_test, y_prob))\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/03_llm_explanations.ipynb

@@ -0,0 +1,88 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ca5c8ec7",
+   "metadata": {
+    "vscode": {
+     "languageId": "plaintext"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Install dependencies\n",
+    "!pip install transformers torch datasets\n",
+    "\n",
+    "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n",
+    "from datasets import load_dataset\n",
+    "import re\n",
+    "\n",
+    "# Load pretrained clinical BART model\n",
+    "MODEL_NAME = \"dmacres/bart-large-mimiciii-v2\"\n",
+    "tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\n",
+    "model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME)\n",
+    "\n",
+    "# Load discharge summaries\n",
+    "notes = load_dataset(\n",
+    "    \"ntphuc149/MIMIC-III-Clinical-Database\",\n",
+    "    \"NOTEEVENTS\",\n",
+    "    split=\"train\"\n",
+    ")\n",
+    "\n",
+    "notes = notes.filter(lambda x: x[\"CATEGORY\"] == \"Discharge summary\")\n",
+    "\n",
+    "# Clean text\n",
+    "def clean_text(text):\n",
+    "    text = text.lower()\n",
+    "    text = re.sub(r\"\\[\\*\\*.*?\\*\\*\\]\", \"\", text)\n",
+    "    text = re.sub(r\"\\n+\", \" \", text)\n",
+    "    return text.strip()\n",
+    "\n",
+    "sample_note = clean_text(notes[0][\"TEXT\"])\n",
+    "\n",
+    "# GenAI explanation function\n",
+    "def generate_explanation(note, risk_score):\n",
+    "    prompt = f\"\"\"\n",
+    "Discharge summary:\n",
+    "{note}\n",
+    "\n",
+    "Predicted readmission risk: {risk_score:.2f}\n",
+    "\n",
+    "Explain the key clinical reasons for readmission risk.\n",
+    "\"\"\"\n",
+    "\n",
+    "    inputs = tokenizer(\n",
+    "        prompt,\n",
+    "        return_tensors=\"pt\",\n",
+    "        truncation=True,\n",
+    "        max_length=1024\n",
+    "    )\n",
+    "\n",
+    "    outputs = model.generate(\n",
+    "        **inputs,\n",
+    "        max_length=200,\n",
+    "        num_beams=4,\n",
+    "        early_stopping=True\n",
+    "    )\n",
+    "\n",
+    "    return tokenizer.decode(outputs[0], skip_special_tokens=True)\n",
+    "\n",
+    "# Demo\n",
+    "risk_score = 0.72  # from your classifier\n",
+    "explanation = generate_explanation(sample_note, risk_score)\n",
+    "\n",
+    "print(\"Predicted Risk:\", risk_score)\n",
+    "print(\"\\nGenerated Explanation:\\n\")\n",
+    "print(explanation)\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

requirements.txt

@@ -0,0 +1,7 @@
+transformers==4.36.2
+huggingface_hub==0.20.3
+datasets==2.16.1
+torch
+scikit-learn
+pandas
+numpy