first

Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+COPY . .
+
+RUN pip install --upgrade pip && \
+    pip install -r requirements.txt && \
+    pip install beautifulsoup4 lxml
+
+RUN mkdir -p data models
+
+EXPOSE 8000
+
+COPY docker_entrypoint.sh /app/docker_entrypoint.sh
+RUN chmod +x /app/docker_entrypoint.sh
+
+CMD ["/app/docker_entrypoint.sh"] 

data/patient_data.csv

@@ -0,0 +1,101 @@
+PatientID,FirstName,LastName,Gender,Age,DateOfBirth,Diagnosis,BloodType,AdmissionDate,DischargeDate
+1001,Brittney,Davies,Female,32,1992-10-24,Migraine,B+,2024-06-19,2025-05-10
+1002,Aaron,Johnson,Male,93,1931-08-09,Diabetes,O+,2025-02-09,2025-03-05
+1003,Nathan,Romero,Male,73,1952-03-22,Asthma,O+,2025-03-15,2025-05-25
+1004,Corey,Garcia,Male,84,1940-08-21,Pneumonia,AB-,2024-11-09,2024-12-03
+1005,Brandon,Scott,Male,42,1983-02-20,Diabetes,A-,2024-12-17,2025-02-12
+1006,Patricia,Bernard,Female,82,1942-08-13,Arthritis,AB+,2025-01-21,2025-05-26
+1007,Patrick,Sandoval,Male,92,1932-06-26,Heart Disease,AB+,2025-01-09,2025-04-09
+1008,Heather,Hughes,Female,24,2000-10-18,Heart Disease,O+,2025-01-31,2025-03-10
+1009,Gina,Kline,Female,35,1989-10-05,Arthritis,AB-,2024-11-22,2025-01-20
+1010,Cory,Turner,Male,44,1981-01-07,Cancer,B-,2024-10-04,2024-12-07
+1011,Mary,Ray,Female,44,1981-04-13,Fracture,A-,2024-08-26,2024-11-12
+1012,Jennifer,Young,Female,80,1944-11-05,Arthritis,B+,2025-06-04,2025-06-25
+1013,Patricia,Johnson,Female,73,1951-06-19,Fracture,O-,2024-09-03,2024-09-08
+1014,Matthew,Davis,Male,8,2016-09-17,Arthritis,B+,2025-01-25,2025-05-09
+1015,Jason,Dodson,Male,62,1962-08-31,COVID-19,A-,2025-03-06,2025-04-27
+1016,Thomas,Baker,Male,97,1927-07-05,COVID-19,B+,2025-01-09,2025-03-29
+1017,Leonard,Cochran,Male,82,1943-05-06,Asthma,AB+,2025-02-28,2025-04-04
+1018,Jessica,Pearson,Female,62,1963-01-01,Heart Disease,B-,2024-07-05,2025-06-11
+1019,Robin,Johnson,Female,65,1960-05-03,Heart Disease,O-,2025-05-13,2025-05-27
+1020,Gary,Hill,Male,87,1938-05-31,Fracture,B+,2024-08-27,2025-03-04
+1021,Kevin,Lee,Male,81,1944-06-06,Pneumonia,O+,2024-10-25,2025-01-09
+1022,Robert,Powell,Male,68,1956-07-04,Migraine,A-,2025-03-11,2025-03-12
+1023,Danielle,Wright,Female,42,1983-01-18,Heart Disease,B+,2025-01-20,2025-06-16
+1024,Hannah,Fields,Female,23,2001-07-25,Fracture,AB-,2025-04-30,2025-06-19
+1025,Adam,Barton,Male,54,1970-08-11,Migraine,B-,2024-08-15,2024-10-19
+1026,Paula,Cochran,Female,83,1942-05-14,Diabetes,B+,2024-07-22,2025-02-12
+1027,Lisa,Christensen,Female,50,1974-10-15,COVID-19,B-,2025-02-24,2025-05-07
+1028,Joseph,Huff,Male,81,1944-01-04,Fracture,AB+,2025-03-31,2025-06-06
+1029,Steven,Spears,Male,99,1926-06-15,COVID-19,A-,2024-11-21,2025-01-04
+1030,Jennifer,Suarez,Female,95,1929-09-11,Asthma,AB+,2024-10-16,2025-06-10
+1031,Benjamin,Ross,Male,86,1938-07-22,Migraine,AB-,2024-10-01,2025-03-03
+1032,Michael,Cunningham,Male,70,1954-08-31,Pneumonia,B-,2025-03-09,2025-04-18
+1033,Tammy,Bullock,Female,39,1986-04-08,Pneumonia,O+,2025-03-16,2025-05-12
+1034,Nicolas,Harrison,Male,73,1951-11-07,Diabetes,AB+,2024-08-27,2024-09-21
+1035,Richard,Cortez,Male,44,1980-11-20,Fracture,B+,2025-02-15,2025-04-05
+1036,Jordan,Hernandez,Female,74,1950-07-01,Pneumonia,B-,2024-09-30,2025-05-06
+1037,Ralph,Newman,Male,14,2010-08-08,Hypertension,B+,2024-09-25,2025-01-31
+1038,Renee,Morrison,Female,63,1961-11-14,Pneumonia,A+,2025-03-08,2025-04-13
+1039,Karen,Clark,Female,91,1933-08-10,Heart Disease,O-,2024-09-30,2025-05-03
+1040,Lisa,Benson,Female,36,1989-04-04,Asthma,A-,2024-07-19,2025-01-13
+1041,Randall,May,Male,52,1973-05-06,Pneumonia,AB-,2025-01-13,2025-02-16
+1042,Michael,Baker,Male,14,2010-08-28,Fracture,B-,2025-02-23,2025-03-12
+1043,Todd,Harrington,Male,21,2003-10-26,COVID-19,O-,2025-01-18,2025-02-18
+1044,Terry,Bryan,Male,44,1980-06-22,COVID-19,A+,2025-05-07,2025-05-31
+1045,Sharon,Smith,Female,83,1942-01-01,Pneumonia,A+,2025-04-01,2025-05-12
+1046,David,Ray,Male,70,1954-11-26,Diabetes,A-,2024-10-09,2025-04-09
+1047,Danielle,Dominguez,Female,66,1959-05-25,COVID-19,AB-,2024-09-18,2024-10-08
+1048,Victoria,Johnson,Female,5,2019-10-05,Hypertension,A+,2025-04-02,2025-06-08
+1049,Robert,Ferguson,Male,68,1956-06-29,Hypertension,B+,2024-08-19,2025-01-31
+1050,Brandon,Hall,Male,71,1954-03-05,COVID-19,B-,2024-11-22,2025-05-02
+1051,Brittany,Bailey,Female,67,1958-01-24,Asthma,A-,2025-02-04,2025-03-07
+1052,Kyle,Ryan,Male,49,1976-05-07,Migraine,AB-,2024-09-13,2024-11-13
+1053,Andrew,Smith,Male,91,1933-07-29,Asthma,B-,2024-12-12,2025-02-11
+1054,Richard,Williams,Male,4,2021-02-14,Pneumonia,O-,2024-09-23,2024-09-30
+1055,Aaron,Walton,Male,58,1967-04-21,Cancer,B-,2024-09-23,2025-05-31
+1056,Stephanie,Johnson,Female,44,1980-12-07,Asthma,O+,2024-08-08,2025-01-11
+1057,Benjamin,Mitchell,Male,56,1968-09-19,Heart Disease,O+,2025-01-14,2025-02-14
+1058,Jeffrey,Spence,Male,6,2018-09-29,Migraine,B-,2024-08-23,2025-02-01
+1059,Jamie,Russell,Female,32,1992-09-08,Cancer,A+,2025-04-04,2025-06-26
+1060,Alexander,Hernandez,Male,37,1987-08-04,Heart Disease,B+,2024-11-18,2025-04-27
+1061,Nicole,Gibson,Female,54,1970-11-10,Fracture,A+,2025-05-28,2025-06-13
+1062,Juan,Thompson,Male,11,2014-03-18,Migraine,AB+,2024-12-30,2025-06-10
+1063,Duane,West,Male,27,1998-02-14,Diabetes,A-,2024-09-22,2024-10-05
+1064,Natalie,Lee,Female,37,1988-05-04,Hypertension,B+,2025-02-17,2025-05-02
+1065,James,Liu,Male,37,1987-12-28,Heart Disease,B-,2024-06-27,2024-11-01
+1066,Pam,Baker,Female,40,1985-05-23,Fracture,B+,2025-02-17,2025-03-27
+1067,David,Williams,Male,27,1997-09-16,Fracture,AB-,2025-02-12,2025-04-16
+1068,Mark,Gray,Male,66,1959-01-27,COVID-19,B+,2025-01-26,2025-06-22
+1069,Jessica,Cannon,Female,98,1926-11-23,Migraine,A-,2024-09-07,2025-05-21
+1070,Jason,Roberts,Male,28,1997-04-02,Arthritis,AB-,2024-12-28,2025-05-18
+1071,Robert,Walker,Male,97,1928-03-08,Pneumonia,A-,2025-05-19,2025-06-12
+1072,Crystal,Williams,Female,22,2002-10-17,Hypertension,O-,2024-09-06,2024-10-25
+1073,Emily,Thomas,Female,10,2015-05-16,COVID-19,AB-,2024-08-31,2025-03-02
+1074,Mark,White,Male,46,1979-01-14,Diabetes,AB+,2025-03-30,2025-06-01
+1075,Jillian,Lucas,Female,47,1977-10-13,Diabetes,A+,2024-10-06,2024-12-23
+1076,Jeffrey,Nguyen,Male,23,2001-09-06,Arthritis,B+,2025-01-24,2025-04-30
+1077,Wendy,Nguyen,Female,13,2012-01-20,Cancer,O-,2024-09-18,2025-01-07
+1078,Lance,Miller,Male,47,1977-09-10,Fracture,AB+,2024-10-12,2025-05-20
+1079,William,Andrews,Male,69,1955-11-03,COVID-19,O+,2024-06-23,2024-12-19
+1080,Heather,King,Female,40,1984-10-13,Heart Disease,B-,2025-04-27,2025-05-19
+1081,Kayla,Fields,Female,25,2000-05-02,Arthritis,AB-,2025-01-04,2025-05-02
+1082,Sarah,Kline,Female,62,1962-07-04,Hypertension,B+,2024-07-20,2024-09-17
+1083,Jeremy,Miller,Male,24,2001-06-02,COVID-19,B-,2025-04-08,2025-06-13
+1084,Melissa,Wallace,Female,70,1955-03-01,Migraine,O+,2024-12-27,2025-05-31
+1085,John,Weiss,Male,67,1958-03-03,Pneumonia,AB-,2025-05-19,2025-05-26
+1086,Darren,Herrera,Male,11,2014-04-08,Asthma,A-,2024-07-04,2024-09-18
+1087,Megan,Kelly,Female,9,2016-03-14,Asthma,A-,2024-06-24,2024-12-10
+1088,John,Robertson,Male,8,2017-04-20,Heart Disease,O-,2024-08-18,2025-04-24
+1089,Michelle,Robles,Female,19,2006-06-09,Diabetes,O-,2024-11-29,2025-06-17
+1090,Lisa,Wright,Female,12,2012-07-14,Cancer,B-,2025-01-21,2025-03-05
+1091,Melissa,Reynolds,Female,12,2013-03-18,Hypertension,O-,2024-09-29,2024-10-24
+1092,Kristen,Sanders,Female,48,1976-10-23,Heart Disease,AB+,2025-05-22,2025-06-22
+1093,Timothy,Short,Male,2,2022-08-08,Heart Disease,B+,2024-12-07,2025-01-12
+1094,Gene,Greene,Male,91,1933-11-12,Diabetes,AB+,2024-09-11,2024-11-19
+1095,Jennifer,Brown,Female,2,2023-01-22,Diabetes,A+,2024-07-09,2025-06-17
+1096,Jeremy,Wright,Male,45,1980-03-29,Cancer,O+,2025-01-25,2025-02-07
+1097,Patricia,Pierce,Female,58,1967-01-20,COVID-19,A+,2024-12-27,2025-06-16
+1098,Cindy,Brooks,Female,40,1984-12-06,Asthma,A+,2024-07-19,2025-04-08
+1099,Seth,Dawson,Male,30,1995-03-05,Diabetes,AB-,2024-12-20,2025-03-30
+1100,Gene,Kelly,Male,70,1954-08-22,Arthritis,AB-,2024-09-05,2025-02-02

data/processed_patient_data.csv

@@ -0,0 +1,101 @@
+age,gender,diagnosis,blood_type,length_of_stay,age_group,admission_season,admission_day,admission_month,admission_year
+32,0,8,4,325,1,1,2,5,4
+93,1,4,6,24,4,0,6,1,5
+73,1,1,6,71,4,0,5,2,5
+84,1,9,3,24,4,3,5,10,4
+42,1,4,1,57,2,3,1,11,4
+82,0,0,2,125,4,0,1,0,5
+92,1,6,2,90,4,0,3,0,5
+24,0,6,6,38,1,0,4,0,5
+35,0,0,3,59,1,3,4,10,4
+44,1,3,5,64,2,3,4,9,4
+44,0,5,1,78,2,2,0,7,4
+80,0,0,4,21,4,1,2,5,5
+73,0,5,7,5,4,2,1,8,4
+8,1,0,4,104,0,0,5,0,5
+62,1,2,1,52,3,0,3,2,5
+97,1,2,4,79,4,0,3,0,5
+82,1,1,2,35,4,0,4,1,5
+62,0,6,5,341,3,2,4,6,4
+65,0,6,7,14,3,1,1,4,5
+87,1,5,4,189,4,2,1,7,4
+81,1,9,6,76,4,3,4,9,4
+68,1,8,1,1,4,0,1,2,5
+42,0,6,4,147,2,0,0,0,5
+23,0,5,3,50,1,1,2,3,5
+54,1,8,5,65,3,2,3,7,4
+83,0,4,4,205,4,2,0,6,4
+50,0,2,5,72,2,0,0,1,5
+81,1,5,2,67,4,0,0,2,5
+99,1,2,1,44,4,3,3,10,4
+95,0,1,2,237,4,3,2,9,4
+86,1,8,3,153,4,3,1,9,4
+70,1,9,5,40,4,0,6,2,5
+39,0,9,6,57,2,0,6,2,5
+73,1,4,2,25,4,2,1,7,4
+44,1,5,4,49,2,0,5,1,5
+74,0,9,5,218,4,2,0,8,4
+14,1,7,4,128,0,2,2,8,4
+63,0,9,0,36,3,0,5,2,5
+91,0,6,7,215,4,2,0,8,4
+36,0,1,1,178,2,2,4,6,4
+52,1,9,3,34,3,0,0,0,5
+14,1,5,5,17,0,0,6,1,5
+21,1,2,7,31,1,0,5,0,5
+44,1,2,0,24,2,1,2,4,5
+83,0,9,0,41,4,1,1,3,5
+70,1,4,1,182,4,3,2,9,4
+66,0,2,3,20,4,2,2,8,4
+5,0,7,0,67,0,1,2,3,5
+68,1,7,4,165,4,2,0,7,4
+71,1,2,5,161,4,3,4,10,4
+67,0,1,1,31,4,0,1,1,5
+49,1,8,3,61,2,2,4,8,4
+91,1,1,5,61,4,3,3,11,4
+4,1,9,7,7,0,2,0,8,4
+58,1,3,5,250,3,2,0,8,4
+44,0,1,6,156,2,2,3,7,4
+56,1,6,6,31,3,0,1,0,5
+6,1,8,5,162,0,2,4,7,4
+32,0,3,0,83,1,1,4,3,5
+37,1,6,4,160,2,3,0,10,4
+54,0,5,0,16,3,1,2,4,5
+11,1,8,2,162,0,3,0,11,4
+27,1,4,1,13,1,2,6,8,4
+37,0,7,4,74,2,0,0,1,5
+37,1,6,5,127,2,1,3,5,4
+40,0,5,4,38,2,0,0,1,5
+27,1,5,3,63,1,0,2,1,5
+66,1,2,4,147,4,0,6,0,5
+98,0,8,1,256,4,2,5,8,4
+28,1,0,3,141,1,3,5,11,4
+97,1,9,1,24,4,1,0,4,5
+22,0,7,7,49,1,2,4,8,4
+10,0,2,3,183,0,2,5,7,4
+46,1,4,2,63,2,0,6,2,5
+47,0,4,0,78,2,3,6,9,4
+23,1,0,4,96,1,0,4,0,5
+13,0,3,7,111,0,2,2,8,4
+47,1,5,2,220,2,3,5,9,4
+69,1,2,6,179,4,1,6,5,4
+40,0,6,5,22,2,1,6,3,5
+25,0,0,3,118,1,0,5,0,5
+62,0,7,4,59,3,2,5,6,4
+24,1,2,5,66,1,1,1,3,5
+70,0,8,6,155,4,3,4,11,4
+67,1,9,3,7,4,1,0,4,5
+11,1,1,1,76,0,2,3,6,4
+9,0,1,1,169,0,1,0,5,4
+8,1,6,7,249,0,2,6,7,4
+19,0,4,7,200,1,3,4,10,4
+12,0,3,5,43,0,0,1,0,5
+12,0,7,7,25,0,2,6,8,4
+48,0,6,2,31,2,1,3,4,5
+2,1,6,4,36,0,3,5,11,4
+91,1,4,2,69,4,2,2,8,4
+2,0,4,0,343,0,2,1,6,4
+45,1,3,6,13,2,0,5,0,5
+58,0,2,0,171,3,3,4,11,4
+40,0,1,0,263,2,2,4,6,4
+30,1,4,3,100,1,3,4,11,4
+70,1,0,3,150,4,2,3,8,4

docker_entrypoint.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+# Start continual training loop in background
+python src/continual_train_loop.py &
+
+# Start web scraper in background
+python src/web_scraper.py &
+
+# Start FastAPI server (foreground)
+uvicorn src.api:app --host 0.0.0.0 --port 8000 --reload 

readme.md

@@ -0,0 +1,431 @@
+---
+title: Healthcare Synthetic Data VAE
+emoji: "🏥"
+colorFrom: blue
+colorTo: green
+sdk: gradio
+sdk_version: "3.41.2"
+app_file: src/api.py
+pinned: false
+---
+# Healthcare Synthetic Data Generation using VAE
+
+A complete pipeline for generating synthetic healthcare data using Variational Autoencoders (VAE) with FastAPI serving capabilities.
+
+## 🏥 Project Overview
+
+This project implements a **Variational Autoencoder (VAE)** to generate synthetic patient data for healthcare AI applications. The system can create realistic patient records while preserving privacy and statistical properties of the original data.
+
+### Key Features
+- 🔬 **Medical Data Generation**: Creates synthetic patient records with realistic correlations
+- 🔒 **Privacy-Preserving**: No direct storage of original patient data
+- 🚀 **Production Ready**: FastAPI deployment with RESTful endpoints
+- 📊 **Quality Validation**: Built-in data quality metrics and evaluation
+- 🎛️ **Configurable**: Easy hyperparameter tuning and model customization
+
+## 🏗️ Project Structure
+
+```
+healthcare-vae/
+├── README.md                    # This file
+├── requirements.txt             # Python dependencies
+├── data/
+│   ├── raw_patient_data.csv    # Original patient data (not included)
+│   └── processed_patient_data.csv  # Preprocessed features
+├── src/
+│   ├── model.py                # VAE architecture
+│   ├── data_preprocessing.py   # Data cleaning and feature engineering
+│   ├── train.py               # Model training script
+│   ├── evaluate.py            # Model evaluation and metrics
+│   └── api.py                 # FastAPI serving application
+├── models/
+│   ├── vae_model.pth          # Trained VAE weights
+│   ├── best_vae_model.pth     # Best model checkpoint
+│   ├── scaler.pkl             # Data preprocessing scaler
+│   └── feature_names.pkl      # Feature column names
+├── notebooks/
+│   ├── data_exploration.ipynb # Data analysis and visualization
+│   └── model_analysis.ipynb   # Model performance analysis
+└── tests/
+    ├── test_model.py          # Model unit tests
+    └── test_api.py            # API endpoint tests
+```
+
+## 🧠 How VAE Works for Healthcare Data
+
+### Mathematical Foundation
+
+**Variational Autoencoder (VAE)** learns a compressed representation of patient data:
+
+```
+Patient Data → Encoder → Latent Space (μ, σ) → Decoder → Synthetic Patient
+```
+
+**Key Components:**
+1. **Encoder**: Maps patient features to latent space parameters
+2. **Latent Space**: Continuous representation of patient "types"
+3. **Decoder**: Generates new patients from latent codes
+4. **Loss Function**: Reconstruction + KL Divergence
+
+### Training Process
+
+```mermaid
+graph TD
+    A[Raw Patient Data] --> B[Data Preprocessing]
+    B --> C[Feature Engineering]
+    C --> D[Train/Validation Split]
+    D --> E[VAE Training]
+    E --> F[Model Validation]
+    F --> G{Good Performance?}
+    G -->|No| H[Adjust Hyperparameters]
+    H --> E
+    G -->|Yes| I[Save Best Model]
+    I --> J[Deploy API]
+```
+
+## 🚀 Quick Start
+
+### 1. Installation
+
+```bash
+# Clone repository
+git clone https://github.com/theaniketgiri/healthcare-vae.git
+cd healthcare-vae
+
+# Install dependencies
+pip install -r requirements.txt
+
+# Create necessary directories
+mkdir -p data models notebooks tests
+```
+
+### 2. Data Preparation
+
+```python
+# Create sample patient data (for demonstration)
+python -c "
+import pandas as pd
+import numpy as np
+
+np.random.seed(42)
+n_patients = 1000
+
+# Generate synthetic patient data
+data = {
+    'patient_id': range(1, n_patients + 1),
+    'age': np.random.normal(50, 15, n_patients).clip(18, 90),
+    'gender': np.random.choice(['M', 'F'], n_patients),
+    'bmi': np.random.normal(25, 5, n_patients).clip(15, 50),
+    'systolic_bp': np.random.normal(120, 20, n_patients).clip(80, 200),
+    'diastolic_bp': np.random.normal(80, 15, n_patients).clip(50, 120),
+    'diabetes': np.random.choice([0, 1], n_patients, p=[0.8, 0.2]),
+    'cholesterol': np.random.normal(200, 40, n_patients).clip(100, 400),
+    'heart_rate': np.random.normal(72, 12, n_patients).clip(50, 120),
+    'smoking': np.random.choice([0, 1], n_patients, p=[0.7, 0.3]),
+    'family_history': np.random.choice([0, 1], n_patients, p=[0.6, 0.4])
+}
+
+df = pd.DataFrame(data)
+df.to_csv('data/raw_patient_data.csv', index=False)
+print('Sample data created!')
+"
+```
+
+### 3. Train the Model
+
+```bash
+# Preprocess data and train VAE
+python src/train.py
+```
+
+### 4. Start the API
+
+```bash
+# Launch FastAPI server
+uvicorn src.api:app --reload --host 0.0.0.0 --port 8000
+```
+
+### 5. Generate Synthetic Data
+
+```bash
+# Test the API
+curl -X POST "http://localhost:8000/generate" \
+     -H "Content-Type: application/json" \
+     -d '{"n_samples": 10, "random_seed": 42}'
+```
+
+## 📊 Data Flow Explanation
+
+### Phase 1: Data Preprocessing (`data_preprocessing.py`)
+
+```python
+Raw Patient Data → Feature Engineering → Normalization → Processed Data
+```
+
+**Operations:**
+- **Missing Value Handling**: Imputation strategies for clinical data
+- **Categorical Encoding**: One-hot encoding for gender, diagnosis codes
+- **Feature Scaling**: StandardScaler for numerical stability
+- **Outlier Detection**: Medical range validation
+- **Feature Engineering**: BMI categories, age groups, risk scores
+
+### Phase 2: Model Architecture (`model.py`)
+
+**VAE Architecture for Healthcare:**
+```
+Input Layer (n_features) 
+    ↓
+Encoder Hidden Layers (64→32→16)
+    ↓
+Latent Space (μ, σ) - 8 dimensions
+    ↓
+Decoder Hidden Layers (16→32→64)
+    ↓
+Output Layer (n_features)
+```
+
+**Why This Architecture?**
+- **Small Latent Space (8D)**: Captures essential patient patterns without overfitting
+- **Symmetric Design**: Encoder mirrors decoder for balanced learning
+- **Dropout Regularization**: Prevents overfitting on small medical datasets
+- **Medical Constraints**: Output activations ensure realistic medical ranges
+
+### Phase 3: Training Process (`train.py`)
+
+**Training Loop:**
+```python
+for epoch in range(EPOCHS):
+    # Forward pass
+    patient_data → encoder → (μ, σ) → sample_z → decoder → reconstructed_patient
+    
+    # Loss calculation
+    reconstruction_loss = ||original - reconstructed||²
+    kl_loss = KL_divergence(latent_distribution, standard_normal)
+    total_loss = reconstruction_loss + β * kl_loss
+    
+    # Optimization
+    optimizer.step()
+    
+    # Validation and early stopping
+    if validation_loss_improved:
+        save_best_model()
+```
+
+**Key Training Features:**
+- **Early Stopping**: Prevents overfitting on medical data
+- **Learning Rate Scheduling**: Adapts learning rate based on progress
+- **Gradient Clipping**: Ensures stable training
+- **Medical Validation**: Checks generated data for medical plausibility
+
+### Phase 4: Synthetic Data Generation
+
+**Generation Process:**
+```python
+# Sample from standard normal distribution
+z ~ N(0, I)  # 8-dimensional latent code
+
+# Decode to patient features
+synthetic_patient = decoder(z)
+
+# Inverse transform to original scale
+real_patient_data = scaler.inverse_transform(synthetic_patient)
+```
+
+**Quality Assurance:**
+- **Statistical Validation**: Mean, variance, correlation preservation
+- **Medical Range Checking**: Ensures realistic vital signs
+- **Diversity Metrics**: Prevents mode collapse
+- **Privacy Metrics**: Ensures no direct patient replication
+
+## 🔧 API Endpoints
+
+### Generate Synthetic Patients
+```http
+POST /generate
+Content-Type: application/json
+
+{
+    "n_samples": 100,
+    "random_seed": 42,
+    "temperature": 1.0
+}
+```
+
+**Response:**
+```json
+{
+    "data": [[patient_features], ...],
+    "metadata": {
+        "n_samples": 100,
+        "latent_dim": 8,
+        "features": ["age", "gender", "bmi", ...]
+    }
+}
+```
+
+### Encode Real Patient
+```http
+POST /encode
+Content-Type: application/json
+
+{
+    "age": 45,
+    "gender": 1,
+    "bmi": 28.5,
+    "systolic_bp": 140,
+    "diabetes": 0,
+    "cholesterol": 220,
+    "heart_rate": 72
+}
+```
+
+### Health Check
+```http
+GET /health
+```
+
+## 📈 Model Performance Metrics
+
+### Training Metrics
+- **Reconstruction Loss**: How well the model recreates original patients
+- **KL Divergence**: How well the latent space follows normal distribution
+- **Validation Loss**: Generalization performance
+
+### Generation Quality Metrics
+- **Statistical Fidelity**: Correlation preservation, distribution matching
+- **Medical Plausibility**: Realistic vital sign ranges, logical relationships
+- **Privacy Protection**: No memorization of training patients
+- **Diversity**: Coverage of different patient types
+
+## ⚙️ Configuration
+
+### Hyperparameters (`train.py`)
+```python
+BATCH_SIZE = 32        # Optimal for small medical datasets
+LEARNING_RATE = 1e-3   # Conservative for stable training
+EPOCHS = 150           # Sufficient for convergence
+LATENT_DIM = 8         # Captures essential patient variations
+BETA = 1.0             # Balance reconstruction vs. regularization
+```
+
+### Model Architecture (`model.py`)
+```python
+INPUT_DIM = 10         # Number of patient features
+HIDDEN_DIMS = [32, 16] # Encoder/decoder layer sizes
+DROPOUT = 0.1          # Regularization strength
+```
+
+## 🔒 Privacy and Compliance
+
+### Privacy Preservation
+- **No Direct Storage**: Original patients not stored in model
+- **Latent Space Learning**: Model learns patterns, not individuals
+- **Differential Privacy**: Optional noise injection for stronger privacy
+- **Audit Trail**: Generation logging for compliance
+
+### HIPAA Compliance Considerations
+- **De-identification**: Remove direct identifiers before training
+- **Access Controls**: Secure API endpoints with authentication
+- **Audit Logging**: Track all data generation requests
+- **Data Minimization**: Only use necessary patient features
+
+## 🧪 Testing and Validation
+
+### Unit Tests
+```bash
+# Run model tests
+python -m pytest tests/test_model.py
+
+# Run API tests
+python -m pytest tests/test_api.py
+```
+
+### Manual Validation
+```python
+# Evaluate model performance
+python src/evaluate.py
+
+# Check generated data quality
+python -c "
+from src.api import generate_synthetic_data
+data = generate_synthetic_data({'n_samples': 100})
+print('Generated data shape:', len(data['data']), 'x', len(data['data'][0]))
+"
+```
+
+## 🚀 Deployment Options
+
+### Local Development
+```bash
+uvicorn src.api:app --reload --port 8000
+```
+
+### Docker Deployment
+```dockerfile
+FROM python:3.9-slim
+COPY . /app
+WORKDIR /app
+RUN pip install -r requirements.txt
+CMD ["uvicorn", "src.api:app", "--host", "0.0.0.0", "--port", "8000"]
+```
+
+### Cloud Deployment
+- **AWS**: ECS, Lambda, or SageMaker
+- **GCP**: Cloud Run, AI Platform
+- **Azure**: Container Instances, ML Service
+
+## 📊 Use Cases
+
+### Healthcare AI Training
+- **Augment Small Datasets**: Increase training data for rare conditions
+- **Balance Datasets**: Generate underrepresented patient groups
+- **Privacy-Safe Sharing**: Share synthetic data instead of real patients
+- **Model Testing**: Stress-test AI systems with edge cases
+
+### Research Applications
+- **Clinical Trial Simulation**: Model patient populations
+- **Treatment Planning**: Explore treatment outcomes
+- **Epidemiological Studies**: Study disease patterns
+- **Health Economics**: Model patient costs and outcomes
+
+## 🔮 Future Enhancements
+
+### Model Improvements
+- **Conditional Generation**: Generate patients with specific conditions
+- **Temporal Models**: Patient progression over time
+- **Multi-Modal**: Include medical images, text notes
+- **Federated Learning**: Train across multiple hospitals
+
+### Technical Enhancements
+- **Real-time Generation**: Streaming synthetic data
+- **Model Monitoring**: Drift detection and retraining
+- **A/B Testing**: Compare different generation strategies
+- **Scalability**: Handle larger datasets and more complex models
+
+## 🤝 Contributing
+
+1. Fork the repository
+2. Create a feature branch: `git checkout -b feature/new-feature`
+3. Make changes and test thoroughly
+4. Submit a pull request with detailed description
+
+## 📄 License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+
+## ⚠️ Disclaimers
+
+- **Research Purpose**: This is for research and development purposes
+- **Medical Advice**: Generated data should not be used for actual medical decisions
+- **Compliance**: Ensure compliance with local healthcare regulations
+- **Validation**: Always validate synthetic data quality for your specific use case
+
+## 📞 Support
+
+For questions or issues:
+- Create an issue on GitHub
+- Email: theaniketgiri@gmail.com
+
+---
+
+**Happy Healthcare AI Development! 🏥🤖**

requirements.txt

@@ -0,0 +1,7 @@
+torch
+pandas
+numpy
+fastapi
+uvicorn
+scikit-learn
+joblib 

src/api.py

@@ -0,0 +1,290 @@
+# src/api.py - Enhanced API with better error handling for patient data
+from fastapi import FastAPI, HTTPException, File, UploadFile
+from pydantic import BaseModel, Field
+import torch
+import numpy as np
+import joblib
+from src.model import TabularVAE
+from typing import List, Optional, Dict, Any
+import os
+import shutil
+from fastapi.responses import JSONResponse, HTMLResponse
+import json
+
+app = FastAPI(title="Healthcare VAE API", version="1.0.0")
+
+# Load model and scaler
+try:
+    # Load feature names and determine input dimension
+    if os.path.exists("models/feature_names.pkl"):
+        feature_names = joblib.load("models/feature_names.pkl")
+        INPUT_DIM = len(feature_names)
+        print(f"Loaded {INPUT_DIM} features: {feature_names}")
+    else:
+        # Fallback to default features
+        feature_names = ["age", "gender", "diagnosis", "blood_type", "length_of_stay", 
+                        "age_group", "admission_season", "admission_day", "admission_month", "admission_year"]
+        INPUT_DIM = len(feature_names)
+        print(f"Using default {INPUT_DIM} features")
+    
+    LATENT_DIM = 8
+    
+    model = TabularVAE(input_dim=INPUT_DIM, latent_dim=LATENT_DIM, hidden_dims=(32, 16))
+    model.load_state_dict(torch.load("models/vae_model.pth", map_location='cpu'))
+    model.eval()
+    
+    scaler = joblib.load("models/scaler.pkl")
+    
+    # Load encoders if available
+    encoders = None
+    if os.path.exists("models/encoders.pkl"):
+        encoders = joblib.load("models/encoders.pkl")
+    
+    print("Model and scaler loaded successfully!")
+except Exception as e:
+    print(f"Error loading model: {e}")
+    print("Please run training first!")
+
+class GenerateRequest(BaseModel):
+    n_samples: int = Field(..., ge=1, le=1000, description="Number of samples to generate")
+    random_seed: Optional[int] = Field(None, description="Random seed for reproducibility")
+    temperature: float = Field(1.0, ge=0.1, le=2.0, description="Sampling temperature")
+
+class PatientData(BaseModel):
+    age: float = Field(..., ge=0, le=120, description="Patient age")
+    gender: str = Field(..., description="Patient gender (Male/Female)")
+    diagnosis: str = Field(..., description="Patient diagnosis")
+    blood_type: str = Field(..., description="Blood type")
+    length_of_stay: Optional[float] = Field(None, description="Length of stay in days")
+    age_group: Optional[int] = Field(None, ge=0, le=4, description="Age group (0-4)")
+    admission_season: Optional[int] = Field(None, ge=0, le=3, description="Admission season (0-3)")
+    admission_day: Optional[int] = Field(None, ge=0, le=6, description="Admission day of week (0-6)")
+    admission_month: Optional[int] = Field(None, ge=0, le=11, description="Admission month (0-11)")
+    admission_year: Optional[int] = Field(None, description="Admission year (normalized)")
+
+class GeneratedResponse(BaseModel):
+    data: List[List[float]]
+    metadata: dict
+
+def convert_numpy_to_python(obj):
+    """Convert numpy types to Python native types for JSON serialization"""
+    if isinstance(obj, np.integer):
+        return int(obj)
+    elif isinstance(obj, np.floating):
+        return float(obj)
+    elif isinstance(obj, np.ndarray):
+        return obj.tolist()
+    elif isinstance(obj, list):
+        return [convert_numpy_to_python(item) for item in obj]
+    elif isinstance(obj, dict):
+        return {key: convert_numpy_to_python(value) for key, value in obj.items()}
+    else:
+        return obj
+
+@app.get("/")
+def read_root():
+    return {"message": "Healthcare VAE API is running!", "features": feature_names}
+
+@app.get("/features")
+def get_features():
+    """Get information about the model features"""
+    return {
+        "feature_names": feature_names,
+        "input_dim": INPUT_DIM,
+        "latent_dim": LATENT_DIM
+    }
+
+@app.post("/generate", response_model=GeneratedResponse)
+def generate_synthetic_data(request: GenerateRequest):
+    try:
+        if request.random_seed is not None:
+            torch.manual_seed(request.random_seed)
+            np.random.seed(request.random_seed)
+        
+        # Generate samples
+        z = torch.randn(request.n_samples, LATENT_DIM) * request.temperature
+        
+        with torch.no_grad():
+            samples = model.decode(z).numpy()
+        
+        # Inverse transform to original scale
+        data = scaler.inverse_transform(samples).tolist()
+        
+        metadata = {
+            "n_samples": request.n_samples,
+            "latent_dim": LATENT_DIM,
+            "temperature": request.temperature,
+            "features": feature_names
+        }
+        
+        return {"data": data, "metadata": metadata}
+    
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Generation failed: {str(e)}")
+
+@app.post("/encode")
+def encode_patient(patient: PatientData):
+    """Encode patient data to latent space"""
+    try:
+        # Convert patient data to feature vector
+        feature_vector = []
+        
+        # Age
+        feature_vector.append(patient.age)
+        
+        # Gender (encode if encoders available)
+        if encoders and 'gender' in encoders:
+            gender_encoded = encoders['gender'].transform([patient.gender])[0]
+            feature_vector.append(gender_encoded)
+        else:
+            # Fallback encoding
+            gender_encoded = 0 if patient.gender.lower() == 'male' else 1
+            feature_vector.append(gender_encoded)
+        
+        # Diagnosis (encode if encoders available)
+        if encoders and 'diagnosis' in encoders:
+            diagnosis_encoded = encoders['diagnosis'].transform([patient.diagnosis])[0]
+            feature_vector.append(diagnosis_encoded)
+        else:
+            # Fallback encoding (simple hash)
+            diagnosis_encoded = hash(patient.diagnosis) % 10
+            feature_vector.append(diagnosis_encoded)
+        
+        # Blood type (encode if encoders available)
+        if encoders and 'blood_type' in encoders:
+            blood_encoded = encoders['blood_type'].transform([patient.blood_type])[0]
+            feature_vector.append(blood_encoded)
+        else:
+            # Fallback encoding (simple hash)
+            blood_encoded = hash(patient.blood_type) % 8
+            feature_vector.append(blood_encoded)
+        
+        # Length of stay
+        los = patient.length_of_stay if patient.length_of_stay is not None else 7.0
+        feature_vector.append(los)
+        
+        # Age group
+        age_group = patient.age_group if patient.age_group is not None else 2
+        feature_vector.append(age_group)
+        
+        # Admission season
+        season = patient.admission_season if patient.admission_season is not None else 0
+        feature_vector.append(season)
+        
+        # Admission day
+        day = patient.admission_day if patient.admission_day is not None else 0
+        feature_vector.append(day)
+        
+        # Admission month
+        month = patient.admission_month if patient.admission_month is not None else 0
+        feature_vector.append(month)
+        
+        # Admission year
+        year = patient.admission_year if patient.admission_year is not None else 4
+        feature_vector.append(year)
+        
+        # Ensure we have the right number of features
+        if len(feature_vector) != INPUT_DIM:
+            # Pad or truncate to match input dimension
+            while len(feature_vector) < INPUT_DIM:
+                feature_vector.append(0.0)
+            feature_vector = feature_vector[:INPUT_DIM]
+        
+        # Convert to array and scale
+        data = np.array([feature_vector])
+        scaled_data = scaler.transform(data)
+        tensor_data = torch.tensor(scaled_data, dtype=torch.float32)
+        
+        with torch.no_grad():
+            mu, logvar = model.encode(tensor_data)
+        
+        # Convert numpy types to Python native types for JSON serialization
+        response = {
+            "latent_mean": convert_numpy_to_python(mu.numpy().tolist()),
+            "latent_logvar": convert_numpy_to_python(logvar.numpy().tolist()),
+            "features_used": feature_names,
+            "feature_values": convert_numpy_to_python(feature_vector)
+        }
+        
+        return response
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Encoding failed: {str(e)}")
+
+@app.get("/health")
+def health_check():
+    """Health check endpoint"""
+    return {
+        "status": "healthy",
+        "model_loaded": True,
+        "input_dim": INPUT_DIM,
+        "latent_dim": LATENT_DIM
+    }
+
+@app.post("/upload_data")
+async def upload_data(file: UploadFile = File(...)):
+    """Upload a CSV file for continual training."""
+    os.makedirs("data", exist_ok=True)
+    file_location = "data/new_data.csv"
+    with open(file_location, "wb") as buffer:
+        shutil.copyfileobj(file.file, buffer)
+    return {"status": "success", "filename": file.filename}
+
+@app.get("/training_progress")
+def get_training_progress():
+    """Get the latest training progress metrics for the web interface."""
+    progress_file = "data/training_progress.json"
+    if not os.path.exists(progress_file):
+        return JSONResponse(content={"status": "no_progress", "message": "No training progress found."}, status_code=404)
+    with open(progress_file, "r") as f:
+        progress = json.load(f)
+    return JSONResponse(content=progress)
+
+@app.get("/dashboard", response_class=HTMLResponse)
+def dashboard():
+    html = '''
+    <!DOCTYPE html>
+    <html lang="en">
+    <head>
+        <meta charset="UTF-8">
+        <title>Training Progress Dashboard</title>
+        <style>
+            body { font-family: Arial, sans-serif; margin: 2em; background: #f9f9f9; }
+            h1 { color: #2c3e50; }
+            #progress { background: #fff; padding: 1em; border-radius: 8px; box-shadow: 0 2px 8px #eee; max-width: 400px; }
+            .label { color: #888; }
+        </style>
+    </head>
+    <body>
+        <h1>Training Progress</h1>
+        <div id="progress">
+            <div><span class="label">Epoch:</span> <span id="epoch">-</span></div>
+            <div><span class="label">Train Loss:</span> <span id="train_loss">-</span></div>
+            <div><span class="label">Val Loss:</span> <span id="val_loss">-</span></div>
+            <div><span class="label">Best Val Loss:</span> <span id="best_val_loss">-</span></div>
+            <div><span class="label">Last Updated:</span> <span id="timestamp">-</span></div>
+        </div>
+        <script>
+        async function fetchProgress() {
+            try {
+                const res = await fetch('/training_progress');
+                if (!res.ok) throw new Error('No progress yet');
+                const data = await res.json();
+                document.getElementById('epoch').textContent = data.epoch;
+                document.getElementById('train_loss').textContent = data.train_loss?.toFixed(4);
+                document.getElementById('val_loss').textContent = data.val_loss?.toFixed(4);
+                document.getElementById('best_val_loss').textContent = data.best_val_loss?.toFixed(4);
+                const date = new Date(data.timestamp * 1000);
+                document.getElementById('timestamp').textContent = date.toLocaleString();
+            } catch (e) {
+                document.getElementById('progress').innerHTML = '<b>No training progress yet.</b>';
+            }
+        }
+        fetchProgress();
+        setInterval(fetchProgress, 3000);
+        </script>
+    </body>
+    </html>
+    '''
+    return HTMLResponse(content=html)
+
+# Run with: uvicorn src.api:app --reload --host 0.0.0.0 --port 8000 

src/continual_train.py

@@ -0,0 +1,27 @@
+import os
+from src.train import train_vae
+import pandas as pd
+
+def continual_train(progress_callback=None):
+    """
+    Fine-tune the VAE on new data. Optionally log progress via progress_callback.
+    """
+    # Assume new data is already in data/new_data.csv and preprocessed
+    if not os.path.exists("data/new_data.csv"):
+        print("No new data found for continual training.")
+        return
+    # Optionally, preprocess new data if needed (skipped for simplicity)
+    # For now, just retrain on all processed data
+    print("Loading all processed data for fine-tuning...")
+    if os.path.exists("data/processed_patient_data.csv"):
+        feature_df = pd.read_csv("data/processed_patient_data.csv")
+        # Optionally, append new data
+        new_df = pd.read_csv("data/new_data.csv")
+        feature_df = pd.concat([feature_df, new_df], ignore_index=True)
+        feature_df.to_csv("data/processed_patient_data.csv", index=False)
+    else:
+        feature_df = pd.read_csv("data/new_data.csv")
+        feature_df.to_csv("data/processed_patient_data.csv", index=False)
+    print(f"Fine-tuning on {feature_df.shape[0]} samples...")
+    # Call train_vae with progress_callback
+    train_vae(progress_callback=progress_callback) 

src/continual_train_loop.py

@@ -0,0 +1,29 @@
+import time
+import os
+import json
+
+def continual_train_loop():
+    print("[Continual Training] Loop started. Waiting for new data...")
+    while True:
+        if os.path.exists("data/new_data.csv"):
+            print("[Continual Training] New data found! Fine-tuning model...")
+            from src.continual_train import continual_train
+            continual_train(progress_callback=log_training_progress)
+            os.remove("data/new_data.csv")
+            print("[Continual Training] Model updated and new data file removed.")
+        time.sleep(60)  # Check every 60 seconds
+
+def log_training_progress(epoch, train_loss, val_loss, best_val_loss):
+    progress = {
+        "epoch": epoch,
+        "train_loss": train_loss,
+        "val_loss": val_loss,
+        "best_val_loss": best_val_loss,
+        "timestamp": time.time()
+    }
+    os.makedirs("data", exist_ok=True)
+    with open("data/training_progress.json", "w") as f:
+        json.dump(progress, f)
+
+if __name__ == "__main__":
+    continual_train_loop() 

src/data_preprocessing.py

@@ -0,0 +1,122 @@
+# src/data_preprocessing.py - Convert patient data to numerical features
+import pandas as pd
+import numpy as np
+from sklearn.preprocessing import LabelEncoder, StandardScaler
+from datetime import datetime
+import joblib
+import os
+
+def preprocess_patient_data(csv_file="data/patient_data.csv"):
+    """
+    Convert patient CSV data to numerical features for VAE training
+    """
+    print("Loading and preprocessing patient data...")
+    
+    # Load data
+    df = pd.read_csv(csv_file)
+    print(f"Original data shape: {df.shape}")
+    
+    # Create numerical features
+    features = {}
+    
+    # 1. Age (already numerical)
+    features['age'] = df['Age'].values
+    
+    # 2. Gender (encode: Male=0, Female=1)
+    gender_encoder = LabelEncoder()
+    features['gender'] = gender_encoder.fit_transform(df['Gender'])
+    
+    # 3. Diagnosis (encode categorical)
+    diagnosis_encoder = LabelEncoder()
+    features['diagnosis'] = diagnosis_encoder.fit_transform(df['Diagnosis'])
+    
+    # 4. Blood Type (encode categorical)
+    blood_encoder = LabelEncoder()
+    features['blood_type'] = blood_encoder.fit_transform(df['BloodType'])
+    
+    # 5. Length of stay (calculate from admission/discharge dates)
+    df['AdmissionDate'] = pd.to_datetime(df['AdmissionDate'])
+    df['DischargeDate'] = pd.to_datetime(df['DischargeDate'])
+    features['length_of_stay'] = (df['DischargeDate'] - df['AdmissionDate']).dt.days
+    
+    # 6. Age group (create age categories)
+    age_bins = [0, 18, 35, 50, 65, 100]
+    age_labels = [0, 1, 2, 3, 4]
+    features['age_group'] = pd.cut(df['Age'], bins=age_bins, labels=age_labels, include_lowest=True).astype(int)
+    
+    # 7. Season of admission (extract from admission date)
+    features['admission_season'] = df['AdmissionDate'].dt.quarter - 1  # 0=Q1, 1=Q2, 2=Q3, 3=Q4
+    
+    # 8. Day of week admission (0=Monday, 6=Sunday)
+    features['admission_day'] = df['AdmissionDate'].dt.dayofweek
+    
+    # 9. Month of admission (0-11)
+    features['admission_month'] = df['AdmissionDate'].dt.month - 1
+    
+    # 10. Year of admission (normalized)
+    features['admission_year'] = df['AdmissionDate'].dt.year - 2020  # Normalize to 2020 as base
+    
+    # Convert to DataFrame
+    feature_df = pd.DataFrame(features)
+    
+    # Handle any missing values
+    feature_df = feature_df.fillna(feature_df.mean())
+    
+    print(f"Processed features shape: {feature_df.shape}")
+    print("Feature columns:", list(feature_df.columns))
+    
+    # Save encoders for later use
+    encoders = {
+        'gender': gender_encoder,
+        'diagnosis': diagnosis_encoder,
+        'blood_type': blood_encoder
+    }
+    
+    os.makedirs("models", exist_ok=True)
+    joblib.dump(encoders, 'models/encoders.pkl')
+    
+    # Save processed data
+    os.makedirs("data", exist_ok=True)
+    feature_df.to_csv('data/processed_patient_data.csv', index=False)
+    
+    print("Data preprocessing completed!")
+    print(f"Number of features: {feature_df.shape[1]}")
+    
+    return feature_df, encoders
+
+def create_sample_data_for_training():
+    """
+    Create a sample dataset if the original data is not available
+    """
+    print("Creating sample patient data for training...")
+    
+    np.random.seed(42)
+    n_samples = 1000
+    
+    # Generate realistic patient data
+    data = {
+        'age': np.random.normal(50, 20, n_samples).clip(1, 100),
+        'gender': np.random.choice([0, 1], n_samples),
+        'bmi': np.random.normal(25, 5, n_samples).clip(15, 50),
+        'blood_pressure': np.random.normal(120, 20, n_samples).clip(80, 200),
+        'diabetes': np.random.choice([0, 1], n_samples, p=[0.8, 0.2]),
+        'cholesterol': np.random.normal(200, 40, n_samples).clip(100, 300),
+        'heart_rate': np.random.normal(75, 15, n_samples).clip(40, 120)
+    }
+    
+    df = pd.DataFrame(data)
+    os.makedirs("data", exist_ok=True)
+    df.to_csv('data/patient_data.csv', index=False)
+    
+    print(f"Sample data created with {n_samples} patients")
+    return df
+
+if __name__ == "__main__":
+    try:
+        # Try to preprocess the real data
+        feature_df, encoders = preprocess_patient_data()
+        print("Successfully processed real patient data!")
+    except Exception as e:
+        print(f"Error processing real data: {e}")
+        print("Creating sample data instead...")
+        create_sample_data_for_training() 

src/model.py

@@ -0,0 +1,57 @@
+# Enhanced version with key improvements
+
+# model.py - Add validation and better loss
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+class TabularVAE(nn.Module):
+    def __init__(self, input_dim: int, hidden_dims=(64, 32), latent_dim=16):
+        super().__init__()
+        self.input_dim = input_dim
+        self.latent_dim = latent_dim
+        
+        # Encoder
+        dims = [input_dim, *hidden_dims]
+        self.encoder_layers = nn.ModuleList([
+            nn.Linear(dims[i], dims[i+1]) for i in range(len(dims)-1)
+        ])
+        self.fc_mu = nn.Linear(hidden_dims[-1], latent_dim)
+        self.fc_logvar = nn.Linear(hidden_dims[-1], latent_dim)
+        
+        # Decoder
+        dims_rev = [latent_dim, *reversed(hidden_dims)]
+        self.decoder_layers = nn.ModuleList([
+            nn.Linear(dims_rev[i], dims_rev[i+1]) for i in range(len(dims_rev)-1)
+        ])
+        self.output_layer = nn.Linear(hidden_dims[0], input_dim)
+        
+        # Add dropout for better generalization
+        self.dropout = nn.Dropout(0.1)
+
+    def encode(self, x):
+        h = x
+        for layer in self.encoder_layers:
+            h = F.relu(layer(h))
+            h = self.dropout(h)
+        mu = self.fc_mu(h)
+        logvar = self.fc_logvar(h)
+        return mu, logvar
+
+    def reparameterize(self, mu, logvar):
+        std = torch.exp(0.5 * logvar)
+        eps = torch.randn_like(std)
+        return mu + eps * std
+
+    def decode(self, z):
+        h = z
+        for layer in self.decoder_layers:
+            h = F.relu(layer(h))
+            h = self.dropout(h)
+        return self.output_layer(h)
+
+    def forward(self, x):
+        mu, logvar = self.encode(x)
+        z = self.reparameterize(mu, logvar)
+        recon = self.decode(z)
+        return recon, mu, logvar 

src/train.py

@@ -0,0 +1,158 @@
+# src/train.py - Enhanced training with validation for patient data
+import torch
+import torch.nn.functional as F
+from torch.utils.data import DataLoader, TensorDataset
+import pandas as pd
+import numpy as np
+from sklearn.preprocessing import StandardScaler
+from sklearn.model_selection import train_test_split
+from src.model import TabularVAE
+import joblib
+import os
+
+# Hyperparameters
+BATCH_SIZE = 32  # Smaller batch size for smaller dataset
+LR = 1e-3
+EPOCHS = 150  # More epochs for smaller dataset
+LATENT_DIM = 8  # Smaller latent dim for smaller dataset
+BETA = 1.0  # KL divergence weight
+
+def vae_loss(recon, x, mu, logvar, beta=1.0):
+    """Enhanced VAE loss with proper normalization"""
+    batch_size = x.size(0)
+    
+    # Reconstruction loss (MSE)
+    recon_loss = F.mse_loss(recon, x, reduction='sum') / batch_size
+    
+    # KL divergence loss
+    kld = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp()) / batch_size
+    
+    return recon_loss + beta * kld, recon_loss, kld
+
+def train_vae(progress_callback=None):
+    # Check if preprocessed data exists, if not create it
+    if not os.path.exists("data/processed_patient_data.csv"):
+        print("Preprocessed data not found. Running data preprocessing...")
+        from src.data_preprocessing import preprocess_patient_data
+        feature_df, encoders = preprocess_patient_data()
+    else:
+        print("Loading preprocessed data...")
+        feature_df = pd.read_csv("data/processed_patient_data.csv")
+    
+    print(f"Dataset shape: {feature_df.shape}")
+    print(f"Features: {list(feature_df.columns)}")
+    
+    # Handle missing values
+    feature_df = feature_df.fillna(feature_df.mean())
+    
+    # Split data
+    train_df, val_df = train_test_split(feature_df, test_size=0.2, random_state=42)
+    
+    # Scale data
+    scaler = StandardScaler()
+    train_data = scaler.fit_transform(train_df.values)
+    val_data = scaler.transform(val_df.values)
+    
+    print(f"Training data shape: {train_data.shape}")
+    print(f"Validation data shape: {val_data.shape}")
+    
+    # Create data loaders
+    train_tensor = torch.tensor(train_data, dtype=torch.float32)
+    val_tensor = torch.tensor(val_data, dtype=torch.float32)
+    
+    train_loader = DataLoader(TensorDataset(train_tensor), batch_size=BATCH_SIZE, shuffle=True)
+    val_loader = DataLoader(TensorDataset(val_tensor), batch_size=BATCH_SIZE, shuffle=False)
+    
+    # Initialize model with correct input dimension
+    input_dim = train_data.shape[1]
+    model = TabularVAE(input_dim=input_dim, latent_dim=LATENT_DIM, hidden_dims=(32, 16))
+    optimizer = torch.optim.Adam(model.parameters(), lr=LR)
+    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=15, factor=0.5)
+    
+    best_val_loss = float('inf')
+    patience_counter = 0
+    early_stopping_patience = 30
+    
+    print(f"Model initialized with {input_dim} input features and {LATENT_DIM} latent dimensions")
+    print(f"Training for {EPOCHS} epochs...")
+    
+    # Training loop
+    for epoch in range(EPOCHS):
+        # Training
+        model.train()
+        train_loss = 0
+        train_recon = 0
+        train_kld = 0
+        
+        for (batch,) in train_loader:
+            optimizer.zero_grad()
+            recon, mu, logvar = model(batch)
+            loss, recon_loss, kld_loss = vae_loss(recon, batch, mu, logvar, BETA)
+            loss.backward()
+            optimizer.step()
+            
+            train_loss += loss.item()
+            train_recon += recon_loss.item()
+            train_kld += kld_loss.item()
+        
+        # Validation
+        model.eval()
+        val_loss = 0
+        val_recon = 0
+        val_kld = 0
+        
+        with torch.no_grad():
+            for (batch,) in val_loader:
+                recon, mu, logvar = model(batch)
+                loss, recon_loss, kld_loss = vae_loss(recon, batch, mu, logvar, BETA)
+                
+                val_loss += loss.item()
+                val_recon += recon_loss.item()
+                val_kld += kld_loss.item()
+        
+        # Calculate averages
+        train_loss /= len(train_loader)
+        val_loss /= len(val_loader)
+        
+        # Learning rate scheduling
+        scheduler.step(val_loss)
+        
+        # Save best model
+        if val_loss < best_val_loss:
+            best_val_loss = val_loss
+            torch.save(model.state_dict(), "models/best_vae_model.pth")
+            patience_counter = 0
+        else:
+            patience_counter += 1
+        
+        # Early stopping
+        if patience_counter >= early_stopping_patience:
+            print(f"Early stopping at epoch {epoch+1}")
+            break
+        
+        # Print progress
+        if epoch % 10 == 0 or epoch == EPOCHS - 1:
+            print(f"Epoch {epoch+1}/{EPOCHS}")
+            print(f"Train Loss: {train_loss:.4f}, Val Loss: {val_loss:.4f}")
+            print(f"Train Recon: {train_recon:.4f}, Train KLD: {train_kld:.4f}")
+            print(f"LR: {optimizer.param_groups[0]['lr']:.6f}")
+        # Call progress callback if provided
+        if progress_callback:
+            progress_callback(epoch+1, train_loss, val_loss, best_val_loss)
+    
+    # Save final model and scaler
+    torch.save(model.state_dict(), "models/vae_model.pth")
+    joblib.dump(scaler, "models/scaler.pkl")
+    
+    # Save feature names for API
+    feature_names = list(feature_df.columns)
+    joblib.dump(feature_names, "models/feature_names.pkl")
+    
+    print("Training completed!")
+    print(f"Best validation loss: {best_val_loss:.4f}")
+    print(f"Model saved with {input_dim} input features")
+    
+    return model, scaler, feature_names
+
+if __name__ == "__main__":
+    model, scaler, feature_names = train_vae() 

src/web_scraper.py

@@ -0,0 +1,58 @@
+import requests
+import pandas as pd
+from bs4 import BeautifulSoup
+import os
+import time
+
+def scrape_table_from_url(url, session=None, table_index=0):
+    sess = session or requests.Session()
+    resp = sess.get(url)
+    soup = BeautifulSoup(resp.text, 'html.parser')
+    tables = soup.find_all('table')
+    if not tables:
+        print(f"No tables found at {url}")
+        return None
+    df = pd.read_html(str(tables[table_index]))[0]
+    return df
+
+def login_and_get_session(login_url, payload):
+    sess = requests.Session()
+    resp = sess.post(login_url, data=payload)
+    if resp.ok:
+        print("Login successful.")
+        return sess
+    else:
+        print("Login failed.")
+        return None
+
+def scrape_multiple_sources(sources, output_csv):
+    all_dfs = []
+    for src in sources:
+        if src.get('login_url'):
+            session = login_and_get_session(src['login_url'], src['login_payload'])
+        else:
+            session = None
+        df = scrape_table_from_url(src['url'], session=session, table_index=src.get('table_index', 0))
+        if df is not None:
+            all_dfs.append(df)
+    if all_dfs:
+        combined = pd.concat(all_dfs, ignore_index=True)
+        os.makedirs('data', exist_ok=True)
+        combined.to_csv(output_csv, index=False)
+        print(f"Combined data saved to {output_csv}")
+    else:
+        print("No data scraped.")
+
+def main_loop():
+    sources = [
+        {"url": "https://www.somepublichealthsite.org/table1.html"},
+        # Add more sources as needed
+    ]
+    while True:
+        print("[Web Scraper] Scraping new data...")
+        scrape_multiple_sources(sources, "data/new_data.csv")
+        print("[Web Scraper] Waiting 6 hours for next scrape...")
+        time.sleep(6 * 60 * 60)  # Wait 6 hours
+
+if __name__ == "__main__":
+    main_loop() 

tests/test_api.py

@@ -0,0 +1,27 @@
+import requests
+
+BASE_URL = "http://localhost:8000"
+
+def test_generate():
+    resp = requests.post(f"{BASE_URL}/generate", json={
+        "n_samples": 2,
+        "temperature": 1.0,
+        "random_seed": 42
+    })
+    print("/generate status:", resp.status_code)
+    print("/generate response:", resp.json())
+
+def test_encode():
+    resp = requests.post(f"{BASE_URL}/encode", json={
+        "age": 45,
+        "gender": "Male",
+        "diagnosis": "Diabetes",
+        "blood_type": "A+",
+        "length_of_stay": 7
+    })
+    print("/encode status:", resp.status_code)
+    print("/encode response:", resp.json())
+
+if __name__ == "__main__":
+    test_generate()
+    test_encode()