Update api/routes.py

api/routes.py

@@ -5,7 +5,7 @@ from db.mongo import users_collection, patients_collection, appointments_collect
 from core.security import hash_password, verify_password, create_access_token, get_current_user
 from datetime import datetime, timedelta
 from bson import ObjectId
-from typing import Optional, List
+from typing import Optional, List, Dict
 from pydantic import BaseModel, Field
 from pymongo import UpdateOne, InsertOne, IndexModel
 import httpx
@@ -14,11 +14,21 @@ import json
 from pathlib import Path
 import glob
 from motor.motor_asyncio import AsyncIOMotorClient
+import logging
+import time
+import uuid
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
 
 router = APIRouter()
 
 # Configuration
-BASE_DIR = Path(__file__).resolve().parent.parent  # goes up from /api to project root
+BASE_DIR = Path(__file__).resolve().parent.parent
 SYNTHEA_DATA_DIR = BASE_DIR / "output" / "fhir"
 os.makedirs(SYNTHEA_DATA_DIR, exist_ok=True)
 
@@ -46,12 +56,17 @@ class PatientUpdate(BaseModel):
 
 # Indexes
 async def create_indexes():
-    await patients_collection.create_indexes([
-        IndexModel([("fhir_id", 1)], unique=True),
-        IndexModel([("full_name", "text")]),
-        IndexModel([("date_of_birth", 1)]),
-        IndexModel([("notes.date", -1)])
-    ])
+    try:
+        await patients_collection.create_indexes([
+            IndexModel([("fhir_id", 1)], unique=True),
+            IndexModel([("full_name", "text")]),
+            IndexModel([("date_of_birth", 1)]),
+            IndexModel([("notes.date", -1)])
+        ])
+        logger.info("Database indexes created successfully")
+    except Exception as e:
+        logger.error(f"Failed to create indexes: {str(e)}")
+        raise
 
 # Helper Functions
 def calculate_age(birth_date: str) -> Optional[int]:
@@ -62,79 +77,95 @@ def calculate_age(birth_date: str) -> Optional[int]:
         today = datetime.now()
         return today.year - birth_date.year - (
             (today.month, today.day) < (birth_date.month, birth_date.day))
-    except ValueError:
+    except ValueError as e:
+        logger.warning(f"Invalid birth date format: {birth_date}, error: {str(e)}")
         return None
 
-async def process_synthea_patient(bundle: dict) -> Optional[dict]:
+async def process_synthea_patient(bundle: dict, file_path: str) -> Optional[dict]:
+    logger.debug(f"Processing patient from file: {file_path}")
     patient_data = {}
     notes = []
     conditions = []
     medications = []
     encounters = []
     
+    # Validate bundle structure
+    if not isinstance(bundle, dict) or 'entry' not in bundle:
+        logger.error(f"Invalid FHIR bundle structure in {file_path}")
+        return None
+    
     for entry in bundle.get('entry', []):
         resource = entry.get('resource', {})
         resource_type = resource.get('resourceType')
         
-        if resource_type == 'Patient':
-            name = resource.get('name', [{}])[0]
-            address = resource.get('address', [{}])[0]
+        if not resource_type:
+            logger.warning(f"Skipping entry with missing resourceType in {file_path}")
+            continue
             
-            patient_data = {
-                'fhir_id': resource.get('id'),
-                'full_name': f"{' '.join(name.get('given', ['']))} {name.get('family', '')}".strip(),
-                'gender': resource.get('gender', 'unknown'),
-                'date_of_birth': resource.get('birthDate', ''),
-                'address': ' '.join(address.get('line', [''])),
-                'city': address.get('city', ''),
-                'state': address.get('state', ''),
-                'postal_code': address.get('postalCode', ''),
-                'country': address.get('country', ''),
-                'marital_status': resource.get('maritalStatus', {}).get('text', ''),
-                'language': resource.get('communication', [{}])[0].get('language', {}).get('text', ''),
-                'source': 'synthea',
-                'last_updated': datetime.utcnow()
-            }
-        
-        elif resource_type == 'Encounter':
-            encounter = {
-                'id': resource.get('id'),
-                'type': resource.get('type', [{}])[0].get('text', ''),
-                'status': resource.get('status'),
-                'period': resource.get('period', {}),
-                'service_provider': resource.get('serviceProvider', {}).get('display', '')
-            }
-            encounters.append(encounter)
-            
-            for note in resource.get('note', []):
-                if note.get('text'):
-                    notes.append({
-                        'date': resource.get('period', {}).get('start', datetime.utcnow().isoformat()),
-                        'type': resource.get('type', [{}])[0].get('text', 'Encounter Note'),
-                        'text': note.get('text'),
-                        'context': f"Encounter: {encounter.get('type')}",
-                        'author': 'System Generated'
-                    })
-        
-        elif resource_type == 'Condition':
-            conditions.append({
-                'id': resource.get('id'),
-                'code': resource.get('code', {}).get('text', ''),
-                'status': resource.get('clinicalStatus', {}).get('text', ''),
-                'onset_date': resource.get('onsetDateTime'),
-                'recorded_date': resource.get('recordedDate'),
-                'verification_status': resource.get('verificationStatus', {}).get('text', '')
-            })
-        
-        elif resource_type == 'MedicationRequest':
-            medications.append({
-                'id': resource.get('id'),
-                'name': resource.get('medicationCodeableConcept', {}).get('text', ''),
-                'status': resource.get('status'),
-                'prescribed_date': resource.get('authoredOn'),
-                'requester': resource.get('requester', {}).get('display', ''),
-                'dosage': resource.get('dosageInstruction', [{}])[0].get('text', '')
-            })
+        try:
+            if resource_type == 'Patient':
+                name = resource.get('name', [{}])[0]
+                address = resource.get('address', [{}])[0]
+                
+                patient_data = {
+                    'fhir_id': resource.get('id'),
+                    'full_name': f"{' '.join(name.get('given', ['']))} {name.get('family', '')}".strip(),
+                    'gender': resource.get('gender', 'unknown'),
+                    'date_of_birth': resource.get('birthDate', ''),
+                    'address': ' '.join(address.get('line', [''])),
+                    'city': address.get('city', ''),
+                    'state': address.get('state', ''),
+                    'postal_code': address.get('postalCode', ''),
+                    'country': address.get('country', ''),
+                    'marital_status': resource.get('maritalStatus', {}).get('text', ''),
+                    'language': resource.get('communication', [{}])[0].get('language', {}).get('text', ''),
+                    'source': 'synthea',
+                    'last_updated': datetime.utcnow()
+                }
+                
+            elif resource_type == 'Encounter':
+                encounter = {
+                    'id': resource.get('id'),
+                    'type': resource.get('type', [{}])[0].get('text', ''),
+                    'status': resource.get('status'),
+                    'period': resource.get('period', {}),
+                    'service_provider': resource.get('serviceProvider', {}).get('display', '')
+                }
+                encounters.append(encounter)
+                
+                for note in resource.get('note', []):
+                    if note.get('text'):
+                        notes.append({
+                            'date': resource.get('period', {}).get('start', datetime.utcnow().isoformat()),
+                            'type': resource.get('type', [{}])[0].get('text', 'Encounter Note'),
+                            'text': note.get('text'),
+                            'context': f"Encounter: {encounter.get('type')}",
+                            'author': 'System Generated'
+                        })
+                
+            elif resource_type == 'Condition':
+                conditions.append({
+                    'id': resource.get('id'),
+                    'code': resource.get('code', {}).get('text', ''),
+                    'status': resource.get('clinicalStatus', {}).get('text', ''),
+                    'onset_date': resource.get('onsetDateTime'),
+                    'recorded_date': resource.get('recordedDate'),
+                    'verification_status': resource.get('verificationStatus', {}).get('text', '')
+                })
+                
+            elif resource_type == 'MedicationRequest':
+                medications.append({
+                    'id': resource.get('id'),
+                    'name': resource.get('medicationCodeableConcept', {}).get('text', ''),
+                    'status': resource.get('status'),
+                    'prescribed_date': resource.get('authoredOn'),
+                    'requester': resource.get('requester', {}).get('display', ''),
+                    'dosage': resource.get('dosageInstruction', [{}])[0].get('text', '')
+                })
+                
+        except Exception as e:
+            logger.error(f"Error processing {resource_type} in {file_path}: {str(e)}")
+            continue
     
     if patient_data:
         patient_data.update({
@@ -144,7 +175,9 @@ async def process_synthea_patient(bundle: dict) -> Optional[dict]:
             'encounters': encounters,
             'import_date': datetime.utcnow()
         })
+        logger.info(f"Successfully processed patient {patient_data.get('fhir_id')} from {file_path}")
         return patient_data
+    logger.warning(f"No valid patient data found in {file_path}")
     return None
 
 # Routes
@@ -153,7 +186,12 @@ async def import_patients(
     limit: int = Query(100, ge=1, le=1000),
     current_user: dict = Depends(get_current_user)
 ):
+    request_id = str(uuid.uuid4())
+    logger.info(f"Starting import request {request_id} by user {current_user.get('email')}")
+    start_time = time.time()
+    
     if current_user.get('role') not in ['admin', 'doctor']:
+        logger.warning(f"Unauthorized import attempt by {current_user.get('email')}")
         raise HTTPException(
             status_code=status.HTTP_403_FORBIDDEN,
             detail="Only administrators and doctors can import data"
@@ -161,36 +199,108 @@ async def import_patients(
     
     try:
         await create_indexes()
+        
+        if not SYNTHEA_DATA_DIR.exists():
+            logger.error(f"Synthea data directory not found: {SYNTHEA_DATA_DIR}")
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail="Data directory not found"
+            )
+            
         files = glob.glob(str(SYNTHEA_DATA_DIR / "*.json"))
+        if not files:
+            logger.warning("No JSON files found in synthea data directory")
+            return {
+                "status": "success",
+                "message": "No data files found",
+                "imported": 0,
+                "request_id": request_id
+            }
+            
         operations = []
         imported = 0
+        errors = []
         
         for file_path in files[:limit]:
             try:
-                with open(file_path, 'r') as f:
-                    bundle = json.load(f)
-                    patient = await process_synthea_patient(bundle)
-                    if patient:
-                        operations.append(UpdateOne(
-                            {"fhir_id": patient['fhir_id']},
-                            {"$setOnInsert": patient},
-                            upsert=True
-                        ))
-                        imported += 1
+                logger.debug(f"Processing file: {file_path}")
+                
+                # Check file accessibility
+                if not os.path.exists(file_path):
+                    logger.error(f"File not found: {file_path}")
+                    errors.append(f"File not found: {file_path}")
+                    continue
+                    
+                # Check file size
+                file_size = os.path.getsize(file_path)
+                if file_size == 0:
+                    logger.warning(f"Empty file: {file_path}")
+                    errors.append(f"Empty file: {file_path}")
+                    continue
+                    
+                with open(file_path, 'r', encoding='utf-8') as f:
+                    try:
+                        bundle = json.load(f)
+                    except json.JSONDecodeError as je:
+                        logger.error(f"Invalid JSON in {file_path}: {str(je)}")
+                        errors.append(f"Invalid JSON in {file_path}: {str(je)}")
+                        continue
+                        
+                patient = await process_synthea_patient(bundle, file_path)
+                if patient:
+                    if not patient.get('fhir_id'):
+                        logger.warning(f"Missing FHIR ID in patient data from {file_path}")
+                        errors.append(f"Missing FHIR ID in {file_path}")
+                        continue
+                        
+                    operations.append(UpdateOne(
+                        {"fhir_id": patient['fhir_id']},
+                        {"$setOnInsert": patient},
+                        upsert=True
+                    ))
+                    imported += 1
+                else:
+                    logger.warning(f"No valid patient data in {file_path}")
+                    errors.append(f"No valid patient data in {file_path}")
+                    
             except Exception as e:
+                logger.error(f"Error processing {file_path}: {str(e)}")
+                errors.append(f"Error in {file_path}: {str(e)}")
                 continue
         
+        response = {
+            "status": "success",
+            "imported": imported,
+            "errors": errors,
+            "request_id": request_id,
+            "duration_seconds": time.time() - start_time
+        }
+        
         if operations:
-            result = await patients_collection.bulk_write(operations)
-            return {
-                "status": "success",
-                "imported": imported,
-                "upserted": result.upserted_count,
-                "existing": len(operations) - result.upserted_count
-            }
-        return {"status": "success", "message": "No new patients found to import"}
-    
+            try:
+                result = await patients_collection.bulk_write(operations)
+                response.update({
+                    "upserted": result.upserted_count,
+                    "existing": len(operations) - result.upserted_count
+                })
+                logger.info(f"Import request {request_id} completed: {imported} patients processed, "
+                           f"{result.upserted_count} upserted, {len(errors)} errors")
+            except Exception as e:
+                logger.error(f"Bulk write failed for request {request_id}: {str(e)}")
+                raise HTTPException(
+                    status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                    detail=f"Database operation failed: {str(e)}"
+                )
+        else:
+            logger.info(f"Import request {request_id} completed: No new patients to import, {len(errors)} errors")
+            response["message"] = "No new patients found to import"
+            
+        return response
+        
+    except HTTPException:
+        raise
     except Exception as e:
+        logger.error(f"Import request {request_id} failed: {str(e)}", exc_info=True)
         raise HTTPException(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
             detail=f"Import failed: {str(e)}"
@@ -204,6 +314,7 @@ async def list_patients(
     limit: int = Query(100, ge=1, le=500),
     skip: int = Query(0, ge=0)
 ):
+    logger.info(f"Listing patients with search: {search}, limit: {limit}, skip: {skip}")
     query = {"source": "synthea"}
     
     if search:
@@ -228,27 +339,37 @@ async def list_patients(
         "medications": {"$slice": 1}
     }
     
-    cursor = patients_collection.find(query, projection).skip(skip).limit(limit)
-    patients = []
-    
-    async for patient in cursor:
-        patients.append({
-            "id": str(patient["_id"]),
-            "fhir_id": patient.get("fhir_id"),
-            "full_name": patient.get("full_name"),
-            "gender": patient.get("gender"),
-            "age": calculate_age(patient.get("date_of_birth")),
-            "stats": {
-                "notes": len(patient.get("notes", [])),
-                "conditions": len(patient.get("conditions", [])),
-                "medications": len(patient.get("medications", []))
-            }
-        })
-    
-    return patients
+    try:
+        cursor = patients_collection.find(query, projection).skip(skip).limit(limit)
+        patients = []
+        
+        async for patient in cursor:
+            patients.append({
+                "id": str(patient["_id"]),
+                "fhir_id": patient.get("fhir_id"),
+                "full_name": patient.get("full_name"),
+                "gender": patient.get("gender"),
+                "age": calculate_age(patient.get("date_of_birth")),
+                "stats": {
+                    "notes": len(patient.get("notes", [])),
+                    "conditions": len(patient.get("conditions", [])),
+                    "medications": len(patient.get("medications", []))
+                }
+            })
+        
+        logger.info(f"Retrieved {len(patients)} patients")
+        return patients
+        
+    except Exception as e:
+        logger.error(f"Failed to list patients: {str(e)}")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Failed to retrieve patients: {str(e)}"
+        )
 
 @router.get("/ehr/patients/{patient_id}", response_model=dict)
 async def get_patient(patient_id: str):
+    logger.info(f"Retrieving patient: {patient_id}")
     try:
         patient = await patients_collection.find_one({
             "$or": [
@@ -258,12 +379,13 @@ async def get_patient(patient_id: str):
         })
         
         if not patient:
+            logger.warning(f"Patient not found: {patient_id}")
             raise HTTPException(
                 status_code=status.HTTP_404_NOT_FOUND,
                 detail="Patient not found"
             )
         
-        return {
+        response = {
             "demographics": {
                 "id": str(patient["_id"]),
                 "fhir_id": patient.get("fhir_id"),
@@ -293,11 +415,21 @@ async def get_patient(patient_id: str):
                 "last_updated": patient.get("last_updated")
             }
         }
-    
+        
+        logger.info(f"Successfully retrieved patient: {patient_id}")
+        return response
+        
+    except ValueError as ve:
+        logger.error(f"Invalid patient ID format: {patient_id}, error: {str(ve)}")
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Invalid patient ID format"
+        )
     except Exception as e:
+        logger.error(f"Failed to retrieve patient {patient_id}: {str(e)}")
         raise HTTPException(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=str(e)
+            detail=f"Failed to retrieve patient: {str(e)}"
         )
 
 @router.post("/ehr/patients/{patient_id}/notes", status_code=status.HTTP_201_CREATED)
@@ -306,7 +438,9 @@ async def add_note(
     note: Note,
     current_user: dict = Depends(get_current_user)
 ):
+    logger.info(f"Adding note for patient {patient_id} by user {current_user.get('email')}")
     if current_user.get('role') not in ['doctor', 'admin']:
+        logger.warning(f"Unauthorized note addition attempt by {current_user.get('email')}")
         raise HTTPException(
             status_code=status.HTTP_403_FORBIDDEN,
             detail="Only clinicians can add notes"
@@ -331,25 +465,36 @@ async def add_note(
         )
         
         if result.modified_count == 0:
+            logger.warning(f"Patient not found for note addition: {patient_id}")
             raise HTTPException(
                 status_code=status.HTTP_404_NOT_FOUND,
                 detail="Patient not found"
             )
         
+        logger.info(f"Note added successfully for patient {patient_id}")
         return {"status": "success", "message": "Note added"}
-    
+        
+    except ValueError as ve:
+        logger.error(f"Invalid patient ID format: {patient_id}, error: {str(ve)}")
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Invalid patient ID format"
+        )
     except Exception as e:
+        logger.error(f"Failed to add note for patient {patient_id}: {str(e)}")
         raise HTTPException(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=str(e)
+            detail=f"Failed to add note: {str(e)}"
         )
 
 # Original Auth Routes (fully implemented)
 @router.post("/signup", status_code=status.HTTP_201_CREATED)
 async def signup(data: SignupForm):
+    logger.info(f"Signup attempt for email: {data.email}")
     email = data.email.lower().strip()
     existing = await users_collection.find_one({"email": email})
     if existing:
+        logger.warning(f"Signup failed: Email already exists: {email}")
         raise HTTPException(
             status_code=status.HTTP_409_CONFLICT,
             detail="Email already exists"
@@ -365,26 +510,38 @@ async def signup(data: SignupForm):
         "updated_at": datetime.utcnow()
     }
     
-    result = await users_collection.insert_one(user_doc)
-    return {
-        "status": "success",
-        "id": str(result.inserted_id),
-        "email": email
-    }
+    try:
+        result = await users_collection.insert_one(user_doc)
+        logger.info(f"User created successfully: {email}")
+        return {
+            "status": "success",
+            "id": str(result.inserted_id),
+            "email": email
+        }
+    except Exception as e:
+        logger.error(f"Failed to create user {email}: {str(e)}")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Failed to create user: {str(e)}"
+        )
 
 @router.post("/admin/doctors", status_code=status.HTTP_201_CREATED)
 async def create_doctor(
     data: DoctorCreate,
     current_user: dict = Depends(get_current_user)
 ):
+    logger.info(f"Doctor creation attempt by {current_user.get('email')}")
     if current_user.get('role') != 'admin':
+        logger.warning(f"Unauthorized doctor creation attempt by {current_user.get('email')}")
         raise HTTPException(
             status_code=status.HTTP_403_FORBIDDEN,
             detail="Only admins can create doctor accounts"
         )
     
-    existing = await users_collection.find_one({"email": data.email.lower()})
+    email = data.email.lower().strip()
+    existing = await users_collection.find_one({"email": email})
     if existing:
+        logger.warning(f"Doctor creation failed: Email already exists: {email}")
         raise HTTPException(
             status_code=status.HTTP_409_CONFLICT,
             detail="Email already exists"
@@ -392,7 +549,7 @@ async def create_doctor(
     
     hashed_pw = hash_password(data.password)
     doctor_doc = {
-        "email": data.email.lower().strip(),
+        "email": email,
         "full_name": data.full_name.strip(),
         "password": hashed_pw,
         "role": "doctor",
@@ -402,17 +559,27 @@ async def create_doctor(
         "updated_at": datetime.utcnow()
     }
     
-    result = await users_collection.insert_one(doctor_doc)
-    return {
-        "status": "success",
-        "id": str(result.inserted_id),
-        "email": data.email
-    }
+    try:
+        result = await users_collection.insert_one(doctor_doc)
+        logger.info(f"Doctor created successfully: {email}")
+        return {
+            "status": "success",
+            "id": str(result.inserted_id),
+            "email": email
+        }
+    except Exception as e:
+        logger.error(f"Failed to create doctor {email}: {str(e)}")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Failed to create doctor: {str(e)}"
+        )
 
 @router.post("/login", response_model=TokenResponse)
 async def login(form_data: OAuth2PasswordRequestForm = Depends()):
+    logger.info(f"Login attempt for email: {form_data.username}")
     user = await users_collection.find_one({"email": form_data.username.lower()})
     if not user or not verify_password(form_data.password, user["password"]):
+        logger.warning(f"Login failed for {form_data.username}: Invalid credentials")
         raise HTTPException(
             status_code=status.HTTP_401_UNAUTHORIZED,
             detail="Invalid credentials",
@@ -420,6 +587,7 @@ async def login(form_data: OAuth2PasswordRequestForm = Depends()):
         )
     
     access_token = create_access_token(data={"sub": user["email"]})
+    logger.info(f"Successful login for {form_data.username}")
     return {
         "access_token": access_token,
         "token_type": "bearer",
@@ -428,14 +596,16 @@ async def login(form_data: OAuth2PasswordRequestForm = Depends()):
 
 @router.get("/me")
 async def get_me(current_user: dict = Depends(get_current_user)):
+    logger.info(f"Fetching user profile for {current_user['email']}")
     user = await users_collection.find_one({"email": current_user["email"]})
     if not user:
+        logger.warning(f"User not found: {current_user['email']}")
         raise HTTPException(
             status_code=status.HTTP_404_NOT_FOUND,
             detail="User not found"
         )
     
-    return {
+    response = {
         "id": str(user["_id"]),
         "email": user["email"],
         "full_name": user.get("full_name", ""),
@@ -443,4 +613,6 @@ async def get_me(current_user: dict = Depends(get_current_user)):
         "specialty": user.get("specialty"),
         "created_at": user.get("created_at"),
         "updated_at": user.get("updated_at")
-    }
+    }
+    logger.info(f"User profile retrieved for {current_user['email']}")
+    return response