Update api/routes/patients.py

api/routes/patients.py

@@ -1,88 +1,384 @@
-from fastapi import APIRouter, HTTPException, Depends, Query, status
+from fastapi import APIRouter, HTTPException, Depends, Query, status, Body
 from db.mongo import patients_collection
 from core.security import get_current_user
-from utils.helpers import calculate_age
-from models.entities import PatientCreate, Note
+from utils.db import create_indexes
+from utils.helpers import calculate_age, standardize_language
+from models.entities import Note, PatientCreate
 from datetime import datetime
 from bson import ObjectId
 from bson.errors import InvalidId
-from typing import Optional, List
+from typing import Optional, List, Dict
+from pymongo import UpdateOne
+from pymongo.errors import BulkWriteError
+import json
+from pathlib import Path
+import glob
 import uuid
+import re
 import logging
+import time
+import os
 
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s'
+)
 logger = logging.getLogger(__name__)
+
 router = APIRouter()
 
+# Configuration
+BASE_DIR = Path(__file__).resolve().parent.parent.parent
+SYNTHEA_DATA_DIR = BASE_DIR / "output" / "fhir"
+os.makedirs(SYNTHEA_DATA_DIR, exist_ok=True)
+
 @router.post("/", status_code=status.HTTP_201_CREATED)
 async def create_patient(
     patient_data: PatientCreate,
     current_user: dict = Depends(get_current_user)
 ):
-    """Create a new patient"""
+    """Create a new patient in the database"""
+    logger.info(f"Creating new patient by user {current_user.get('email')}")
+    
     if current_user.get('role') not in ['admin', 'doctor']:
+        logger.warning(f"Unauthorized create attempt by {current_user.get('email')}")
         raise HTTPException(
             status_code=status.HTTP_403_FORBIDDEN,
-            detail="Only admins and doctors can create patients"
+            detail="Only administrators and doctors can create patients"
         )
-
+    
     try:
-        patient = patient_data.dict()
+        # Prepare the patient document
+        patient_doc = patient_data.dict()
         now = datetime.utcnow().isoformat()
         
-        patient.update({
+        # Add system-generated fields
+        patient_doc.update({
             "fhir_id": str(uuid.uuid4()),
             "import_date": now,
             "last_updated": now,
             "source": "manual",
             "created_by": current_user.get('email')
         })
+        
+        # Ensure arrays exist even if empty
+        for field in ['conditions', 'medications', 'encounters', 'notes']:
+            if field not in patient_doc:
+                patient_doc[field] = []
+        
+        # Insert the patient document
+        result = await patients_collection.insert_one(patient_doc)
+        
+        # Return the created patient with the generated ID
+        created_patient = await patients_collection.find_one(
+            {"_id": result.inserted_id}
+        )
+        
+        if not created_patient:
+            logger.error("Failed to retrieve created patient")
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail="Failed to retrieve created patient"
+            )
+        
+        created_patient["id"] = str(created_patient["_id"])
+        del created_patient["_id"]
+        
+        logger.info(f"Successfully created patient {created_patient['fhir_id']}")
+        return created_patient
+        
+    except Exception as e:
+        logger.error(f"Failed to create patient: {str(e)}")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Failed to create patient: {str(e)}"
+        )
 
-        result = await patients_collection.insert_one(patient)
-        created = await patients_collection.find_one({"_id": result.inserted_id})
+async def process_synthea_patient(bundle: dict, file_path: str) -> Optional[dict]:
+    logger.debug(f"Processing patient from file: {file_path}")
+    patient_data = {}
+    notes = []
+    conditions = []
+    medications = []
+    encounters = []
+    
+    # Validate bundle structure
+    if not isinstance(bundle, dict) or 'entry' not in bundle:
+        logger.error(f"Invalid FHIR bundle structure in {file_path}")
+        return None
+    
+    for entry in bundle.get('entry', []):
+        resource = entry.get('resource', {})
+        resource_type = resource.get('resourceType')
         
-        return {
-            "id": str(created["_id"]),
-            **{k: v for k, v in created.items() if k != "_id"}
-        }
+        if not resource_type:
+            logger.warning(f"Skipping entry with missing resourceType in {file_path}")
+            continue
+            
+        try:
+            if resource_type == 'Patient':
+                name = resource.get('name', [{}])[0]
+                address = resource.get('address', [{}])[0]
+                
+                # Construct full name and remove numbers
+                raw_full_name = f"{' '.join(name.get('given', ['']))} {name.get('family', '')}".strip()
+                clean_full_name = re.sub(r'\d+', '', raw_full_name).strip()
+                
+                patient_data = {
+                    'fhir_id': resource.get('id'),
+                    'full_name': clean_full_name,
+                    'gender': resource.get('gender', 'unknown'),
+                    'date_of_birth': resource.get('birthDate', ''),
+                    'address': ' '.join(address.get('line', [''])),
+                    'city': address.get('city', ''),
+                    'state': address.get('state', ''),
+                    'postal_code': address.get('postalCode', ''),
+                    'country': address.get('country', ''),
+                    'marital_status': resource.get('maritalStatus', {}).get('text', ''),
+                    'language': standardize_language(resource.get('communication', [{}])[0].get('language', {}).get('text', '')),
+                    'source': 'synthea',
+                    'last_updated': datetime.utcnow().isoformat()
+                }
+                
+            elif resource_type == 'Encounter':
+                encounter = {
+                    'id': resource.get('id'),
+                    'type': resource.get('type', [{}])[0].get('text', ''),
+                    'status': resource.get('status'),
+                    'period': resource.get('period', {}),
+                    'service_provider': resource.get('serviceProvider', {}).get('display', '')
+                }
+                encounters.append(encounter)
+                
+                for note in resource.get('note', []):
+                    if note.get('text'):
+                        notes.append({
+                            'date': resource.get('period', {}).get('start', datetime.utcnow().isoformat()),
+                            'type': resource.get('type', [{}])[0].get('text', 'Encounter Note'),
+                            'text': note.get('text'),
+                            'context': f"Encounter: {encounter.get('type')}",
+                            'author': 'System Generated'
+                        })
+                
+            elif resource_type == 'Condition':
+                conditions.append({
+                    'id': resource.get('id'),
+                    'code': resource.get('code', {}).get('text', ''),
+                    'status': resource.get('clinicalStatus', {}).get('text', ''),
+                    'onset_date': resource.get('onsetDateTime'),
+                    'recorded_date': resource.get('recordedDate'),
+                    'verification_status': resource.get('verificationStatus', {}).get('text', '')
+                })
+                
+            elif resource_type == 'MedicationRequest':
+                medications.append({
+                    'id': resource.get('id'),
+                    'name': resource.get('medicationCodeableConcept', {}).get('text', ''),
+                    'status': resource.get('status'),
+                    'prescribed_date': resource.get('authoredOn'),
+                    'requester': resource.get('requester', {}).get('display', ''),
+                    'dosage': resource.get('dosageInstruction', [{}])[0].get('text', '')
+                })
+                
+        except Exception as e:
+            logger.error(f"Error processing {resource_type} in {file_path}: {str(e)}")
+            continue
+    
+    if patient_data:
+        patient_data.update({
+            'notes': notes,
+            'conditions': conditions,
+            'medications': medications,
+            'encounters': encounters,
+            'import_date': datetime.utcnow().isoformat()
+        })
+        logger.info(f"Successfully processed patient {patient_data.get('fhir_id')} from {file_path}")
+        return patient_data
+    logger.warning(f"No valid patient data found in {file_path}")
+    return None
 
+@router.post("/import", status_code=status.HTTP_201_CREATED)
+async def import_patients(
+    limit: int = Query(100, ge=1, le=1000),
+    current_user: dict = Depends(get_current_user)
+):
+    request_id = str(uuid.uuid4())
+    logger.info(f"Starting import request {request_id} by user {current_user.get('email')}")
+    start_time = time.time()
+    
+    if current_user.get('role') not in ['admin', 'doctor']:
+        logger.warning(f"Unauthorized import attempt by {current_user.get('email')}")
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Only administrators and doctors can import data"
+        )
+    
+    try:
+        await create_indexes()
+        
+        if not SYNTHEA_DATA_DIR.exists():
+            logger.error(f"Synthea data directory not found: {SYNTHEA_DATA_DIR}")
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail="Data directory not found"
+            )
+            
+        # Filter out non-patient files
+        files = [
+            f for f in glob.glob(str(SYNTHEA_DATA_DIR / "*.json"))
+            if not re.search(r'(hospitalInformation|practitionerInformation)\d+\.json$', f)
+        ]
+        if not files:
+            logger.warning("No valid patient JSON files found in synthea data directory")
+            return {
+                "status": "success",
+                "message": "No patient data files found",
+                "imported": 0,
+                "request_id": request_id
+            }
+            
+        operations = []
+        imported = 0
+        errors = []
+        
+        for file_path in files[:limit]:
+            try:
+                logger.debug(f"Processing file: {file_path}")
+                
+                # Check file accessibility
+                if not os.path.exists(file_path):
+                    logger.error(f"File not found: {file_path}")
+                    errors.append(f"File not found: {file_path}")
+                    continue
+                    
+                # Check file size
+                file_size = os.path.getsize(file_path)
+                if file_size == 0:
+                    logger.warning(f"Empty file: {file_path}")
+                    errors.append(f"Empty file: {file_path}")
+                    continue
+                    
+                with open(file_path, 'r', encoding='utf-8') as f:
+                    try:
+                        bundle = json.load(f)
+                    except json.JSONDecodeError as je:
+                        logger.error(f"Invalid JSON in {file_path}: {str(je)}")
+                        errors.append(f"Invalid JSON in {file_path}: {str(je)}")
+                        continue
+                        
+                patient = await process_synthea_patient(bundle, file_path)
+                if patient:
+                    if not patient.get('fhir_id'):
+                        logger.warning(f"Missing FHIR ID in patient data from {file_path}")
+                        errors.append(f"Missing FHIR ID in {file_path}")
+                        continue
+                        
+                    operations.append(UpdateOne(
+                        {"fhir_id": patient['fhir_id']},
+                        {"$setOnInsert": patient},
+                        upsert=True
+                    ))
+                    imported += 1
+                else:
+                    logger.warning(f"No valid patient data in {file_path}")
+                    errors.append(f"No valid patient data in {file_path}")
+                    
+            except Exception as e:
+                logger.error(f"Error processing {file_path}: {str(e)}")
+                errors.append(f"Error in {file_path}: {str(e)}")
+                continue
+        
+        response = {
+            "status": "success",
+            "imported": imported,
+            "errors": errors,
+            "request_id": request_id,
+            "duration_seconds": time.time() - start_time
+        }
+        
+        if operations:
+            try:
+                result = await patients_collection.bulk_write(operations, ordered=False)
+                response.update({
+                    "upserted": result.upserted_count,
+                    "existing": len(operations) - result.upserted_count
+                })
+                logger.info(f"Import request {request_id} completed: {imported} patients processed, "
+                           f"{result.upserted_count} upserted, {len(errors)} errors")
+            except BulkWriteError as bwe:
+                logger.error(f"Partial bulk write failure for request {request_id}: {str(bwe.details)}")
+                response.update({
+                    "upserted": bwe.details.get('nUpserted', 0),
+                    "existing": len(operations) - bwe.details.get('nUpserted', 0),
+                    "write_errors": [
+                        f"Index {err['index']}: {err['errmsg']}" for err in bwe.details.get('writeErrors', [])
+                    ]
+                })
+                logger.info(f"Import request {request_id} partially completed: {imported} patients processed, "
+                           f"{response['upserted']} upserted, {len(errors)} errors")
+            except Exception as e:
+                logger.error(f"Bulk write failed for request {request_id}: {str(e)}")
+                raise HTTPException(
+                    status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                    detail=f"Database operation failed: {str(e)}"
+                )
+        else:
+            logger.info(f"Import request {request_id} completed: No new patients to import, {len(errors)} errors")
+            response["message"] = "No new patients found to import"
+            
+        return response
+        
+    except HTTPException:
+        raise
     except Exception as e:
-        logger.error(f"Create patient error: {str(e)}")
+        logger.error(f"Import request {request_id} failed: {str(e)}", exc_info=True)
         raise HTTPException(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail="Failed to create patient"
+            detail=f"Import failed: {str(e)}"
         )
 
 @router.get("/", response_model=List[dict])
 async def list_patients(
-    search: Optional[str] = None,
-    skip: int = 0,
-    limit: int = 100,
+    search: Optional[str] = Query(None),
+    min_notes: int = Query(0, ge=0),
+    min_conditions: int = Query(0, ge=0),
+    limit: int = Query(100, ge=1, le=500),
+    skip: int = Query(0, ge=0),
     current_user: dict = Depends(get_current_user)
 ):
-    """List patients with pagination and search"""
+    logger.info(f"Listing patients with search: {search}, limit: {limit}, skip: {skip}")
+    query = {}
+    
+    if search:
+        query["$or"] = [
+            {"full_name": {"$regex": search, "$options": "i"}},
+            {"fhir_id": search}
+        ]
+    
+    if min_notes > 0:
+        query[f"notes.{min_notes-1}"] = {"$exists": True}
+    
+    if min_conditions > 0:
+        query[f"conditions.{min_conditions-1}"] = {"$exists": True}
+    
     try:
-        query = {}
-        if search:
-            query["$or"] = [
-                {"full_name": {"$regex": search, "$options": "i"}},
-                {"fhir_id": search}
-            ]
-
         cursor = patients_collection.find(query).skip(skip).limit(limit)
         patients = []
         
-        async for doc in cursor:
-            doc["id"] = str(doc["_id"])
-            doc["age"] = calculate_age(doc.get("date_of_birth"))
-            patients.append(doc)
+        async for patient in cursor:
+            patient["id"] = str(patient["_id"])
+            patient["age"] = calculate_age(patient.get("date_of_birth"))
+            patients.append(patient)
         
+        logger.info(f"Retrieved {len(patients)} patients")
         return patients
-
+        
     except Exception as e:
-        logger.error(f"List patients error: {str(e)}")
+        logger.error(f"Failed to list patients: {str(e)}")
         raise HTTPException(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail="Failed to fetch patients"
+            detail=f"Failed to retrieve patients: {str(e)}"
         )
 
 @router.get("/{patient_id}", response_model=dict)
@@ -90,46 +386,56 @@ async def get_patient(
     patient_id: str,
     current_user: dict = Depends(get_current_user)
 ):
-    """Get patient by ID or FHIR ID"""
+    logger.info(f"Retrieving patient: {patient_id}")
     try:
-        # Try as ObjectId first
+        # First try to find by ObjectId
         try:
             patient = await patients_collection.find_one({"_id": ObjectId(patient_id)})
         except (InvalidId, ValueError):
-            # Fall back to FHIR ID
+            # If not valid ObjectId, try by fhir_id
             patient = await patients_collection.find_one({"fhir_id": patient_id})
-
+        
         if not patient:
-            raise HTTPException(status_code=404, detail="Patient not found")
-
+            logger.warning(f"Patient not found: {patient_id}")
+            raise HTTPException(
+                status_code=status.HTTP_404_NOT_FOUND,
+                detail="Patient not found"
+            )
+        
         patient["id"] = str(patient["_id"])
         patient["age"] = calculate_age(patient.get("date_of_birth"))
+        
+        logger.info(f"Successfully retrieved patient: {patient_id}")
         return patient
-
+        
     except Exception as e:
-        logger.error(f"Get patient error: {str(e)}")
+        logger.error(f"Failed to retrieve patient {patient_id}: {str(e)}")
         raise HTTPException(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail="Failed to fetch patient"
+            detail=f"Failed to retrieve patient: {str(e)}"
         )
 
 @router.post("/{patient_id}/notes", status_code=status.HTTP_201_CREATED)
-async def add_patient_note(
+async def add_note(
     patient_id: str,
     note: Note,
     current_user: dict = Depends(get_current_user)
 ):
-    """Add a note to patient record"""
+    logger.info(f"Adding note for patient {patient_id} by user {current_user.get('email')}")
     if current_user.get('role') not in ['doctor', 'admin']:
-        raise HTTPException(status_code=403, detail="Only clinicians can add notes")
-
+        logger.warning(f"Unauthorized note addition attempt by {current_user.get('email')}")
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Only clinicians can add notes"
+        )
+    
     try:
         note_data = note.dict()
         note_data.update({
-            "author": current_user.get('email'),
+            "author": current_user.get('full_name', 'System'),
             "timestamp": datetime.utcnow().isoformat()
         })
-
+        
         result = await patients_collection.update_one(
             {"$or": [
                 {"_id": ObjectId(patient_id)},
@@ -140,17 +446,29 @@ async def add_patient_note(
                 "$set": {"last_updated": datetime.utcnow().isoformat()}
             }
         )
-
+        
         if result.modified_count == 0:
-            raise HTTPException(status_code=404, detail="Patient not found")
-
-        return {"status": "success"}
-
+            logger.warning(f"Patient not found for note addition: {patient_id}")
+            raise HTTPException(
+                status_code=status.HTTP_404_NOT_FOUND,
+                detail="Patient not found"
+            )
+        
+        logger.info(f"Note added successfully for patient {patient_id}")
+        return {"status": "success", "message": "Note added"}
+        
+    except ValueError as ve:
+        logger.error(f"Invalid patient ID format: {patient_id}, error: {str(ve)}")
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Invalid patient ID format"
+        )
     except Exception as e:
-        logger.error(f"Add note error: {str(e)}")
+        logger.error(f"Failed to add note for patient {patient_id}: {str(e)}")
         raise HTTPException(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail="Failed to add note"
+            detail=f"Failed to add note: {str(e)}"
         )
 
+# Export the router as 'patients' for api.__init__.py
 patients = router