fix: Robust JSON parsing and field mapping for report generation

Frontend improvements:
- Added comprehensive JSON parsing with multiple fallback strategies
- Handle both object and string responses from backend
- Added field-level validation to ensure each report field gets distinct data
- Warning if colposcopicFindings contains JSON (indicates parsing failure)
- Detailed console logging for debugging field population

Backend improvements:
- Validate AI response is valid JSON before returning
- Strip markdown code fences in backend (before sending to frontend)
- Better error messages for JSON parsing failures
- Log parsed JSON keys to verify structure

This ensures report data populates correctly into individual form fields.

backend/app.py

@@ -13,6 +13,7 @@ import uvicorn
 import traceback
 import json
 from typing import List, Dict, Optional
+import re
 
 # Load .env file for local development.
 # Search from this file's directory upward so it works whether the server
@@ -394,11 +395,29 @@ Do NOT include any other keys. Do NOT wrap in markdown. Return raw JSON only."""
         if not response_text:
             raise Exception("All model attempts failed. Please check API key and model availability.")
 
-        return JSONResponse({
-            "status": "success",
-            "report": response_text,
-            "model": used_model
-        })
+        # Ensure response_text is valid JSON before returning
+        try:
+            # Strip markdown if present
+            cleaned_text = response_text.strip()
+            if cleaned_text.startswith('```'):
+                cleaned_text = re.sub(r'^```[a-z]*\n?', '', cleaned_text, flags=re.IGNORECASE)
+                cleaned_text = re.sub(r'\n?```\s*$', '', cleaned_text)
+                cleaned_text = cleaned_text.strip()
+            
+            # Parse to verify it's valid JSON
+            parsed_json = json.loads(cleaned_text)
+            print(f"✅ Report is valid JSON with keys: {list(parsed_json.keys())}")
+            
+            # Return as JSON object (not string) so it's properly encoded by FastAPI
+            return JSONResponse({
+                "status": "success",
+                "report": cleaned_text,  # Return the cleaned JSON string
+                "model": used_model
+            })
+        except json.JSONDecodeError as je:
+            print(f"⚠️ Response is not valid JSON: {je}")
+            print(f"Response text: {response_text[:500]}")
+            raise Exception(f"Gemini returned invalid JSON: {str(je)}")
 
     except Exception as e:
         error_msg = str(e)

src/pages/ReportPage.tsx

@@ -206,38 +206,61 @@ export function ReportPage({
         throw new Error('Invalid response from backend');
       }
 
-      // Strip possible markdown code fences before parsing
-      let reportText: string = data.report.trim();
-      console.log('[ReportPage] Raw AI response:', reportText.substring(0, 200));
-      
-      if (reportText.startsWith('```')) {
-        reportText = reportText.replace(/^```[a-z]*\n?/i, '').replace(/```\s*$/, '').trim();
-        console.log('[ReportPage] Stripped markdown, new length:', reportText.length);
-      }
-
-      // Parse structured JSON returned by Gemini
+      // Get the report text - it should be a JSON string
+      let reportText: string = String(data.report).trim();
       let parsed: Record<string, string> = {};
-      try {
-        parsed = JSON.parse(reportText);
-        console.log('[ReportPage] Successfully parsed JSON with keys:', Object.keys(parsed));
-      } catch (parseErr) {
-        console.error('JSON parsing failed. Attempting to extract JSON from response...', parseErr);
+      
+      console.log('[ReportPage] Raw response type:', typeof data.report);
+      console.log('[ReportPage] Raw response preview:', reportText.substring(0, 300));
+      
+      // If response is already an object (shouldn't happen but handle it)
+      if (typeof data.report === 'object' && data.report !== null) {
+        console.log('[ReportPage] Response is already an object, using directly');
+        parsed = data.report;
+      } else {
+        // Response is a string - need to parse it
+        reportText = String(data.report).trim();
+        
+        // Remove markdown code fence wrappers if present
+        if (reportText.startsWith('```')) {
+          reportText = reportText.replace(/^```(?:json)?\n/i, '').replace(/\n```\s*$/i, '').trim();
+          console.log('[ReportPage] Removed markdown fence');
+        }
+        
+        // Clean up any escape sequences
+        reportText = reportText.replace(/\\n/g, '\n').replace(/\\t/g, '\t');
         
-        // Try to find JSON object in the response
-        const jsonMatch = reportText.match(/\{[\s\S]*\}/);
-        if (jsonMatch) {
-          try {
-            parsed = JSON.parse(jsonMatch[0]);
-            console.log('[ReportPage] Extracted JSON from response:', Object.keys(parsed));
-          } catch (innerErr) {
-            console.error('Failed to extract JSON:', innerErr);
-            throw new Error('Could not parse AI response as valid JSON');
+        console.log('[ReportPage] Cleaned text preview:', reportText.substring(0, 200));
+        
+        // Try direct JSON parse first
+        try {
+          parsed = JSON.parse(reportText);
+          console.log('[ReportPage] ✅ Direct parse successful, keys:', Object.keys(parsed));
+        } catch (directErr) {
+          console.log('[ReportPage] Direct parse failed, attempting JSON extraction...');
+          
+          // Try to find and extract the JSON object from the string
+          const jsonMatch = reportText.match(/\{[\s\S]*\}/);
+          if (jsonMatch) {
+            try {
+              parsed = JSON.parse(jsonMatch[0]);
+              console.log('[ReportPage] ✅ Extracted JSON successful, keys:', Object.keys(parsed));
+            } catch (extractErr) {
+              console.error('[ReportPage] JSON extraction failed:', extractErr);
+              throw new Error('Could not parse AI response as valid JSON');
+            }
+          } else {
+            throw new Error('No JSON object found in response');
           }
-        } else {
-          throw new Error('No JSON object found in AI response');
         }
       }
 
+      // At this point, parsed should be an object with the report fields
+      // Validate structure
+      if (!parsed || typeof parsed !== 'object') {
+        throw new Error('Parsed response is not a valid object');
+      }
+
       // Validate that all required keys are present
       const requiredKeys = ['examQuality', 'transformationZone', 'acetowL', 'nativeFindings', 
                            'aceticFindings', 'biopsySites', 'biopsyNotes', 'colposcopicFindings', 
@@ -262,19 +285,24 @@ export function ReportPage({
         biopsyNotes: String(parsed.biopsyNotes || '').trim(),
       };
 
-      console.log('[ReportPage] AI generated fields:', {
-        examQuality: updatedFormData.examQuality.substring(0, 30),
-        transformationZone: updatedFormData.transformationZone.substring(0, 30),
-        acetowL: updatedFormData.acetowL.substring(0, 30),
-        nativeFindings: updatedFormData.nativeFindings.substring(0, 50),
-        aceticFindings: updatedFormData.aceticFindings.substring(0, 50),
-        colposcopicFindings: updatedFormData.colposcopicFindings.substring(0, 50),
-        treatmentPlan: updatedFormData.treatmentPlan.substring(0, 50),
-        followUp: updatedFormData.followUp.substring(0, 50),
-        biopsySites: updatedFormData.biopsySites.substring(0, 30),
-        biopsyNotes: updatedFormData.biopsyNotes.substring(0, 50),
+      // Verify each field is populated correctly
+      console.log('[ReportPage] Field validation:');
+      Object.entries(updatedFormData).forEach(([key, value]) => {
+        if (value.length > 0) {
+          console.log(`  ✅ ${key}: ${value.substring(0, 60)}...`);
+        } else {
+          console.warn(`  ❌ ${key}: EMPTY!`);
+        }
       });
 
+      // Additional safety check - if colposcopicFindings contains JSON structure, something went wrong
+      if (updatedFormData.colposcopicFindings.includes('{') && updatedFormData.colposcopicFindings.includes('"')) {
+        console.error('[ReportPage] ⚠️  WARNING: colposcopicFindings contains JSON! Parsing may have failed.');
+        console.error('Value:', updatedFormData.colposcopicFindings.substring(0, 200));
+      }
+
+      console.log('[ReportPage] AI generated fields:', updatedFormData);
+
       setFormData(prev => ({
         ...prev,
         ...updatedFormData
@@ -284,7 +312,7 @@ export function ReportPage({
       if (missingKeys.length === 0) {
         setAiError(null);
       } else {
-        setAiError(`✓ Report generated! Note: Some fields were not returned by AI (${missingKeys.join(', ')})`);
+        setAiError(`✓ Report generated! Some fields were missing: ${missingKeys.join(', ')}`);
       }
 
     } catch (err: any) {