Update main.py

main.py

@@ -1,26 +1,25 @@
 # app.py
-
 import os
 import io
 import uuid
 import re
 import json
 import traceback
-from datetime import datetime, timedelta
+from datetime import datetime
 
 from flask import Flask, request, jsonify
 from flask_cors import CORS
 import firebase_admin
 from firebase_admin import credentials, db, storage, auth
-import pandas as pd
 from pathlib import Path
 
-# Import the Sozo business logic
+# Import the refactored Sozo business logic
 from sozo_gen import (
-    generate_report_draft, 
-    generate_single_chart, 
+    generate_report_draft,
+    generate_single_chart,
     generate_video_from_project,
-    load_dataframe_safely
+    load_dataframe_safely,
+    deepgram_tts
 )
 
 # -----------------------------------------------------------------------------
@@ -115,416 +114,445 @@ def google_signin():
         return jsonify({'success': True, 'user': {'uid': uid, **user_data}}), 200
     except Exception as e: return jsonify({'error': str(e)}), 400
 
-import logging
-
-# Configure console logging
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
-logger = logging.getLogger(__name__)
-
-# -----------------------------------------------------------------------------
-# 4. SOZO BUSINESS STUDIO API ENDPOINTS WITH LOGGING
-# -----------------------------------------------------------------------------
-
-import logging
-
-# Configure console logging
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
-logger = logging.getLogger(__name__)
 
 # -----------------------------------------------------------------------------
-# 4. SOZO BUSINESS STUDIO API ENDPOINTS WITH LOGGING
+# 3. SOZO BUSINESS STUDIO API ENDPOINTS
 # -----------------------------------------------------------------------------
 
 @app.route('/api/sozo/projects', methods=['POST'])
 def create_sozo_project():
-    logger.info("POST /api/sozo/projects - Creating new project")
-    
+    logger.info("Endpoint /api/sozo/projects POST: Received request to create new project.")
     try:
-        token = request.headers.get('Authorization', '').split(' ')[1]
+        auth_header = request.headers.get('Authorization', '')
+        if not auth_header.startswith('Bearer '):
+            logger.warning("Create project failed: Missing or invalid auth header.")
+            return jsonify({'error': 'Missing or invalid token'}), 401
+        
+        token = auth_header.split(' ')[1]
         uid = verify_token(token)
         if not uid:
-            logger.warning("Unauthorized access attempt to create project")
+            logger.warning("Create project failed: Invalid token.")
             return jsonify({'error': 'Unauthorized'}), 401
-        
-        logger.info(f"User {uid} creating project")
-        
+        logger.info(f"Token verified for user UID: {uid}")
+
         if 'file' not in request.files:
-            logger.warning(f"User {uid} - No file provided")
+            logger.warning(f"User {uid}: Create project failed: No file part in request.")
             return jsonify({'error': 'No file part'}), 400
         
         file = request.files['file']
-        if file.filename == '':
-            logger.warning(f"User {uid} - Empty filename")
-            return jsonify({'error': 'No selected file'}), 400
-        
-        logger.info(f"User {uid} - Processing file: {file.filename}")
-        
         context = request.form.get('context', '')
         project_id = uuid.uuid4().hex
-        
-        logger.info(f"Generated project ID: {project_id}")
+        logger.info(f"User {uid}: Generated new project ID: {project_id}")
         
         file_bytes = file.read()
-        file.seek(0)
         ext = Path(file.filename).suffix
         
-        logger.info(f"File size: {len(file_bytes)} bytes, extension: {ext}")
-        
         blob_name = f"sozo_projects/{uid}/{project_id}/data{ext}"
+        logger.info(f"User {uid}: Uploading raw data to storage at {blob_name}")
         blob = bucket.blob(blob_name)
         blob.upload_from_string(file_bytes, content_type=file.content_type)
-        
-        logger.info(f"File uploaded to storage: {blob_name}")
+        logger.info(f"User {uid}: Successfully uploaded raw data for project {project_id}")
         
         project_ref = db.reference(f'sozo_projects/{project_id}')
         project_data = {
             'uid': uid,
+            'id': project_id,
             'status': 'uploaded',
             'createdAt': datetime.utcnow().isoformat(),
+            'updatedAt': datetime.utcnow().isoformat(),
             'userContext': context,
             'originalDataUrl': blob.public_url,
             'originalFilename': file.filename
         }
+        logger.info(f"User {uid}: Saving project metadata to database for project {project_id}")
         project_ref.set(project_data)
         
-        logger.info(f"Project data saved to database: {project_id}")
-        
         df = load_dataframe_safely(io.BytesIO(file_bytes), file.filename)
         preview_json = df.head().to_json(orient='records')
         
-        logger.info(f"Project {project_id} created successfully for user {uid}")
-        
+        logger.info(f"User {uid}: Project {project_id} created successfully.")
         return jsonify({
             'success': True,
-            'project_id': project_id,
+            'project': project_data,
             'preview': json.loads(preview_json)
         }), 201
         
     except Exception as e:
-        logger.error(f"Error creating project: {str(e)}")
-        logger.error(f"Traceback: {traceback.format_exc()}")
+        logger.error(f"CRITICAL ERROR during project creation: {traceback.format_exc()}")
         return jsonify({'error': str(e)}), 500
 
-@app.route('/api/sozo/projects/<string:project_id>/generate-report', methods=['POST'])
-def generate_sozo_report(project_id):
-    logger.info(f"POST /api/sozo/projects/{project_id}/generate-report - Generating report")
-    
+@app.route('/api/sozo/projects', methods=['GET'])
+def get_sozo_projects():
+    logger.info("Endpoint /api/sozo/projects GET: Received request to list projects.")
     try:
-        token = request.headers.get('Authorization', '').split(' ')[1]
+        auth_header = request.headers.get('Authorization', '')
+        if not auth_header.startswith('Bearer '):
+            logger.warning("List projects failed: Missing or invalid auth header.")
+            return jsonify({'error': 'Missing or invalid token'}), 401
+            
+        token = auth_header.split(' ')[1]
         uid = verify_token(token)
         if not uid:
-            logger.warning(f"Unauthorized access attempt to generate report for project {project_id}")
+            logger.warning("List projects failed: Invalid token.")
             return jsonify({'error': 'Unauthorized'}), 401
+        logger.info(f"Token verified for user UID: {uid}. Fetching projects.")
+
+        projects_ref = db.reference('sozo_projects')
+        user_projects = projects_ref.order_by_child('uid').equal_to(uid).get()
         
-        logger.info(f"User {uid} generating report for project {project_id}")
-        
-        project_ref = db.reference(f'sozo_projects/{project_id}')
-        project_data = project_ref.get()
-        
-        if not project_data or project_data.get('uid') != uid:
-            logger.warning(f"Project {project_id} not found or unauthorized for user {uid}")
-            return jsonify({'error': 'Project not found or unauthorized'}), 404
-        
-        logger.info(f"Project {project_id} validated for user {uid}")
-        
-        blob_path = "/".join(project_data['originalDataUrl'].split('/')[4:])
-        blob = bucket.blob(blob_path)
-        file_bytes = blob.download_as_bytes()
-        
-        logger.info(f"Downloaded file data for project {project_id}, size: {len(file_bytes)} bytes")
-        
-        draft_data = generate_report_draft(
-            io.BytesIO(file_bytes),
-            project_data['originalFilename'],
-            project_data['userContext'],
-            uid,
-            project_id,
-            bucket
-        )
-        
-        logger.info(f"Report draft generated for project {project_id}")
-        
-        # Clean and validate data before updating Firebase
-        try:
-            # Ensure rawMarkdown is a string and handle any encoding issues
-            raw_markdown = draft_data.get('raw_md', '')
-            if raw_markdown:
-                # Clean any problematic characters that might cause JSON parsing issues
-                raw_markdown = str(raw_markdown).replace('\x00', '').replace('\ufeff', '')
-                logger.info(f"Raw markdown length: {len(raw_markdown)}")
-            
-            # Ensure chartUrls is a proper list/dict
-            chart_urls = draft_data.get('chartUrls', [])
-            logger.info(f"Chart URLs type: {type(chart_urls)}, count: {len(chart_urls)}")
-            logger.info(f"Chart URLs content: {chart_urls}")
-            
-            # Convert chart_urls to a simple serializable format
-            if isinstance(chart_urls, dict):
-                # Convert dict to list of values or keep as dict but ensure all values are serializable
-                serializable_chart_urls = {}
-                for key, value in chart_urls.items():
-                    # Ensure key is string and value is serializable
-                    clean_key = str(key).replace('\x00', '').replace('\ufeff', '')
-                    clean_value = str(value).replace('\x00', '').replace('\ufeff', '') if value else ''
-                    serializable_chart_urls[clean_key] = clean_value
-                chart_urls = serializable_chart_urls
-            elif isinstance(chart_urls, list):
-                # Clean list items
-                chart_urls = [str(item).replace('\x00', '').replace('\ufeff', '') for item in chart_urls if item]
-            else:
-                logger.warning(f"Chart URLs is not a list/dict: {type(chart_urls)}")
-                chart_urls = []
-            
-            # Try to serialize the data to JSON first to catch any issues
-            import json
-            test_data = {
-                'status': 'draft',
-                'rawMarkdown': raw_markdown,
-                'chartUrls': chart_urls
-            }
-            
-            # Test JSON serialization
-            json_str = json.dumps(test_data, ensure_ascii=False)
-            logger.info(f"JSON serialization test passed, length: {len(json_str)}")
-            
-            logger.info(f"Updating project {project_id} with cleaned data")
-            project_ref.update(test_data)
-            
-        except json.JSONEncodeError as json_error:
-            logger.error(f"JSON serialization error for project {project_id}: {str(json_error)}")
-            logger.error(f"Problematic data - Raw markdown preview: {raw_markdown[:100]}...")
-            logger.error(f"Problematic data - Chart URLs: {chart_urls}")
-            raise json_error
-        except Exception as update_error:
-            logger.error(f"Error preparing update data for project {project_id}: {str(update_error)}")
-            logger.error(f"Draft data keys: {list(draft_data.keys()) if draft_data else 'None'}")
-            logger.error(f"Raw markdown type: {type(draft_data.get('raw_md'))}")
-            logger.error(f"Chart URLs type: {type(draft_data.get('chartUrls'))}")
-            raise update_error
-        
-        logger.info(f"Project {project_id} updated with draft data")
-        
-        return jsonify({
-            'success': True,
-            'project': {**project_data, **update_data}
-        }), 200
-        
+        if not user_projects:
+            logger.info(f"User {uid}: No projects found.")
+            return jsonify([]), 200
+
+        # Firebase returns a dictionary, convert it to a list for the client
+        projects_list = [project for project in user_projects.values()]
+        logger.info(f"User {uid}: Found and returning {len(projects_list)} projects.")
+        return jsonify(projects_list), 200
+
     except Exception as e:
-        logger.error(f"Error generating report for project {project_id}: {str(e)}")
-        logger.error(f"Traceback: {traceback.format_exc()}")
-        db.reference(f'sozo_projects/{project_id}').update({
-            'status': 'failed',
-            'error': str(e)
-        })
+        logger.error(f"CRITICAL ERROR during project list: {traceback.format_exc()}")
         return jsonify({'error': str(e)}), 500
 
-@app.route('/api/sozo/projects', methods=['GET'])
-def get_sozo_projects():
-    logger.info("GET /api/sozo/projects - Fetching user projects")
-    
+@app.route('/api/sozo/projects/<string:project_id>', methods=['GET'])
+def get_sozo_project(project_id):
+    logger.info(f"Endpoint /api/sozo/projects/{project_id} GET: Received request for single project.")
     try:
-        token = request.headers.get('Authorization', '').split(' ')[1]
+        auth_header = request.headers.get('Authorization', '')
+        if not auth_header.startswith('Bearer '):
+            logger.warning(f"Get project {project_id} failed: Missing or invalid auth header.")
+            return jsonify({'error': 'Missing or invalid token'}), 401
+
+        token = auth_header.split(' ')[1]
         uid = verify_token(token)
         if not uid:
-            logger.warning("Unauthorized access attempt to fetch projects")
+            logger.warning(f"Get project {project_id} failed: Invalid token.")
             return jsonify({'error': 'Unauthorized'}), 401
-        
-        logger.info(f"User {uid} fetching projects")
-        
-        all_projects = db.reference('sozo_projects').order_by_child('uid').equal_to(uid).get()
-        
-        project_count = len(all_projects) if all_projects else 0
-        logger.info(f"Found {project_count} projects for user {uid}")
-        
-        return jsonify(all_projects or {}), 200
-        
+        logger.info(f"Token verified for user UID: {uid}. Fetching project {project_id}.")
+
+        project_ref = db.reference(f'sozo_projects/{project_id}')
+        project_data = project_ref.get()
+
+        if not project_data:
+            logger.warning(f"User {uid}: Attempted to access non-existent project {project_id}.")
+            return jsonify({'error': 'Project not found'}), 404
+
+        if project_data.get('uid') != uid:
+            logger.error(f"User {uid}: UNAUTHORIZED attempt to access project {project_id} owned by {project_data.get('uid')}.")
+            return jsonify({'error': 'Unauthorized to access this project'}), 403
+
+        logger.info(f"User {uid}: Successfully fetched project {project_id}.")
+        return jsonify(project_data), 200
+
     except Exception as e:
-        logger.error(f"Error fetching projects: {str(e)}")
+        logger.error(f"CRITICAL ERROR during get project {project_id}: {traceback.format_exc()}")
         return jsonify({'error': str(e)}), 500
 
-@app.route('/api/sozo/projects/<string:project_id>', methods=['GET'])
-def get_sozo_project(project_id):
-    logger.info(f"GET /api/sozo/projects/{project_id} - Fetching project")
-    
+@app.route('/api/sozo/projects/<string:project_id>', methods=['PUT'])
+def update_sozo_project(project_id):
+    logger.info(f"Endpoint /api/sozo/projects/{project_id} PUT: Received request to update project.")
     try:
-        token = request.headers.get('Authorization', '').split(' ')[1]
+        auth_header = request.headers.get('Authorization', '')
+        if not auth_header.startswith('Bearer '):
+            logger.warning(f"Update project {project_id} failed: Missing or invalid auth header.")
+            return jsonify({'error': 'Missing or invalid token'}), 401
+
+        token = auth_header.split(' ')[1]
         uid = verify_token(token)
         if not uid:
-            logger.warning(f"Unauthorized access attempt to fetch project {project_id}")
+            logger.warning(f"Update project {project_id} failed: Invalid token.")
             return jsonify({'error': 'Unauthorized'}), 401
+        logger.info(f"Token verified for user UID: {uid}. Updating project {project_id}.")
+
+        project_ref = db.reference(f'sozo_projects/{project_id}')
+        project_data = project_ref.get()
+
+        if not project_data:
+            logger.warning(f"User {uid}: Attempted to update non-existent project {project_id}.")
+            return jsonify({'error': 'Project not found'}), 404
+
+        if project_data.get('uid') != uid:
+            logger.error(f"User {uid}: UNAUTHORIZED attempt to update project {project_id} owned by {project_data.get('uid')}.")
+            return jsonify({'error': 'Unauthorized to update this project'}), 403
         
-        logger.info(f"User {uid} fetching project {project_id}")
-        
-        project_data = db.reference(f'sozo_projects/{project_id}').get()
+        logger.info(f"User {uid}: Ownership of project {project_id} verified.")
         
-        if not project_data or project_data.get('uid') != uid:
-            logger.warning(f"Project {project_id} not found or unauthorized for user {uid}")
-            return jsonify({'error': 'Project not found or unauthorized'}), 404
+        update_data = request.get_json()
+        if not update_data:
+            return jsonify({'error': 'No update data provided'}), 400
+
+        # Define fields the user is allowed to update
+        allowed_updates = ['userContext', 'originalFilename']
+        final_updates = {key: update_data[key] for key in update_data if key in allowed_updates}
         
-        logger.info(f"Project {project_id} fetched successfully for user {uid}")
+        if not final_updates:
+            logger.warning(f"User {uid}: Update for project {project_id} contained no valid fields.")
+            return jsonify({'error': 'No valid fields to update'}), 400
+
+        final_updates['updatedAt'] = datetime.utcnow().isoformat()
         
-        return jsonify(project_data), 200
+        logger.info(f"User {uid}: Applying updates to project {project_id}: {final_updates}")
+        project_ref.update(final_updates)
         
+        updated_project = project_ref.get()
+        logger.info(f"User {uid}: Successfully updated project {project_id}.")
+        return jsonify(updated_project), 200
+
     except Exception as e:
-        logger.error(f"Error fetching project {project_id}: {str(e)}")
+        logger.error(f"CRITICAL ERROR during update project {project_id}: {traceback.format_exc()}")
         return jsonify({'error': str(e)}), 500
 
 @app.route('/api/sozo/projects/<string:project_id>', methods=['DELETE'])
 def delete_sozo_project(project_id):
-    logger.info(f"DELETE /api/sozo/projects/{project_id} - Deleting project")
-    
+    logger.info(f"Endpoint /api/sozo/projects/{project_id} DELETE: Received request to delete project.")
     try:
-        token = request.headers.get('Authorization', '').split(' ')[1]
+        auth_header = request.headers.get('Authorization', '')
+        if not auth_header.startswith('Bearer '):
+            logger.warning(f"Delete project {project_id} failed: Missing or invalid auth header.")
+            return jsonify({'error': 'Missing or invalid token'}), 401
+
+        token = auth_header.split(' ')[1]
         uid = verify_token(token)
         if not uid:
-            logger.warning(f"Unauthorized access attempt to delete project {project_id}")
+            logger.warning(f"Delete project {project_id} failed: Invalid token.")
             return jsonify({'error': 'Unauthorized'}), 401
-        
-        logger.info(f"User {uid} attempting to delete project {project_id}")
-        
-        # Get project data first to verify ownership and get file paths
+        logger.info(f"Token verified for user UID: {uid}. Deleting project {project_id}.")
+
         project_ref = db.reference(f'sozo_projects/{project_id}')
         project_data = project_ref.get()
-        
+
         if not project_data:
-            logger.warning(f"Project {project_id} not found for deletion")
+            logger.warning(f"User {uid}: Attempted to delete non-existent project {project_id}.")
             return jsonify({'error': 'Project not found'}), 404
-        
+
         if project_data.get('uid') != uid:
-            logger.warning(f"User {uid} unauthorized to delete project {project_id}")
+            logger.error(f"User {uid}: UNAUTHORIZED attempt to delete project {project_id} owned by {project_data.get('uid')}.")
             return jsonify({'error': 'Unauthorized to delete this project'}), 403
         
-        logger.info(f"Project {project_id} verified for deletion by user {uid}")
-        
-        # Delete files from storage
-        deleted_files = []
-        try:
-            # Delete original data file
-            if 'originalDataUrl' in project_data:
-                blob_path = "/".join(project_data['originalDataUrl'].split('/')[4:])
-                blob = bucket.blob(blob_path)
-                if blob.exists():
-                    blob.delete()
-                    deleted_files.append(blob_path)
-                    logger.info(f"Deleted original data file: {blob_path}")
-            
-            # Delete chart files if they exist
-            chart_urls = project_data.get('chartUrls', {})
-            if isinstance(chart_urls, dict):
-                for chart_name, chart_url in chart_urls.items():
-                    try:
-                        chart_blob_path = "/".join(chart_url.split('/')[4:])
-                        chart_blob = bucket.blob(chart_blob_path)
-                        if chart_blob.exists():
-                            chart_blob.delete()
-                            deleted_files.append(chart_blob_path)
-                            logger.info(f"Deleted chart file: {chart_blob_path}")
-                    except Exception as chart_error:
-                        logger.warning(f"Error deleting chart {chart_name}: {str(chart_error)}")
-            
-            # Delete entire project folder if it exists
-            project_folder = f"sozo_projects/{uid}/{project_id}/"
-            blobs = bucket.list_blobs(prefix=project_folder)
-            for blob in blobs:
-                if blob.name not in deleted_files:  # Avoid duplicate deletions
-                    blob.delete()
-                    deleted_files.append(blob.name)
-                    logger.info(f"Deleted additional file: {blob.name}")
-            
-            logger.info(f"Deleted {len(deleted_files)} files from storage for project {project_id}")
-            
-        except Exception as storage_error:
-            logger.error(f"Error deleting storage files for project {project_id}: {str(storage_error)}")
-            # Continue with database deletion even if storage cleanup fails
-        
-        # Delete project from database
+        logger.info(f"User {uid}: Ownership of project {project_id} verified. Proceeding with deletion.")
+
+        # Delete all associated files from Firebase Storage
+        project_folder_prefix = f"sozo_projects/{uid}/{project_id}/"
+        logger.info(f"User {uid}: Deleting all files from storage folder: {project_folder_prefix}")
+        blobs_to_delete = bucket.list_blobs(prefix=project_folder_prefix)
+        deleted_files_count = 0
+        for blob in blobs_to_delete:
+            logger.info(f"User {uid}: Deleting file {blob.name} from storage.")
+            blob.delete()
+            deleted_files_count += 1
+        logger.info(f"User {uid}: Deleted {deleted_files_count} files from storage for project {project_id}.")
+
+        # Delete project from Realtime Database
+        logger.info(f"User {uid}: Deleting project {project_id} from database.")
         project_ref.delete()
-        logger.info(f"Project {project_id} deleted from database")
-        
-        logger.info(f"Project {project_id} successfully deleted by user {uid}")
-        
-        return jsonify({
-            'success': True,
-            'message': f'Project {project_id} deleted successfully',
-            'deleted_files': len(deleted_files)
-        }), 200
         
+        logger.info(f"User {uid}: Successfully deleted project {project_id}.")
+        return jsonify({'success': True, 'message': f'Project {project_id} and all associated files deleted.'}), 200
+
     except Exception as e:
-        logger.error(f"Error deleting project {project_id}: {str(e)}")
-        logger.error(f"Traceback: {traceback.format_exc()}")
+        logger.error(f"CRITICAL ERROR during delete project {project_id}: {traceback.format_exc()}")
         return jsonify({'error': str(e)}), 500
 
-@app.route('/api/sozo/projects/<string:project_id>/markdown', methods=['PUT'])
-def update_sozo_markdown(project_id):
+@app.route('/api/sozo/projects/<string:project_id>/generate-report', methods=['POST'])
+def generate_sozo_report(project_id):
+    logger.info(f"Endpoint /generate-report POST for project {project_id}")
     try:
         token = request.headers.get('Authorization', '').split(' ')[1]
         uid = verify_token(token)
         if not uid: return jsonify({'error': 'Unauthorized'}), 401
+        logger.info(f"Token verified for user {uid} for report generation.")
+        
         project_ref = db.reference(f'sozo_projects/{project_id}')
-        if not project_ref.get() or project_ref.get().get('uid') != uid: return jsonify({'error': 'Project not found or unauthorized'}), 404
-        data = request.get_json()
-        if 'raw_md' not in data: return jsonify({'error': 'raw_md is required'}), 400
-        project_ref.update({'rawMarkdown': data['raw_md']})
-        return jsonify({'success': True}), 200
-    except Exception as e: return jsonify({'error': str(e)}), 500
+        project_data = project_ref.get()
+        if not project_data or project_data.get('uid') != uid:
+            logger.warning(f"User {uid} failed to generate report: Project {project_id} not found or not owned.")
+            return jsonify({'error': 'Project not found or unauthorized'}), 404
+        
+        logger.info(f"User {uid}: Ownership verified for project {project_id}. Starting report generation.")
+        project_ref.update({'status': 'generating_report', 'updatedAt': datetime.utcnow().isoformat()})
+        
+        blob_path = f"sozo_projects/{uid}/{project_id}/data{Path(project_data['originalFilename']).suffix}"
+        logger.info(f"User {uid}: Downloading data from {blob_path}")
+        blob = bucket.blob(blob_path)
+        file_bytes = blob.download_as_bytes()
+        df = load_dataframe_safely(io.BytesIO(file_bytes), project_data['originalFilename'])
+        
+        logger.info(f"User {uid}: Calling core logic to generate report draft for project {project_id}.")
+        draft_data = generate_report_draft(df, project_data['userContext'])
+        
+        logger.info(f"User {uid}: Saving report content specification to database for project {project_id}.")
+        project_ref.child('report_content').set(draft_data)
+        
+        project_ref.update({'status': 'draft_complete', 'updatedAt': datetime.utcnow().isoformat()})
+        
+        full_project_data = project_ref.get()
+        logger.info(f"User {uid}: Report generation for project {project_id} complete.")
+        return jsonify({'success': True, 'project': full_project_data}), 200
+        
+    except Exception as e:
+        db.reference(f'sozo_projects/{project_id}').update({'status': 'failed', 'error': str(e), 'updatedAt': datetime.utcnow().isoformat()})
+        logger.error(f"CRITICAL ERROR generating report for {project_id}: {traceback.format_exc()}")
+        return jsonify({'error': str(e)}), 500
 
-@app.route('/api/sozo/projects/<string:project_id>/charts', methods=['POST'])
-def regenerate_sozo_chart(project_id):
+@app.route('/api/sozo/projects/<string:project_id>/generate-video', methods=['POST'])
+def generate_sozo_video(project_id):
+    logger.info(f"Endpoint /generate-video POST for project {project_id}")
     try:
         token = request.headers.get('Authorization', '').split(' ')[1]
         uid = verify_token(token)
         if not uid: return jsonify({'error': 'Unauthorized'}), 401
+        logger.info(f"Token verified for user {uid} for video script generation.")
+
         project_ref = db.reference(f'sozo_projects/{project_id}')
         project_data = project_ref.get()
-        if not project_data or project_data.get('uid') != uid: return jsonify({'error': 'Project not found or unauthorized'}), 404
+        if not project_data or project_data.get('uid') != uid:
+            logger.warning(f"User {uid} failed to generate video script: Project {project_id} not found or not owned.")
+            return jsonify({'error': 'Project not found or unauthorized'}), 404
+
         data = request.get_json()
-        description = data.get('description')
-        if not description: return jsonify({'error': 'Chart description is required'}), 400
+        voice_model = data.get('voice_model', 'aura-2-andromeda-en')
         
-        blob_path = "/".join(project_data['originalDataUrl'].split('/')[4:])
+        logger.info(f"User {uid}: Ownership verified for project {project_id}. Starting video script generation with voice {voice_model}.")
+        project_ref.update({'status': 'generating_video', 'updatedAt': datetime.utcnow().isoformat()})
+
+        blob_path = f"sozo_projects/{uid}/{project_id}/data{Path(project_data['originalFilename']).suffix}"
+        logger.info(f"User {uid}: Downloading data from {blob_path}")
         blob = bucket.blob(blob_path)
         file_bytes = blob.download_as_bytes()
         df = load_dataframe_safely(io.BytesIO(file_bytes), project_data['originalFilename'])
         
-        new_chart_url = generate_single_chart(df, description, uid, project_id, bucket)
-        if not new_chart_url: return jsonify({'error': 'Chart generation failed'}), 500
+        logger.info(f"User {uid}: Calling core logic to generate video script for project {project_id}.")
+        video_script = generate_video_from_project(df, project_data.get('report_content', {}).get('raw_md', ''), voice_model)
+
+        logger.info(f"User {uid}: Uploading generated audio files to storage for project {project_id}.")
+        for scene in video_script.get("scenes", []):
+            if scene.get("audio_content"):
+                audio_bytes = scene.pop("audio_content")
+                audio_blob_name = f"sozo_projects/{uid}/{project_id}/audio/{scene['scene_id']}.mp3"
+                audio_blob = bucket.blob(audio_blob_name)
+                audio_blob.upload_from_string(audio_bytes, content_type="audio/mpeg")
+                scene["audio_storage_path"] = audio_blob.public_url
+        
+        logger.info(f"User {uid}: Saving video script specification to database for project {project_id}.")
+        project_ref.child('video_script').set(video_script)
+        project_ref.update({'status': 'video_script_complete', 'updatedAt': datetime.utcnow().isoformat()})
         
-        project_ref.child('chartUrls').update({description: new_chart_url})
-        return jsonify({'success': True, 'description': description, 'new_url': new_chart_url}), 200
+        full_project_data = project_ref.get()
+        logger.info(f"User {uid}: Video script generation for project {project_id} complete.")
+        return jsonify({'success': True, 'project': full_project_data}), 200
+
     except Exception as e:
-        traceback.print_exc()
+        db.reference(f'sozo_projects/{project_id}').update({'status': 'failed', 'error': str(e), 'updatedAt': datetime.utcnow().isoformat()})
+        logger.error(f"CRITICAL ERROR generating video for {project_id}: {traceback.format_exc()}")
         return jsonify({'error': str(e)}), 500
 
-@app.route('/api/sozo/projects/<string:project_id>/generate-video', methods=['POST'])
-def generate_sozo_video(project_id):
+@app.route('/api/sozo/projects/<string:project_id>/charts', methods=['POST'])
+def regenerate_sozo_chart(project_id):
+    logger.info(f"Endpoint /charts POST for project {project_id}")
     try:
         token = request.headers.get('Authorization', '').split(' ')[1]
         uid = verify_token(token)
         if not uid: return jsonify({'error': 'Unauthorized'}), 401
+        logger.info(f"Token verified for user {uid} for chart regeneration.")
+
         project_ref = db.reference(f'sozo_projects/{project_id}')
         project_data = project_ref.get()
-        if not project_data or project_data.get('uid') != uid: return jsonify({'error': 'Project not found or unauthorized'}), 404
-        
+        if not project_data or project_data.get('uid') != uid:
+            logger.warning(f"User {uid} failed to regenerate chart: Project {project_id} not found or not owned.")
+            return jsonify({'error': 'Project not found or unauthorized'}), 404
+
         data = request.get_json()
-        voice_model = data.get('voice_model', 'aura-2-andromeda-en')
-        if voice_model not in ['aura-2-andromeda-en', 'aura-2-orpheus-en']: return jsonify({'error': 'Invalid voice model specified'}), 400
+        description = data.get('description')
+        chart_id_to_replace = data.get('chart_id')
+        if not description or not chart_id_to_replace:
+            return jsonify({'error': 'Chart description and chart_id are required'}), 400
         
-        project_ref.update({'status': 'generating_video'})
-        blob_path = "/".join(project_data['originalDataUrl'].split('/')[4:])
+        logger.info(f"User {uid}: Regenerating chart '{chart_id_to_replace}' for project {project_id} with new description: '{description}'")
+        blob_path = f"sozo_projects/{uid}/{project_id}/data{Path(project_data['originalFilename']).suffix}"
         blob = bucket.blob(blob_path)
         file_bytes = blob.download_as_bytes()
         df = load_dataframe_safely(io.BytesIO(file_bytes), project_data['originalFilename'])
         
-        video_url = generate_video_from_project(df, project_data.get('rawMarkdown', ''), uid, project_id, voice_model, bucket)
-        if not video_url: raise Exception("Video generation failed in core function.")
+        new_chart_spec = generate_single_chart(df, description)
+        
+        logger.info(f"User {uid}: Updating chart spec in database for project {project_id}.")
+        report_content_ref = project_ref.child('report_content')
+        report_content = report_content_ref.get()
+        chart_specs = report_content.get('chart_specs', [])
         
-        project_ref.update({'status': 'video_complete', 'videoUrl': video_url})
-        return jsonify({'success': True, 'video_url': video_url}), 200
+        chart_found = False
+        for i, spec in enumerate(chart_specs):
+            if spec.get('id') == chart_id_to_replace:
+                chart_specs[i] = new_chart_spec
+                chart_found = True
+                break
+        
+        if not chart_found:
+            logger.warning(f"User {uid}: Chart with id {chart_id_to_replace} not found in project {project_id}.")
+            return jsonify({'error': f'Chart with id {chart_id_to_replace} not found'}), 404
+            
+        report_content_ref.child('chart_specs').set(chart_specs)
+        project_ref.update({'updatedAt': datetime.utcnow().isoformat()})
+        
+        logger.info(f"User {uid}: Successfully regenerated chart {chart_id_to_replace} for project {project_id}.")
+        return jsonify({'success': True, 'new_chart_spec': new_chart_spec}), 200
     except Exception as e:
-        db.reference(f'sozo_projects/{project_id}').update({'status': 'failed', 'error': str(e)})
-        traceback.print_exc()
+        logger.error(f"CRITICAL ERROR regenerating chart for {project_id}: {traceback.format_exc()}")
         return jsonify({'error': str(e)}), 500
 
+@app.route('/api/sozo/projects/<string:project_id>/update-narration-audio', methods=['POST'])
+def update_narration_audio(project_id):
+    logger.info(f"Endpoint /update-narration-audio POST for project {project_id}")
+    try:
+        token = request.headers.get('Authorization', '').split(' ')[1]
+        uid = verify_token(token)
+        if not uid: return jsonify({'error': 'Unauthorized'}), 401
+        logger.info(f"Token verified for user {uid} for narration update.")
+
+        data = request.get_json()
+        scene_id = data.get('scene_id')
+        narration_text = data.get('narration_text')
+        voice_model = data.get('voice_model', 'aura-2-andromeda-en')
+        if not scene_id or narration_text is None:
+            return jsonify({'error': 'scene_id and narration_text are required'}), 400
+
+        logger.info(f"User {uid}: Updating narration for scene {scene_id} in project {project_id}.")
+        audio_bytes = deepgram_tts(narration_text, voice_model)
+        if not audio_bytes:
+            logger.error(f"User {uid}: Deepgram TTS failed for project {project_id}, scene {scene_id}.")
+            return jsonify({'error': 'Failed to generate audio'}), 500
+
+        audio_blob_name = f"sozo_projects/{uid}/{project_id}/audio/{scene_id}.mp3"
+        logger.info(f"User {uid}: Uploading new audio to {audio_blob_name}.")
+        audio_blob = bucket.blob(audio_blob_name)
+        audio_blob.upload_from_string(audio_bytes, content_type="audio/mpeg")
+        new_audio_url = audio_blob.public_url
+
+        logger.info(f"User {uid}: Updating database with new narration and audio URL for project {project_id}.")
+        scene_ref = db.reference(f'sozo_projects/{project_id}/video_script/scenes')
+        scenes = scene_ref.get()
+        scene_found = False
+        if scenes:
+            for i, scene in enumerate(scenes):
+                if scene.get('scene_id') == scene_id:
+                    scene_ref.child(str(i)).update({
+                        'narration': narration_text,
+                        'audio_storage_path': new_audio_url
+                    })
+                    scene_found = True
+                    break
+        
+        if not scene_found:
+            logger.warning(f"User {uid}: Scene {scene_id} not found in database for project {project_id} during narration update.")
+            return jsonify({'error': 'Scene not found in database'}), 404
+
+        project_ref = db.reference(f'sozo_projects/{project_id}')
+        project_ref.update({'updatedAt': datetime.utcnow().isoformat()})
+
+        logger.info(f"User {uid}: Successfully updated narration for scene {scene_id} in project {project_id}.")
+        return jsonify({'success': True, 'new_audio_url': new_audio_url}), 200
+    except Exception as e:
+        logger.error(f"CRITICAL ERROR updating narration for {project_id}: {traceback.format_exc()}")
+        return jsonify({'error': str(e)}), 500
 # -----------------------------------------------------------------------------
 # 5. UNIVERSAL ENDPOINTS (Waitlist, Feedback, Credits)
 # -----------------------------------------------------------------------------