Upload 190 files

.gitattributes

@@ -1,35 +1,35 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

app.ipynb

@@ -1,127 +1,127 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "578b4ea2",
-   "metadata": {
-    "vscode": {
-     "languageId": "plaintext"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "import json\n",
-    "import warnings\n",
-    "from fastapi import FastAPI, Request\n",
-    "from fastapi.responses import JSONResponse\n",
-    "from fastapi.middleware.cors import CORSMiddleware\n",
-    "import os \n",
-    "\n",
-    "# Import routes\n",
-    "# from routes import collections, ingestion, sessions, chat, system\n",
-    "from Redis.sessions import Redis_session_router\n",
-    "from Redis.sessions_new import redis_session_route_new\n",
-    "from Redis.sessions_old import Redis_session_router_old\n",
-    "# from Routes.main_chat_bot import main_chatbot_route\n",
-    "from Routes.generate_report import Report_Generation_Router\n",
-    "# from report_generation.report_generation import Report_Generation_Router_v2\n",
-    "from s3.file_insertion_s3 import s3_bucket_router\n",
-    "from s3.rough2 import test_router\n",
-    "from backend.main import login_apis_router\n",
-    "from s3.r3 import s3_bucket_router1\n",
-    "from s3_viewer_backend.s3_viewer import s3_viewer_router \n",
-    "# from vector_db.retrival_qa_agent import RetrievalQA_router\n",
-    "from main_chat_2.main_chat import main_chat_router_v4\n",
-    "from main_chat5.main_chat_router import main_chat_router_v5\n",
-    "from report_generation.ppt_v2 import Report_Generation_Router_ppt_v2\n",
-    "from report_generation.pdf_v3 import Report_Generation_Router_pdf_v2\n",
-    "\n",
-    "\n",
-    "\n",
-    "from Routes.main_agent_chat_bot_v2 import main_chatbot_route_v2\n",
-    "from vector_db import qdrant_crud\n",
-    "\n",
-    "from vector_db.qdrant_crud import Qdrant_router\n",
-    "from agent_tools.autovis_tool import Autoviz_router\n",
-    "# from main_chat_2.main_chat import main_chat_router_v3\n",
-    "from DB_store_backup.agentic_context_convoid_management import Db_store_router\n",
-    "# Suppress warnings\n",
-    "warnings.filterwarnings(\"ignore\", message=\"Qdrant client version.*is incompatible.*\")\n",
-    "\n",
-    "app = FastAPI(title=\"Combined AI Agent with Qdrant Collections and Redis Session Management\")\n",
-    "\n",
-    "# Add CORS middleware\n",
-    "app.add_middleware(\n",
-    "    CORSMiddleware,\n",
-    "    allow_origins=[\"*\"],\n",
-    "    allow_credentials=True,\n",
-    "    allow_methods=[\"*\"],\n",
-    "    allow_headers=[\"*\"],\n",
-    ")\n",
-    "\n",
-    "print([route.path for route in app.routes])\n",
-    "# Include routers\n",
-    "\n",
-    "# app.include_router(main_chatbot_route)\n",
-    "app.include_router(Report_Generation_Router)\n",
-    "app.include_router(Redis_session_router)\n",
-    "app.include_router(redis_session_route_new)\n",
-    "# app.include_router(Redis_session_router_old)\n",
-    "app.include_router(Db_store_router)\n",
-    "app.include_router(s3_bucket_router)\n",
-    "app.include_router(Autoviz_router)\n",
-    "# app.include_router(main_chat_router_v3)\n",
-    "app.include_router(s3_viewer_router)\n",
-    "\n",
-    "app.include_router(s3_bucket_router1)\n",
-    "app.include_router(main_chat_router_v4)\n",
-    "app.include_router(test_router)\n",
-    "app.include_router(Qdrant_router)\n",
-    "# app.include_router(RetrievalQA_router)\n",
-    "app.include_router(main_chatbot_route_v2)\n",
-    "# app.include_router(Report_Generation_Router_v2)\n",
-    "app.include_router(Report_Generation_Router_ppt_v2)\n",
-    "app.include_router(Report_Generation_Router_pdf_v2)\n",
-    "\n",
-    "app.include_router(main_chat_router_v5)\n",
-    "#==================================login and user management routes==================================\n",
-    "app.include_router(login_apis_router)\n",
-    "\n",
-    "\n",
-    "# ------------------- MIDDLEWARE -------------------\n",
-    "\n",
-    "@app.middleware(\"http\")\n",
-    "async def add_success_flag(request: Request, call_next):\n",
-    "    response = await call_next(request)\n",
-    "\n",
-    "    # Only modify JSON responses\n",
-    "    if \"application/json\" in response.headers.get(\"content-type\", \"\"):\n",
-    "        try:\n",
-    "            body = b\"\".join([chunk async for chunk in response.body_iterator])\n",
-    "            data = json.loads(body.decode())\n",
-    "\n",
-    "            # Add success flag\n",
-    "            data[\"success\"] = 200 <= response.status_code < 300\n",
-    "\n",
-    "            # Build new JSONResponse (auto handles Content-Length)\n",
-    "            response = JSONResponse(\n",
-    "                content=data,\n",
-    "                status_code=response.status_code,\n",
-    "                headers={k: v for k, v in response.headers.items() if k.lower() != \"content-length\"},\n",
-    "            )\n",
-    "        except Exception:\n",
-    "            # fallback if response is not JSON parseable\n",
-    "            pass\n",
-    "    return response"
-   ]
-  }
- ],
- "metadata": {
-  "language_info": {
-   "name": "python"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "578b4ea2",
+   "metadata": {
+    "vscode": {
+     "languageId": "plaintext"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "import warnings\n",
+    "from fastapi import FastAPI, Request\n",
+    "from fastapi.responses import JSONResponse\n",
+    "from fastapi.middleware.cors import CORSMiddleware\n",
+    "import os \n",
+    "\n",
+    "# Import routes\n",
+    "# from routes import collections, ingestion, sessions, chat, system\n",
+    "from Redis.sessions import Redis_session_router\n",
+    "from Redis.sessions_new import redis_session_route_new\n",
+    "from Redis.sessions_old import Redis_session_router_old\n",
+    "# from Routes.main_chat_bot import main_chatbot_route\n",
+    "from Routes.generate_report import Report_Generation_Router\n",
+    "# from report_generation.report_generation import Report_Generation_Router_v2\n",
+    "from s3.file_insertion_s3 import s3_bucket_router\n",
+    "from s3.rough2 import test_router\n",
+    "from backend.main import login_apis_router\n",
+    "from s3.r3 import s3_bucket_router1\n",
+    "from s3_viewer_backend.s3_viewer import s3_viewer_router \n",
+    "# from vector_db.retrival_qa_agent import RetrievalQA_router\n",
+    "from main_chat_2.main_chat import main_chat_router_v4\n",
+    "from main_chat5.main_chat_router import main_chat_router_v5\n",
+    "from report_generation.ppt_v2 import Report_Generation_Router_ppt_v2\n",
+    "from report_generation.pdf_v3 import Report_Generation_Router_pdf_v2\n",
+    "\n",
+    "\n",
+    "\n",
+    "from Routes.main_agent_chat_bot_v2 import main_chatbot_route_v2\n",
+    "from vector_db import qdrant_crud\n",
+    "\n",
+    "from vector_db.qdrant_crud import Qdrant_router\n",
+    "from agent_tools.autovis_tool import Autoviz_router\n",
+    "# from main_chat_2.main_chat import main_chat_router_v3\n",
+    "from DB_store_backup.agentic_context_convoid_management import Db_store_router\n",
+    "# Suppress warnings\n",
+    "warnings.filterwarnings(\"ignore\", message=\"Qdrant client version.*is incompatible.*\")\n",
+    "\n",
+    "app = FastAPI(title=\"Combined AI Agent with Qdrant Collections and Redis Session Management\")\n",
+    "\n",
+    "# Add CORS middleware\n",
+    "app.add_middleware(\n",
+    "    CORSMiddleware,\n",
+    "    allow_origins=[\"*\"],\n",
+    "    allow_credentials=True,\n",
+    "    allow_methods=[\"*\"],\n",
+    "    allow_headers=[\"*\"],\n",
+    ")\n",
+    "\n",
+    "print([route.path for route in app.routes])\n",
+    "# Include routers\n",
+    "\n",
+    "# app.include_router(main_chatbot_route)\n",
+    "app.include_router(Report_Generation_Router)\n",
+    "app.include_router(Redis_session_router)\n",
+    "app.include_router(redis_session_route_new)\n",
+    "# app.include_router(Redis_session_router_old)\n",
+    "app.include_router(Db_store_router)\n",
+    "app.include_router(s3_bucket_router)\n",
+    "app.include_router(Autoviz_router)\n",
+    "# app.include_router(main_chat_router_v3)\n",
+    "app.include_router(s3_viewer_router)\n",
+    "\n",
+    "app.include_router(s3_bucket_router1)\n",
+    "app.include_router(main_chat_router_v4)\n",
+    "app.include_router(test_router)\n",
+    "app.include_router(Qdrant_router)\n",
+    "# app.include_router(RetrievalQA_router)\n",
+    "app.include_router(main_chatbot_route_v2)\n",
+    "# app.include_router(Report_Generation_Router_v2)\n",
+    "app.include_router(Report_Generation_Router_ppt_v2)\n",
+    "app.include_router(Report_Generation_Router_pdf_v2)\n",
+    "\n",
+    "app.include_router(main_chat_router_v5)\n",
+    "#==================================login and user management routes==================================\n",
+    "app.include_router(login_apis_router)\n",
+    "\n",
+    "\n",
+    "# ------------------- MIDDLEWARE -------------------\n",
+    "\n",
+    "@app.middleware(\"http\")\n",
+    "async def add_success_flag(request: Request, call_next):\n",
+    "    response = await call_next(request)\n",
+    "\n",
+    "    # Only modify JSON responses\n",
+    "    if \"application/json\" in response.headers.get(\"content-type\", \"\"):\n",
+    "        try:\n",
+    "            body = b\"\".join([chunk async for chunk in response.body_iterator])\n",
+    "            data = json.loads(body.decode())\n",
+    "\n",
+    "            # Add success flag\n",
+    "            data[\"success\"] = 200 <= response.status_code < 300\n",
+    "\n",
+    "            # Build new JSONResponse (auto handles Content-Length)\n",
+    "            response = JSONResponse(\n",
+    "                content=data,\n",
+    "                status_code=response.status_code,\n",
+    "                headers={k: v for k, v in response.headers.items() if k.lower() != \"content-length\"},\n",
+    "            )\n",
+    "        except Exception:\n",
+    "            # fallback if response is not JSON parseable\n",
+    "            pass\n",
+    "    return response"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

app.py

@@ -1,115 +1,115 @@
-import json
-import warnings
-from fastapi import FastAPI, Request
-from fastapi.responses import JSONResponse
-from fastapi.middleware.cors import CORSMiddleware
-import os 
-import json
-import warnings
-from fastapi import FastAPI, Request
-from fastapi.responses import JSONResponse
-from fastapi.middleware.cors import CORSMiddleware
-import os 
-
-# Import routes
-# from routes import collections, ingestion, sessions, chat, system
-from Redis.sessions import Redis_session_router
-from Redis.sessions_new import redis_session_route_new
-from Redis.sessions_old import Redis_session_router_old
-# from Routes.main_chat_bot import main_chatbot_route
-from Routes.generate_report import Report_Generation_Router
-# from report_generation.report_generation import Report_Generation_Router_v2
-from s3.file_insertion_s3 import s3_bucket_router
-from s3.rough2 import test_router
-from backend.main import login_apis_router
-# from s3.r3 import s3_bucket_router1
-from s3.r4 import s3_bucket_router1
-from s3_viewer_backend.s3_viewer import s3_viewer_router 
-# from vector_db.retrival_qa_agent import RetrievalQA_router
-from main_chat_2.main_chat import main_chat_router_v4
-from main_chat5.main_chat_router import main_chat_router_v5
-from main_chat_v6.main_chat_router import main_chat_router_v6
-from report_generation.ppt_v2 import Report_Generation_Router_ppt_v2
-from report_generation.pdf_v3 import Report_Generation_Router_pdf_v2
-
-
-
-from Routes.main_agent_chat_bot_v2 import main_chatbot_route_v2
-# from vector_db import qdrant_crud
-
-# from vector_db.qdrant_crud import Qdrant_router
-from agent_tools.autovis_tool import Autoviz_router
-# from main_chat_2.main_chat import main_chat_router_v3
-from DB_store_backup.agentic_context_convoid_management import Db_store_router
-# Suppress warnings
-warnings.filterwarnings("ignore", message="Qdrant client version.*is incompatible.*")
-
-app = FastAPI(title="Combined AI Agent with Qdrant Collections and Redis Session Management")
-
-# Add CORS middleware
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
-
-print([route.path for route in app.routes])
-# Include routers
-
-# app.include_router(main_chatbot_route)
-app.include_router(Report_Generation_Router)
-app.include_router(Redis_session_router)
-app.include_router(redis_session_route_new)
-# app.include_router(Redis_session_router_old)
-app.include_router(Db_store_router)
-app.include_router(s3_bucket_router)
-app.include_router(Autoviz_router)
-# app.include_router(main_chat_router_v3)
-app.include_router(s3_viewer_router)
-
-app.include_router(s3_bucket_router1)
-app.include_router(main_chat_router_v4)
-app.include_router(test_router)
-
-# app.include_router(RetrievalQA_router)
-app.include_router(main_chatbot_route_v2)
-# app.include_router(Report_Generation_Router_v2)
-app.include_router(Report_Generation_Router_ppt_v2)
-app.include_router(Report_Generation_Router_pdf_v2)
-
-app.include_router(main_chat_router_v5)
-app.include_router(main_chat_router_v6)
-#==================================login and user management routes==================================
-app.include_router(login_apis_router)
-#---------------------------Qdrant router -----------------------
-
-# app.include_router(Qdrant_router)
-# ------------------- MIDDLEWARE -------------------
-
-@app.middleware("http")
-async def add_success_flag(request: Request, call_next):
-    response = await call_next(request)
-
-    # Only modify JSON responses
-    if "application/json" in response.headers.get("content-type", ""):
-        try:
-            body = b"".join([chunk async for chunk in response.body_iterator])
-            data = json.loads(body.decode())
-
-            # Add success flag
-            data["success"] = 200 <= response.status_code < 300
-
-            # Build new JSONResponse (auto handles Content-Length)
-            response = JSONResponse(
-                content=data,
-                status_code=response.status_code,
-                headers={k: v for k, v in response.headers.items() if k.lower() != "content-length"},
-            )
-        except Exception:
-            # fallback if response is not JSON parseable
-            pass
-    return response
-
-
+import json
+import warnings
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse
+from fastapi.middleware.cors import CORSMiddleware
+import os 
+import json
+import warnings
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse
+from fastapi.middleware.cors import CORSMiddleware
+import os 
+
+# Import routes
+# from routes import collections, ingestion, sessions, chat, system
+from Redis.sessions import Redis_session_router
+from Redis.sessions_new import redis_session_route_new
+from Redis.sessions_old import Redis_session_router_old
+# from Routes.main_chat_bot import main_chatbot_route
+from Routes.generate_report import Report_Generation_Router
+# from report_generation.report_generation import Report_Generation_Router_v2
+from s3.file_insertion_s3 import s3_bucket_router
+from s3.rough2 import test_router
+from backend.main import login_apis_router
+# from s3.r3 import s3_bucket_router1
+from s3.r6 import s3_bucket_router1
+from s3_viewer_backend.s3_viewer import s3_viewer_router 
+# from vector_db.retrival_qa_agent import RetrievalQA_router
+from main_chat_2.main_chat import main_chat_router_v4
+from main_chat5.main_chat_router import main_chat_router_v5
+from main_chat_v6.main_chat_router import main_chat_router_v6
+from report_generation.ppt_v2 import Report_Generation_Router_ppt_v2
+from report_generation.pdf_v3 import Report_Generation_Router_pdf_v2
+
+
+
+from Routes.main_agent_chat_bot_v2 import main_chatbot_route_v2
+# from vector_db import qdrant_crud
+
+# from vector_db.qdrant_crud import Qdrant_router
+from agent_tools.autovis_tool import Autoviz_router
+# from main_chat_2.main_chat import main_chat_router_v3
+from DB_store_backup.agentic_context_convoid_management import Db_store_router
+# Suppress warnings
+warnings.filterwarnings("ignore", message="Qdrant client version.*is incompatible.*")
+
+app = FastAPI(title="Combined AI Agent with Qdrant Collections and Redis Session Management")
+
+# Add CORS middleware
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+print([route.path for route in app.routes])
+# Include routers
+
+# app.include_router(main_chatbot_route)
+app.include_router(Report_Generation_Router)
+app.include_router(Redis_session_router)
+app.include_router(redis_session_route_new)
+# app.include_router(Redis_session_router_old)
+app.include_router(Db_store_router)
+app.include_router(s3_bucket_router)
+app.include_router(Autoviz_router)
+# app.include_router(main_chat_router_v3)
+app.include_router(s3_viewer_router)
+
+app.include_router(s3_bucket_router1)
+app.include_router(main_chat_router_v4)
+app.include_router(test_router)
+
+# app.include_router(RetrievalQA_router)
+app.include_router(main_chatbot_route_v2)
+# app.include_router(Report_Generation_Router_v2)
+app.include_router(Report_Generation_Router_ppt_v2)
+app.include_router(Report_Generation_Router_pdf_v2)
+
+# app.include_router(main_chat_router_v5)
+app.include_router(main_chat_router_v6)
+#==================================login and user management routes==================================
+app.include_router(login_apis_router)
+#---------------------------Qdrant router -----------------------
+
+# app.include_router(Qdrant_router)
+# ------------------- MIDDLEWARE -------------------
+
+@app.middleware("http")
+async def add_success_flag(request: Request, call_next):
+    response = await call_next(request)
+
+    # Only modify JSON responses
+    if "application/json" in response.headers.get("content-type", ""):
+        try:
+            body = b"".join([chunk async for chunk in response.body_iterator])
+            data = json.loads(body.decode())
+
+            # Add success flag
+            data["success"] = 200 <= response.status_code < 300
+
+            # Build new JSONResponse (auto handles Content-Length)
+            response = JSONResponse(
+                content=data,
+                status_code=response.status_code,
+                headers={k: v for k, v in response.headers.items() if k.lower() != "content-length"},
+            )
+        except Exception:
+            # fallback if response is not JSON parseable
+            pass
+    return response
+
+

main_chat_v6/data_agent.py

@@ -1,452 +1,448 @@
 """
-Improved Data Agent with Real-time Phased Streaming
-Phases: thinking (agent reasoning), answer (final response), charts (visualizations)
+data_agent_v3.py
+
+Robust production-ready data agent with:
+- RobustPhaseDetector: hybrid phase detection (heuristics + JSON balancing + tool-call hooks)
+- sql_tool: runs SQL via DuckDB safely using asyncio.to_thread + timeout
+- load_and_register_datasets: loads Parquet/CSV via your s3.read_files helper
+- stream_analysis: streams model output, detects phases robustly, and yields (phase, content) tuples
+
+Notes:
+- Replace `Runner.run_streamed` with your actual model runner
+- `ResponseTextDeltaEvent` from OpenAI SDK is used as an example; adapt if your runner emits a different shape
 """
 
 import asyncio
 import json
+import logging
 import re
-from typing import AsyncGenerator, List, Tuple
-import duckdb
+from typing import AsyncGenerator, Dict, List, Tuple, Optional
+from dataclasses import dataclass, field
+from concurrent.futures import ThreadPoolExecutor
+
 import pandas as pd
-from openai.types.responses import ResponseTextDeltaEvent
-from agents import Agent, Runner, function_tool
-from s3.read_files import read_parquet_from_s3
+import duckdb
 
-# =====================================
-# CONFIGURATION
-# =====================================
+# Import your Runner & ResponseTextDeltaEvent types (adapt if different)
+from openai.types.responses import ResponseTextDeltaEvent  # keep for type-checking; adapt as needed
+from agents import Agent, Runner, function_tool  # adapt to your agent system
+from s3.read_files import read_parquet_from_s3  # existing helper to load Parquet from S3
 
-ALLOWED_SQL_KEYWORDS = ("select", "with")
+logger = logging.getLogger("data_agent_v3")
+logging.basicConfig(level=logging.INFO)
+
+# ========= Configuration =========
+DUCKDB_FILE = "agent_duckdb.db"  # file-backed DB to allow connection reuse
+DUCKDB_THREADS = 2
+DUCKDB_MEMORY_LIMIT = "1GB"
+SQL_TIMEOUT_SECONDS = 15  # per-query timeout
 MAX_RESULT_ROWS = 10000
 
-_registered_datasets = {}
-last_sql_query: str = ""
+# ThreadPoolExecutor for blocking DuckDB operations
+_BLOCKING_EXECUTOR = ThreadPoolExecutor(max_workers=8)
+
+
+# ========= DuckDB manager (safe, pooled via to_thread) =========
+class DuckDBManager:
+    """
+    Simple manager that returns a connection that's safe to use from a thread.
+    We use thread-based execution of DuckDB queries via asyncio.to_thread to avoid blocking the event loop.
+    """
+
+    def __init__(self, db_path: str = DUCKDB_FILE):
+        self.db_path = db_path
+        # A lightweight connection initializer to warm pragmas
+        self._init_db()
+
+    def _init_db(self):
+        # Establish a short-lived connection and set pragmas
+        conn = duckdb.connect(self.db_path)
+        try:
+            conn.execute(f"PRAGMA threads={DUCKDB_THREADS}")
+            # Memory limit (DuckDB may accept memory_limit pragma as a string in newer versions)
+            try:
+                conn.execute(f"PRAGMA memory_limit='{DUCKDB_MEMORY_LIMIT}'")
+            except Exception:
+                # Older duckdb versions may ignore
+                pass
+        finally:
+            conn.close()
+
+    def execute_query(self, sql: str, registered_tables: Dict[str, pd.DataFrame]) -> pd.DataFrame:
+        """
+        Blocking function: executes sql and returns a pandas DataFrame.
+        Designed to run in a worker thread via asyncio.to_thread.
+        """
+        conn = duckdb.connect(self.db_path)
+        try:
+            # Register provided DataFrames into the connection
+            for name, df in registered_tables.items():
+                conn.register(name, df)
+            # Execute query
+            res = conn.execute(sql).fetchdf()
+            return res
+        finally:
+            conn.close()
+
+
+# Create global manager instance
+_duckdb_manager = DuckDBManager()
+
+
+# ========= Dataset loading =========
+_registered_datasets: Dict[str, pd.DataFrame] = {}
 
-# =====================================
-# DATASET LOADING
-# =====================================
 
 def generate_dataset_context(df: pd.DataFrame, table_name: str) -> str:
-    """Generate comprehensive dataset context"""
-    context_parts = []
-    context_parts.append(f"Table: {table_name}")
-    context_parts.append(f"Rows: {len(df)}, Columns: {len(df.columns)}")
-    context_parts.append(f"Column Names: {', '.join(df.columns.tolist())}")
-    context_parts.append("\nData Types:")
-    for col, dtype in df.dtypes.items():
-        context_parts.append(f"  - {col}: {dtype}")
-    
-    context_parts.append("\nSample Data (first 3 rows):")
-    sample_str = df.head(3).to_string(index=False, max_cols=10)
-    context_parts.append(sample_str)
-    
-    numeric_cols = df.select_dtypes(include=['number']).columns.tolist()
-    if numeric_cols:
-        context_parts.append("\nNumeric Column Statistics:")
-        desc = df[numeric_cols].describe().transpose()
-        for col in numeric_cols[:5]:
+    parts = [f"Table: {table_name}", f"Rows: {len(df)}, Columns: {len(df.columns)}", f"Column Names: {', '.join(df.columns.tolist())}"]
+    parts.append("\nSample (first 3 rows):")
+    parts.append(df.head(3).to_string(index=False, max_cols=10))
+    numeric = df.select_dtypes(include=["number"]).columns.tolist()
+    if numeric:
+        parts.append("\nNumeric stats (first 5 numeric cols):")
+        desc = df[numeric].describe().transpose()
+        for col in numeric[:5]:
             if col in desc.index:
-                context_parts.append(
-                    f"  - {col}: mean={desc.loc[col, 'mean']:.2f}, "
-                    f"min={desc.loc[col, 'min']:.2f}, max={desc.loc[col, 'max']:.2f}"
-                )
-    
-    return "\n".join(context_parts)
-
-def load_and_register_datasets(file_paths: List[str]) -> str:
-    """Load datasets and register them for SQL queries"""
+                parts.append(f" - {col}: mean={desc.loc[col,'mean']:.2f}, min={desc.loc[col,'min']:.2f}, max={desc.loc[col,'max']:.2f}")
+    return "\n".join(parts)
+
+
+def load_and_register_datasets(file_paths):
     global _registered_datasets
-    _registered_datasets.clear()
-    context_parts = ["Available datasets for SQL queries:\n"]
-    
-    for idx, file_path in enumerate(file_paths, start=1):
-        table_name = f"CSV{idx}"
-        try:
-            print(f"Loading {file_path} as {table_name}")
-            df = read_parquet_from_s3(file_path)
-            _registered_datasets[table_name] = df
-            dataset_context = generate_dataset_context(df, table_name)
-            context_parts.append(f"\n{dataset_context}\n")
-            context_parts.append("="*70)
-            print(f"Loaded {table_name}: {len(df)} rows, {len(df.columns)} columns")
-        except Exception as e:
-            print(f"Error loading {file_path}: {str(e)}")
-            raise
-    
-    return "\n".join(context_parts)
-
-# =====================================
-# SQL TOOL
-# =====================================
+    _registered_datasets = {}
+
+    ctx_parts = []
+    for i, fp in enumerate(file_paths, start=1):
+        table_name = f"CSV{i}"
+        df = read_parquet_from_s3(fp)
+
+        # 🔥 FIX COLUMN NAMES HERE
+        df.columns = (
+            df.columns
+              .str.strip()
+              .str.replace(" ", "_")
+              .str.replace(r"[^\w]+", "", regex=True)
+        )
+
+        _registered_datasets[table_name] = df
+        ctx_parts.append(generate_dataset_context(df, table_name))
+
+    return "\n\n".join(ctx_parts)
+
+
+
+# ========= Robust Phase Detector =========
+
+@dataclass
+class RobustPhaseDetector:
+    """
+    Hybrid phase detector that combines:
+    - soft XML/tag hints (best-effort)
+    - content heuristics (thinking/reasoning/answer)
+    - tool-call hooks (you should emit explicit tokens in Runner for tool calls)
+    - JSON brace balancing for charts
+    """
+    buffer: str = ""
+    thinking_emitted: bool = False
+    reasoning_emitted: bool = False
+    answer_started: bool = False
+    charts: List[Dict] = field(default_factory=list)
+    brace_count: int = 0
+    current_json_chunk: str = ""
+    chart_candidates: List[str] = field(default_factory=list)
+
+    thinking_keywords = [
+        "let me", "first", "need to", "i will", "step", "query", "sql", "check", "analyze", "understand",
+        "explore", "calculate", "group by", "join", "filter", "plan", "strategy"
+    ]
+    reasoning_keywords = [
+        "i will run", "running query", "execute", "to compute", "will compute", "plan to", "step 1", "step 2",
+    ]
+    answer_markers = [
+        "results show", "analysis reveals", "key finding", "## ", "conclusion", "in summary", "the results",
+        "final", "therefore", "overall", "we find"
+    ]
+
+    def feed(self, delta: str) -> List[Tuple[str, object]]:
+        """
+        Accept raw streamed text (delta) and return list of (phase, content)
+        phases: THINKING, REASONING, ANSWER, CHART, ERROR, DONE
+        """
+        emissions: List[Tuple[str, object]] = []
+        if not delta:
+            return emissions
+
+        # Append delta to buffer for heuristic detection
+        self.buffer += delta
+
+        # --------- 1) JSON/Chart detection via brace balancing ----------
+        # Look for a possible JSON start (with "$schema" or vega indicator) in incoming data
+        # We try to capture complete JSON objects using brace balancing.
+        # If JSON is extracted successfully, emit CHART with parsed dict.
+        # Do this before other heuristics so charts show asap.
+        # Scan buffer char by char (efficient on chunk sizes)
+        i = 0
+        while i < len(delta):
+            ch = delta[i]
+            if ch == "{":
+                # if starting new JSON, and previous brace_count==0, start capturing
+                if self.brace_count == 0:
+                    # start new chunk
+                    self.current_json_chunk = "{"
+                else:
+                    self.current_json_chunk += "{"
+                self.brace_count += 1
+            elif ch == "}":
+                self.current_json_chunk += "}"
+                self.brace_count -= 1
+                if self.brace_count == 0:
+                    # Potential full JSON captured
+                    candidate = self.current_json_chunk.strip()
+                    # quick check for vega/schema
+                    if '"$schema"' in candidate or "vega" in candidate.lower() or "data" in candidate.lower():
+                        try:
+                            parsed = json.loads(candidate)
+                            # dedupe by content
+                            if parsed not in self.charts:
+                                self.charts.append(parsed)
+                                emissions.append(("CHART", parsed))
+                                # remove candidate from buffer to avoid reprocessing
+                                self.buffer = self.buffer.replace(candidate, "", 1)
+                        except Exception:
+                            # Not yet valid JSON - maybe streaming partial, continue trying next deltas
+                            # Keep candidate in current_json_chunk for next feed
+                            pass
+                    self.current_json_chunk = ""
+            else:
+                if self.brace_count > 0:
+                    self.current_json_chunk += ch
+            i += 1
+
+        # --------- 2) Tool-call / reasoning boundary detection ----------
+        # If Runner emits explicit markers for tool calls, you should hook into them.
+        # Here we support a common pattern: if buffer contains something like "[TOOL_CALL_START]" or "SQL TOOL:",
+        # we treat it as REASONING beginning.
+        tool_markers = ["[TOOL_CALL_START]", "[SQL_TOOL]", "sql_tool(", "EXECUTE SQL:", "RUN SQL:"]
+        lower_buffer = self.buffer.lower()
+
+        # THINKING detection (soft)
+        if not self.thinking_emitted:
+            if len(self.buffer) > 80 and any(kw in lower_buffer for kw in self.thinking_keywords):
+                snippet = self._take_and_trim_for_emission(self.buffer, max_chars=1000)
+                emissions.append(("THINKING", snippet))
+                self.thinking_emitted = True
+                # keep buffer for subsequent phases
+        # REASONING detection
+        if not self.reasoning_emitted:
+            if any(m.lower() in lower_buffer for m in tool_markers) or any(kw in lower_buffer for kw in self.reasoning_keywords):
+                snippet = self._take_and_trim_for_emission(self.buffer, max_chars=2000)
+                emissions.append(("REASONING", snippet))
+                self.reasoning_emitted = True
+
+        # ANSWER detection
+        if (self.thinking_emitted or self.reasoning_emitted) and not self.answer_started:
+            if any(marker in lower_buffer for marker in self.answer_markers) or len(self.buffer) > 1500:
+                # consider this the start of the answer/narrative
+                snippet = self._take_and_trim_for_emission(self.buffer, max_chars=4000)
+                emissions.append(("ANSWER", snippet))
+                self.answer_started = True
+                # keep buffer for subsequent answer growth
+
+        # Also attempt to detect inline "final" answer markers even after answer_started to emit deltas
+        if self.answer_started:
+            # Split newly appended buffer into reasonable paragraph chunks for streaming UX
+            new_paragraphs = self._drain_buffer_into_paragraphs()
+            for p in new_paragraphs:
+                emissions.append(("ANSWER", p))
+
+        return emissions
+
+    def _take_and_trim_for_emission(self, buf: str, max_chars: int = 1000) -> str:
+        """Return upto max_chars of buffer for emission (non-destructive)"""
+        snippet = buf[:max_chars]
+        return snippet.strip()
+
+    def _drain_buffer_into_paragraphs(self) -> List[str]:
+        """
+        Extract paragraphs from buffer (up to some limit), leaving remainder in buffer.
+        This helps stream incremental answer paragraphs rather than massive blobs.
+        """
+        # Split on two newlines or single newline when long
+        paragraphs = []
+        # If very long buffer, cut into 800-char chunks
+        if len(self.buffer) > 0:
+            if "\n\n" in self.buffer:
+                parts = self.buffer.split("\n\n")
+            else:
+                # chunky split
+                parts = [self.buffer[i:i+800] for i in range(0, len(self.buffer), 800)]
+            # emit first 1-3 parts and keep rest
+            take = min(3, len(parts))
+            for _ in range(take):
+                part = parts.pop(0).strip()
+                if part:
+                    paragraphs.append(part)
+            # reconstruct buffer with remaining parts
+            self.buffer = "\n\n".join(parts)
+        return paragraphs
+
+
+# ========= SQL TOOL =========
 
 @function_tool
 async def sql_tool(sql_query: str, user_query: str = "") -> dict:
-    """Execute SQL query and return results"""
-    global _registered_datasets, last_sql_query
-    last_sql_query = sql_query
-
-    print(f"\n{'='*70}")
-    print(f"Executing SQL:\n{sql_query}")
-    print(f"{'='*70}\n")
-
-    normalized_query = sql_query.strip().lower()
-    if not normalized_query.startswith(ALLOWED_SQL_KEYWORDS):
-        raise ValueError(
-            f"Only {', '.join(k.upper() for k in ALLOWED_SQL_KEYWORDS)} queries allowed."
-        )
+    """
+    Async SQL tool that executes on DuckDB using asyncio.to_thread (thread-safe)
+    - Enforces a per-query timeout using asyncio.wait_for
+    - Registers in-memory pandas DataFrames for tables (CSV1, CSV2, ...)
+    """
+    if not sql_query:
+        raise ValueError("Empty SQL query")
+
+    normalized = sql_query.strip().lower()
+    if not (normalized.startswith("select") or normalized.startswith("with")):
+        raise ValueError("Only SELECT/WITH queries allowed for safety")
 
     if not _registered_datasets:
-        raise ValueError("No datasets loaded.")
+        raise ValueError("No datasets registered for SQL execution")
 
-    conn = duckdb.connect(":memory:")
+    # Execute in a thread pool and enforce timeout
     try:
-        for name, df in _registered_datasets.items():
-            conn.register(name, df)
-        result_df = conn.execute(sql_query).df()
+        df = await asyncio.wait_for(
+            asyncio.to_thread(_duckdb_manager.execute_query, sql_query, _registered_datasets),
+            timeout=SQL_TIMEOUT_SECONDS,
+        )
+    except asyncio.TimeoutError:
+        raise TimeoutError(f"SQL execution exceeded {SQL_TIMEOUT_SECONDS}s timeout")
     except Exception as e:
-        conn.close()
+        # Bubble up a clear error
+        logger.exception("SQL execution failed")
         raise ValueError(f"SQL execution failed: {str(e)}")
-    finally:
-        conn.close()
 
-    if len(result_df) > MAX_RESULT_ROWS:
-        result_df = result_df.head(MAX_RESULT_ROWS)
-
-    summary_lines = [
-        f"Returned {len(result_df)} rows, {len(result_df.columns)} columns"
-    ]
-    if not result_df.empty:
-        summary_lines.append("\nPreview (first 10 rows):")
-        summary_lines.append(result_df.head(10).to_string(index=False, max_rows=10))
-    
-    result_summary = "\n".join(summary_lines)
+    # Trim large results
+    if len(df) > MAX_RESULT_ROWS:
+        df = df.head(MAX_RESULT_ROWS)
 
-    print(f"SQL executed successfully\n")
+    # Build summary
+    result_summary = [f"Returned {len(df)} rows, {len(df.columns)} columns"]
+    if not df.empty:
+        result_summary.append("\nPreview (first 10 rows):")
+        result_summary.append(df.head(10).to_string(index=False, max_rows=10))
     return {
         "sql_query": sql_query,
-        "result_summary": result_summary,
-        "columns": result_df.columns.tolist(),
-        "row_count": len(result_df)
+        "result_summary": "\n".join(result_summary),
+        "columns": df.columns.tolist(),
+        "row_count": len(df),
+        "data_preview": df.head(50).to_dict(orient="records")  # useful for immediate charts
     }
 
-# =====================================
-# DATA ANALYST AGENT WITH STRUCTURED OUTPUT
-# =====================================
+
+# ========= Agent creation utility =========
 
 def create_data_analyst_agent(dataset_context: str) -> Agent:
-    """Create data analyst agent with explicit phase instructions"""
-    return Agent(
-        name="DataStory_AI",
-        model="gpt-4o",
-        tools=[sql_tool],
-        instructions=f"""You are **DataStory AI**, an elite data analyst who combines deep analytical reasoning with compelling storytelling.
+    """
+    Build the agent with detailed instructions. Keep structured hints, but DO NOT rely on them.
+    The Runner should stream raw text; the RobustPhaseDetector will detect phases.
+    """
+    instructions = f"""
+You are DataStory AI. Use the dataset context below to plan SQL queries, run them with sql_tool(sql_query="..."),
+and produce a final data story. You may include chart JSON blobs (Vega-Lite) in your output; when you do, ensure valid JSON.
 
 DATASET CONTEXT:
 {dataset_context}
 
-===== CRITICAL: STRUCTURED RESPONSE FORMAT =====
-
-You MUST structure your response in THREE distinct phases using XML tags:
-
-**PHASE 1: THINKING (Your Reasoning Process)**
-<thinking>
-- Explain your analytical approach step-by-step
-- Describe what SQL queries you'll run and WHY
-- Outline the insights you're looking for
-- Discuss any data quality considerations
-- Map out your visualization strategy
-Example:
-"To answer this question, I need to:
-1. First examine the distribution of sales across regions by querying CSV1
-2. Calculate year-over-year growth rates to identify trends
-3. Identify top performers and outliers
-4. Create visualizations showing: (a) regional comparison, (b) time series trend, (c) top 10 performers"
-</thinking>
-
-**PHASE 2: ANSWER (Your Analysis & Narrative)**
-<answer>
-Write your complete data story here:
-- Present key findings with specific numbers
-- Provide context and business implications  
-- Explain patterns, trends, and anomalies
-- Use clear section headers
-- Be specific: "Sales increased by 23.5% in Q3" not "Sales increased significantly"
-</answer>
-
-**PHASE 3: CHARTS (Visualizations)**
-For each visualization, use this EXACT format:
-
-<datastory_chart>
-<chart_title>Descriptive Title</chart_title>
-<chart_description>What this chart shows with specific numbers</chart_description>
-<vega_spec>
-{{
-  "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
-  "title": {{"text": "Chart Title", "fontSize": 18, "fontWeight": "bold"}},
-  "width": 700,
-  "height": 400,
-  "data": {{"values": [
-    {{"category": "A", "value": 100}},
-    {{"category": "B", "value": 150}}
-  ]}},
-  "mark": {{"type": "bar", "tooltip": true, "cornerRadius": 4}},
-  "encoding": {{
-    "x": {{"field": "category", "type": "nominal", "axis": {{"labelAngle": 0}}}},
-    "y": {{"field": "value", "type": "quantitative", "axis": {{"title": "Value"}}}}
-  }},
-  "config": {{
-    "axis": {{"labelFontSize": 12, "titleFontSize": 14}},
-    "legend": {{"labelFontSize": 12, "titleFontSize": 14}}
-  }}
-}}
-</vega_spec>
-</datastory_chart>
-
-===== WORKFLOW =====
-1. **START WITH THINKING**: Always begin with <thinking> tags explaining your analytical plan
-2. **Execute SQL queries**: Use sql_tool(sql_query="...", user_query="...")
-3. **WRITE ANSWER**: Present insights in <answer> tags with specific numbers
-4. **CREATE CHARTS**: Generate 2-3 visualizations wrapped in <datastory_chart> tags
-
-===== SQL QUERY RULES =====
-- Table names: CSV1, CSV2, CSV3, etc. (from dataset context above)
-- Only SELECT and WITH statements allowed
-- Always include user_query parameter in sql_tool calls
-- Use meaningful column aliases
-- Limit results appropriately for visualizations
-
-===== VISUALIZATION BEST PRACTICES =====
-- Include ACTUAL data in "values" array (not placeholders)
-- Use appropriate chart types: bar, line, point, area, arc (pie)
-- Add tooltips: "mark": {{"type": "bar", "tooltip": true}}
-- Use clear axis labels and titles
-- Color schemes: use "color" encoding for categorical data
-- For time series: parse dates with "timeUnit"
-
-===== QUALITY STANDARDS =====
-✓ Be specific with numbers: "Revenue increased by $2.3M (18.7%)" 
-✓ Provide context: Compare to benchmarks, previous periods, or goals
-✓ Explain WHY patterns exist, not just WHAT they are
-✓ Structure with clear headers: ## Key Finding 1, ## Key Finding 2
-✓ Generate 2-3 complementary visualizations
-✓ End with actionable recommendations
-
-===== EXAMPLE RESPONSE STRUCTURE =====
-
-<thinking>
-Let me analyze the sales data systematically:
-
-1. **Data Exploration**: I'll query CSV1 to understand sales distribution across regions and time periods
-2. **Trend Analysis**: Calculate monthly growth rates to identify seasonal patterns  
-3. **Performance Ranking**: Identify top 10 products by revenue
-4. **Visualization Plan**:
-   - Chart 1: Regional sales comparison (bar chart)
-   - Chart 2: Monthly trend with growth rate (line chart)
-   - Chart 3: Top 10 products (horizontal bar)
-
-SQL Strategy: First, aggregate sales by region, then by month, finally rank products.
-</thinking>
-
-<answer>
-## Executive Summary
-Analysis of 50,000 transactions reveals strong regional disparities and seasonal trends...
-
-## Regional Performance
-The West region dominates with $4.2M in sales (42% of total), followed by East at $3.1M (31%)...
-
-[Continue with detailed findings, specific numbers, and business implications]
-</answer>
-
-<datastory_chart>
-<chart_title>Sales by Region - West Region Leads with 42% Share</chart_title>
-<chart_description>West region generated $4.2M, 35% ahead of second-place East region ($3.1M)</chart_description>
-<vega_spec>
-{{
-  "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
-  "title": {{"text": "Sales by Region ($ Millions)", "fontSize": 18}},
-  "width": 700,
-  "height": 400,
-  "data": {{"values": [
-    {{"region": "West", "sales": 4.2}},
-    {{"region": "East", "sales": 3.1}},
-    {{"region": "North", "sales": 1.8}},
-    {{"region": "South", "sales": 1.5}}
-  ]}},
-  "mark": {{"type": "bar", "tooltip": true, "color": "#4C78A8"}},
-  "encoding": {{
-    "x": {{"field": "region", "type": "nominal", "sort": "-y"}},
-    "y": {{"field": "sales", "type": "quantitative", "title": "Sales ($M)"}}
-  }}
-}}
-</vega_spec>
-</datastory_chart>
-
-REMEMBER: Always use the three-phase structure with XML tags. Your reasoning goes in <thinking>, analysis in <answer>, and charts in <datastory_chart> blocks.
+Important: Your structured tags (like <thinking>) are optional; the system detects phases heuristically.
 """
-    )
-
-# =====================================
-# PHASE-AWARE CHART BUFFER
-# =====================================
-
-class PhaseAwareBuffer:
-    """Buffer that separates content into thinking, answer, and chart phases"""
-    
-    def __init__(self):
-        self.full_buffer = ""
-        self.thinking_complete = False
-        self.answer_complete = False
-        self.extracted_charts = []
-    
-    def add_text(self, text: str) -> List[Tuple[str, str]]:
-        """
-        Add text and extract phase-separated content.
-        Returns: List of (phase, content) tuples
-        """
-        self.full_buffer += text
-        emissions = []
-        
-        # Extract thinking phase
-        if not self.thinking_complete:
-            thinking_match = re.search(r'<thinking>(.*?)</thinking>', self.full_buffer, re.DOTALL)
-            if thinking_match:
-                thinking_content = thinking_match.group(1).strip()
-                emissions.append(("thinking", thinking_content))
-                self.thinking_complete = True
-                # Remove from buffer
-                self.full_buffer = self.full_buffer[thinking_match.end():]
-        
-        # Extract answer phase
-        if self.thinking_complete and not self.answer_complete:
-            answer_match = re.search(r'<answer>(.*?)</answer>', self.full_buffer, re.DOTALL)
-            if answer_match:
-                answer_content = answer_match.group(1).strip()
-                emissions.append(("answer", answer_content))
-                self.answer_complete = True
-                # Remove from buffer
-                self.full_buffer = self.full_buffer[answer_match.end():]
-        
-        # Extract charts (can be multiple)
-        if self.answer_complete:
-            chart_pattern = re.compile(r'<datastory_chart>(.*?)</datastory_chart>', re.DOTALL)
-            matches = list(chart_pattern.finditer(self.full_buffer))
-            
-            for match in matches:
-                chart_content = match.group(1)
-                
-                # Parse Vega spec
-                vega_pattern = r'<vega_spec>(.*?)</vega_spec>'
-                vega_match = re.search(vega_pattern, chart_content, re.DOTALL)
-                
-                if vega_match:
-                    try:
-                        vega_json_str = vega_match.group(1).strip()
-                        vega_spec = json.loads(vega_json_str)
-                        
-                        # Only emit if not already extracted
-                        if vega_spec not in self.extracted_charts:
-                            self.extracted_charts.append(vega_spec)
-                            emissions.append(("charts", vega_spec))
-                    except json.JSONDecodeError as e:
-                        print(f"Failed to parse chart JSON: {e}")
-                
-                # Remove from buffer
-                self.full_buffer = self.full_buffer[match.end():]
-        
-        return emissions
+    return Agent(name="DataStory_AI", model="gpt-4o", tools=[sql_tool], instructions=instructions)
 
-# =====================================
-# PHASED STREAMING WITH AGENT REASONING
-# =====================================
 
-async def stream_analysis(query: str, file_paths: List[str]) -> AsyncGenerator[Tuple[str, str], None]:
+# ========= Streaming analysis =========
+
+async def stream_analysis(query: str, file_paths: List[str]) -> AsyncGenerator[Tuple[str, object], None]:
     """
-    Stream analysis with agent's actual reasoning in thinking phase.
-    Yields: (phase, content) tuples where phase is:
-    - THINKING: Agent's reasoning process
-    - ANSWER: Final analysis and narrative
-    - CHART: Vega-Lite chart specification
-    - ERROR, DONE: Control signals
+    Stream analysis for data queries. Yields tuples (phase_tag, content)
+    Phases: THINKING, REASONING, ANSWER, CHART, ERROR, DONE
     """
-    print(f"\n{'='*70}")
-    print("ANALYSIS REQUEST (phased streaming with agent reasoning)")
-    print(f"Files: {', '.join(file_paths)}")
-    print(f"Query: {query}")
-    print(f"{'='*70}\n")
-
-    # Load datasets
+    logger.info("Starting stream_analysis: query=%s files=%s", query, file_paths)
+    # 1) Load datasets
     try:
         dataset_context = load_and_register_datasets(file_paths)
-        print(f"\nDataset Context loaded: {len(dataset_context)} chars\n")
     except Exception as e:
+        logger.exception("Failed to load datasets")
         yield ("ERROR", f"Failed to load datasets: {str(e)}")
         yield ("DONE", "")
         return
 
-    # Create agent
     agent = create_data_analyst_agent(dataset_context)
-    
-    # Initialize phase-aware buffer
-    phase_buffer = PhaseAwareBuffer()
-    
+    detector = RobustPhaseDetector()
+
+    # IMPORTANT: Runner.run_streamed must produce an object with .stream_events() or be adapted.
+    # The event stream must yield events that contain incremental text deltas, e.g. ResponseTextDeltaEvent
     try:
         result = Runner.run_streamed(agent, input=query)
-        
-        async for event in result.stream_events():
-            if (
-                event.type == "raw_response_event"
-                and isinstance(event.data, ResponseTextDeltaEvent)
-            ):
-                delta: str = event.data.delta
-                
-                # Add to buffer and extract phase-separated content
-                emissions = phase_buffer.add_text(delta)
-                
-                for phase, content in emissions:
-                    if phase == "thinking":
-                        # Stream thinking in chunks for better UX
-                        sentences = content.split('\n')
-                        for sentence in sentences:
-                            if sentence.strip():
-                                yield ("THINKING", sentence.strip() + "\n")
-                                await asyncio.sleep(0.05)
-                    
-                    elif phase == "answer":
-                        # Stream answer in chunks
-                        paragraphs = content.split('\n\n')
-                        for para in paragraphs:
-                            if para.strip():
-                                yield ("ANSWER", para.strip() + "\n\n")
-                                await asyncio.sleep(0.05)
-                    
-                    elif phase == "charts":
-                        # Emit complete chart
-                        yield ("CHART", content)
-                        await asyncio.sleep(0.1)
 
-    except Exception as e:
-        yield ("ERROR", f"Streaming error: {str(e)}")
-    
+        # Example: the Runner here is assumed to have an async generator `stream_events()`
+        async for event in result.stream_events():
+            # If your runner provides a different event shape, adapt this block
+            # We look for ResponseTextDeltaEvent style events containing .data.delta
+            try:
+                if getattr(event, "type", "") == "raw_response_event" and isinstance(getattr(event, "data", None), ResponseTextDeltaEvent):
+                    delta = event.data.delta
+                    emissions = detector.feed(delta)
+
+                    for phase, content in emissions:
+                        # Normalize outputs so router can wrap them into SSE easily
+                        if phase == "THINKING":
+                            # Stream line-by-line for better UX
+                            for line in str(content).splitlines():
+                                if line.strip():
+                                    yield ("THINKING", line.strip())
+                        elif phase == "REASONING":
+                            # REASONING can be multi-line; stream in paragraphs
+                            for para in str(content).split("\n\n"):
+                                if para.strip():
+                                    yield ("REASONING", para.strip())
+                        elif phase == "ANSWER":
+                            # Answer may be paragraph chunk
+                            yield ("ANSWER", content if isinstance(content, str) else str(content))
+                        elif phase == "CHART":
+                            # content is parsed JSON dict
+                            yield ("CHART", content)
+                        else:
+                            yield ("ANSWER", str(content))
+                else:
+                    # Unknown event type: try to extract text if possible
+                    # Many runners emit plain delta strings
+                    delta_text = None
+                    # try attributes often present
+                    if hasattr(event, "data") and isinstance(event.data, str):
+                        delta_text = event.data
+                    elif hasattr(event, "delta"):
+                        delta_text = event.delta
+                    elif isinstance(event, str):
+                        delta_text = event
+
+                    if delta_text:
+                        emissions = detector.feed(delta_text)
+                        for phase, content in emissions:
+                            if phase == "CHART":
+                                yield ("CHART", content)
+                            else:
+                                yield (phase, content)
+            except Exception as inner_e:
+                logger.exception("Error processing event chunk")
+                yield ("ERROR", f"Processing error: {str(inner_e)}")
+
+    except asyncio.CancelledError:
+        logger.info("stream_analysis cancelled (client disconnected or server shutdown)")
+        yield ("ERROR", "Stream cancelled by client")
+    except Exception as exc:
+        logger.exception("stream_analysis uncaught exception")
+        yield ("ERROR", f"Streaming error: {str(exc)}")
+
+    # final done sentinel
     yield ("DONE", "")
 
-# =====================================
-# LEGACY STREAMING (BACKWARD COMPATIBILITY)
-# =====================================
 
-async def stream_analysis_legacy(query: str, file_paths: List[str]) -> AsyncGenerator[str, None]:
-    """
-    Original streaming without phases - for backward compatibility
-    """
-    dataset_context = load_and_register_datasets(file_paths)
-    agent = create_data_analyst_agent(dataset_context)
-    result = Runner.run_streamed(agent, input=query)
-    
-    async for event in result.stream_events():
-        if (
-            event.type == "raw_response_event"
-            and isinstance(event.data, ResponseTextDeltaEvent)
-        ):
-            yield event.data.delta
-    
-    yield "[DONE]"
+# ========= Legacy helper (optional) =========
+

main_chat_v6/main_chat_router.py

@@ -1,41 +1,49 @@
 """
-Main Router with True Agent Reasoning in Thinking Phase
-Phases: thinking (agent's reasoning), answer (analysis), charts (visualizations)
+main_chat_router_v7.py
+Production-ready FastAPI router with phased SSE streaming.
+
+- Uses RobustPhaseDetector from data_agent_v3
+- Streams JSON events with phases: thinking, reasoning, answer, charts, done
+- Handles cancellation and timeouts gracefully
 """
 
-from fastapi import APIRouter, HTTPException
-from fastapi.responses import StreamingResponse
-from datetime import datetime
-from typing import AsyncGenerator, List, Optional, Dict, Tuple
 import json
-import asyncio
-import os
-import sys
 import uuid
-from pathlib import Path
-import httpx
-import re
+import asyncio
+import logging
+from datetime import datetime
+from typing import AsyncGenerator, List, Dict, Optional
 
-PROJECT_ROOT = Path(__file__).resolve().parents[2]
-if str(PROJECT_ROOT) not in sys.path:
-    sys.path.insert(0, str(PROJECT_ROOT))
+from fastapi import APIRouter, HTTPException, Request
+from fastapi.responses import StreamingResponse
 
-from Redis.sessions_new import *
-from main_chat_v6.data_agent import stream_analysis
-from main_chat_v6.multiagent import stream_multiagent
+# Import the new data agent implementation
+from main_chat_v6.data_agent import *
+from main_chat_v6.multiagent import *
+
+# Your session/redis helpers (preserve existing API)
+from Redis.sessions_new import (
+    create_session,
+    get_session,
+    get_message_history,
+    add_message,
+    update_session_title_if_needed,
+)
+
+# DB storage helper
+import httpx
 from pydantic import BaseModel
-from dotenv import load_dotenv
-from retrieve_secret import *
 
-load_dotenv()
+logger = logging.getLogger("main_chat_router_v7")
+logging.basicConfig(level=logging.INFO)
+
+main_chat_router_v6 = APIRouter(prefix="/main_chatbot/v7", tags=["main_chatbot_v7"])
 
-# =====================================
-# MODELS
-# =====================================
 
 class Author(BaseModel):
     role: str
 
+
 class ChatRequest(BaseModel):
     user_login_id: str
     session_id: Optional[str] = None
@@ -47,423 +55,184 @@ class ChatRequest(BaseModel):
     create_time: Optional[str] = None
     token: Optional[str] = None
 
-class ChatResponse(BaseModel):
-    session_id: str
-    user_message: str
-    assistant_response: Dict
-    is_new_session: bool
-    session_title: str
-    timestamp: str
-    files_processed: Optional[List[str]] = None
-    artifacts_paths: Optional[List[str]] = None
-
-main_chat_router_v6 = APIRouter(prefix="/main_chatbot/v6", tags=["main_chatbot_v6"])
-
-# =====================================
-# HELPER FUNCTIONS
-# =====================================
 
 async def make_convo_id(suffix: str = "ss") -> List[str]:
-    """Generate UUID with suffix in mutable list"""
     base_uuid = str(uuid.uuid4())
     convo_id = base_uuid[:-2] + suffix
     return [convo_id]
 
-def determine_agent_type(request: ChatRequest) -> bool:
-    """Determine which agent to use. Returns True for data_analyst."""
-    if request.metadata and "file_paths" in request.metadata:
-        file_paths = request.metadata["file_paths"]
-        if file_paths and len(file_paths) > 0:
-            return True
-    
-    query_lower = request.query.lower()
-    data_keywords = [
-        "analyze", "analysis", "dataset", "csv", "data", "statistics",
-        "visualize", "visualization", "chart", "graph", "plot",
-        "sql", "query", "select", "table", "column", "row"
-    ]
-    
-    return any(keyword in query_lower for keyword in data_keywords)
-
-def create_phase_chunk(content: str, phase: str) -> dict:
-    """Create standardized phase chunk"""
-    return {
-        "type": "chat:completion",
-        "data": {
-            "delta_content": content,
-            "phase": phase
-        }
-    }
 
-async def store_conversation_to_db(
-    user_login_id: str,
-    session_id: str,
-    user_query: str,
-    ai_response: str,
-    metadata: Dict[str, any],
-    artifacts: list = None,
-    suffix: str = "ss",
-    convo_ids: List[str] = None,
-    db_api_url: str = "https://mr-mvp-api-dev.dev.ingenspark.com/Db_store_router/conversations"
-) -> Optional[Dict]:
-    """Store conversation to database API"""
+async def store_conversation_to_db(payload: dict, db_api_url: str = "https://mr-mvp-api-dev.dev.ingenspark.com/Db_store_router/conversations") -> Optional[Dict]:
     try:
-        if convo_ids is None or not convo_ids:
-            convo_ids = await make_convo_id(suffix)
-        
-        convo_id = convo_ids[0]
-        query_id = f"q_{uuid.uuid4()}"
-        
-        payload = {
-            "user_id": user_login_id,
-            "convo_id": convo_id,
-            "data": {
-                "user_query": {
-                    "query_id": query_id,
-                    "text": user_query,
-                    "user_metadata": {
-                        "location": metadata.get("user", {}).get("location", "Unknown"),
-                        "language": metadata.get("user", {}).get("language", "English")
-                    }
-                },
-                "response": {
-                    "text": ai_response,
-                    "status": "success",
-                    "response_time": datetime.now().isoformat() + "Z",
-                    "duration": f"{metadata.get('processing_time', 0):.2f}s",
-                    "artifacts": artifacts if artifacts else []
-                },
-                "metadata": {
-                    "processing_node": {
-                        "model_version": metadata.get("model_used", "gpt-4o"),
-                        "api_latency_ms": int(metadata.get("processing_time", 0) * 1000),
-                        "search_mode": metadata.get("search_mode", "sql_analysis")
-                    }
-                }
-            },
-            "is_saved": False
-        }
-        
         async with httpx.AsyncClient(timeout=30.0) as client:
-            response = await client.post(db_api_url, json=payload, headers={
-                "accept": "application/json",
-                "Content-Type": "application/json"
-            })
-            response.raise_for_status()
-            return response.json()
-            
+            r = await client.post(db_api_url, json=payload, headers={"accept": "application/json", "Content-Type": "application/json"})
+            r.raise_for_status()
+            return r.json()
     except Exception as e:
-        print(f"Error storing conversation to DB: {e}")
+        logger.exception("Error storing conversation to DB")
         return None
 
-# =====================================
-# PHASED STREAMING WITH AGENT REASONING
-# =====================================
-
-async def stream_chat_response_phased(
-    request: ChatRequest,
-    session_id: str,
-    is_new_session: bool,
-    conversation_history: List,
-    use_data_analyst: bool,
-    convo_ids: List[str]
-) -> AsyncGenerator[str, None]:
-    """
-    Stream response with phases from agent:
-    - thinking: Agent's actual reasoning process
-    - answer: Analysis and narrative
-    - charts: Visualizations
-    """
-    full_response = ""
-    artifacts = []
-    user_message_id = str(uuid.uuid4())
-    ai_message_id = str(uuid.uuid4())
-    request_id = f"req_{uuid.uuid4().hex[:26].upper()}"
-    start_time = datetime.now()
-    
-    thinking_buffer = ""
-    answer_buffer = ""
-    
-    try:
-        # ===== INITIAL METADATA =====
-        initial_metadata = {
-            "status": "ok",
-            "api_version": "v1",
-            "request_id": request_id,
-            "event": "stream_started",
-            "data": {
-                "session_id": session_id,
-                "user_message_id": user_message_id,
-                "ai_message_id": ai_message_id,
-                "is_new_session": is_new_session,
-                "agent_type": "data_analyst_gpt4o" if use_data_analyst else "multiagent",
-                "timestamp": datetime.now().timestamp(),
-                "userLoginId": request.user_login_id
-            }
-        }
-        yield f"data: {json.dumps(initial_metadata)}\n\n"
-        await asyncio.sleep(0)
-
-        # ===== STREAM FROM AGENT =====
-        if use_data_analyst:
-            file_paths = request.metadata.get("file_paths", []) if request.metadata else []
-            
-            if not file_paths:
-                error_msg = "Data analyst requires file_paths in metadata"
-                yield f"data: {json.dumps({'error': error_msg})}\n\n"
-                yield "data: [DONE]\n\n"
-                return
-            
-            chart_count = 0
-            
-            # Stream from data analyst agent
-            async for phase, content in stream_analysis(request.query, file_paths):
-                if phase == "DONE":
-                    break
-                
-                elif phase == "ERROR":
-                    error_response = {
-                        "status": "error",
-                        "event": "error_response",
-                        "data": {
-                            "error": {
-                                "code": "STREAM_ERROR",
-                                "message": content,
-                                "details": {}
-                            },
-                            "request_id": request_id,
-                            "timestamp": datetime.now().timestamp()
-                        }
-                    }
-                    yield f"data: {json.dumps(error_response)}\n\n"
-                
-                elif phase == "THINKING":
-                    # Agent's reasoning - stream as thinking phase
-                    thinking_buffer += content
-                    phase_chunk = create_phase_chunk(content, "thinking")
-                    yield f"data: {json.dumps(phase_chunk)}\n\n"
-                    await asyncio.sleep(0)
-                
-                elif phase == "ANSWER":
-                    # Agent's analysis - stream as answer phase
-                    answer_buffer += content
-                    full_response += content
-                    phase_chunk = create_phase_chunk(content, "answer")
-                    yield f"data: {json.dumps(phase_chunk)}\n\n"
-                    await asyncio.sleep(0)
-                
-                elif phase == "CHART":
-                    # Visualization artifact
-                    chart_count += 1
-                    artifacts.append(content)
-                    
-                    chart_event = {
-                        "type": "chart:artifact",
-                        "data": {
-                            "chart_index": chart_count,
-                            "vega_spec": content,
-                            "phase": "charts"
-                        }
-                    }
-                    yield f"data: {json.dumps(chart_event)}\n\n"
-                    await asyncio.sleep(0.1)
-        
-        else:
-            # Multiagent flow (if needed)
-            async for chunk in stream_multiagent(request.query):
-                if chunk == "[DONE]":
-                    break
-                elif chunk.startswith("[ERROR]"):
-                    error_response = {
-                        "status": "error",
-                        "event": "error_response",
-                        "data": {
-                            "error": {
-                                "code": "STREAM_ERROR",
-                                "message": chunk[7:],
-                                "details": {}
-                            },
-                            "request_id": request_id,
-                            "timestamp": datetime.now().timestamp()
-                        }
-                    }
-                    yield f"data: {json.dumps(error_response)}\n\n"
-                else:
-                    full_response += chunk
-                    phase_chunk = create_phase_chunk(chunk, "answer")
-                    yield f"data: {json.dumps(phase_chunk)}\n\n"
-                    await asyncio.sleep(0)
-
-        # ===== FINAL PAYLOAD =====
-        processing_time = (datetime.now() - start_time).total_seconds()
-        session_data = get_session(request.user_login_id, session_id)
-        
-        # Combine thinking + answer for full response storage
-        full_response_for_storage = thinking_buffer + "\n\n" + answer_buffer
-        
-        final_payload = {
-            "status": "ok",
-            "api_version": "v1",
-            "request_id": request_id,
-            "event": "message_response",
-            "data": {
-                "type": "assistant_message",
-                "conversation_id": convo_ids[0],
-                "session_id": session_id,
-                "user_message_id": user_message_id,
-                "ai_message_id": ai_message_id,
-                "userLoginId": request.user_login_id,
-                "agent_used": "data_analyst_gpt4o" if use_data_analyst else "multiagent",
-                "timestamp": datetime.now().timestamp(),
-                "title": session_data.get("title", "New Chat"),
-                "total_messages": len(get_message_history(request.user_login_id, session_id)),
-                "message": {
-                    "id": ai_message_id,
-                    "author": {"role": "assistant", "name": "DataStory AI"},
-                    "metadata": {
-                        "model_used": "gpt-4o",
-                        "processing_time": processing_time,
-                        "thinking_length": len(thinking_buffer),
-                        "answer_length": len(answer_buffer),
-                        "charts_generated": len(artifacts)
-                    },
-                    "content": {
-                        "content_type": "text",
-                        "parts": [full_response_for_storage],
-                        "text": full_response_for_storage
-                    },
-                    "status": "finished_successfully",
-                    "timestamp_": datetime.now().timestamp()
-                },
-                "artifacts": artifacts
-            }
-        }
-        
-        yield f"data: {json.dumps(final_payload)}\n\n"
-        
-        # Save to Redis
-        if full_response_for_storage:
-            add_message(request.user_login_id, session_id, "assistant", full_response_for_storage)
-            update_session_title_if_needed(request.user_login_id, session_id, is_new_session)
-        
-        # Store to database
-        suffix = "dd" if use_data_analyst else "ss"
-        db_result = await store_conversation_to_db(
-            user_login_id=request.user_login_id,
-            session_id=session_id,
-            user_query=request.query,
-            ai_response=full_response_for_storage,
-            metadata={
-                "model_used": "gpt-4o",
-                "processing_time": processing_time,
-                "search_mode": "sql_analysis" if use_data_analyst else "hybrid"
-            },
-            artifacts=artifacts,
-            suffix=suffix,
-            convo_ids=convo_ids
-        )
-        
-        # Completion signal
-        completion_response = {
-            "event": "stream_complete",
-            "data": {
-                "status": "finished_successfully",
-                "timestamp": datetime.now().timestamp(),
-                "db_stored": db_result is not None,
-                "artifacts_count": len(artifacts),
-                "phases_completed": ["thinking", "answer", "charts"] if artifacts else ["thinking", "answer"]
-            }
-        }
-        yield f"data: {json.dumps(completion_response)}\n\n"
-        yield "data: [DONE]\n\n"
 
-    except Exception as e:
-        error_data = {
-            "status": "error",
-            "api_version": "v1",
-            "request_id": request_id,
-            "event": "error_response",
-            "data": {
-                "error": {
-                    "code": "STREAM_ERROR",
-                    "message": str(e),
-                    "details": {"type": "stream_error"}
-                },
-                "timestamp": datetime.now().timestamp()
-            }
-        }
-        yield f"data: {json.dumps(error_data)}\n\n"
-        yield "data: [DONE]\n\n"
+def create_phase_chunk(content: str, phase: str, idx: int = 0) -> dict:
+    """Standardized chunk definition for SSE"""
+    return {
+        "type": "chat:completion",
+        "data": {
+            "delta_content": content,
+            "phase": phase,
+            "index": idx,
+            "timestamp": datetime.utcnow().isoformat() + "Z",
+        },
+    }
 
-# =====================================
-# ENDPOINTS
-# =====================================
 
 @main_chat_router_v6.get("/")
 async def root():
     return {
-        "message": "AI Data Analysis Chatbot API with Agent Reasoning",
-        "version": "2.2.0",
-        "model": "gpt-4o",
-        "features": [
-            "True Agent Reasoning in Thinking Phase",
-            "Phased Streaming (thinking → answer → charts)",
-            "Session Management",
-            "Vega-Lite Artifacts",
-            "DB Storage"
-        ],
-        "phases": {
-            "thinking": "Agent's step-by-step reasoning process",
-            "answer": "Final analysis with specific numbers",
-            "charts": "Vega-Lite visualizations with actual data"
-        }
+        "message": "AI Data Analysis Chatbot API (v7)",
+        "version": "3.0.0",
+        "notes": "Production-grade phased streaming (thinking → reasoning → answer → charts → ...)",
     }
 
+
 @main_chat_router_v6.post("/chat/stream", response_class=StreamingResponse)
-async def chat_stream(request: ChatRequest):
-    """Streaming endpoint with agent reasoning phases"""
+async def chat_stream(request: Request, payload: ChatRequest):
+    """
+    Streaming endpoint.
+    The SSE stream yields JSON lines of the form: data: { ... }\n\n
+    Emitted phases: THINKING, REASONING, ANSWER, CHART, ERROR, DONE
+    """
+    # Create or resume session
     try:
         is_new_session = False
-        
-        if request.session_id is None:
-            session_data = create_session(request.user_login_id, request.org_id, request.metadata)
+        if payload.session_id is None:
+            session_data = create_session(payload.user_login_id, payload.org_id, payload.metadata)
             session_id = session_data["session_id"]
             is_new_session = True
             conversation_history = []
         else:
+            session_id = payload.session_id
             try:
-                session_data = get_session(request.user_login_id, request.session_id)
-                session_id = request.session_id
-                conversation_history = get_message_history(request.user_login_id, session_id, limit=5)
+                session_data = get_session(payload.user_login_id, session_id)
+                conversation_history = get_message_history(payload.user_login_id, session_id, limit=5)
             except HTTPException as e:
                 if e.status_code == 404:
-                    session_data = create_session(request.user_login_id, request.org_id, request.metadata)
+                    session_data = create_session(payload.user_login_id, payload.org_id, payload.metadata)
                     session_id = session_data["session_id"]
                     is_new_session = True
                     conversation_history = []
                 else:
                     raise
 
-        add_message(request.user_login_id, session_id, "user", request.query)
-        use_data_analyst = determine_agent_type(request)
+        # Persist the user's message in session storage immediately
+        add_message(payload.user_login_id, session_id, "user", payload.query)
+
+        # Decide which streaming generator to call
+        use_data_analyst = False
+        if payload.metadata and payload.metadata.get("file_paths"):
+            use_data_analyst = True
+
         convo_ids = await make_convo_id(suffix="dd" if use_data_analyst else "ss")
 
-        return StreamingResponse(
-            stream_chat_response_phased(
-                request=request,
-                session_id=session_id,
-                is_new_session=is_new_session,
-                conversation_history=conversation_history,
-                use_data_analyst=use_data_analyst,
-                convo_ids=convo_ids
-            ),
-            media_type="text/event-stream",
-            headers={
-                "Cache-Control": "no-cache",
-                "Connection": "keep-alive",
-                "X-Accel-Buffering": "no",
+        # The actual generator that will yield SSE chunks
+        async def event_generator() -> AsyncGenerator[str, None]:
+            request_id = f"req_{uuid.uuid4().hex[:26].upper()}"
+            start_time = datetime.utcnow()
+            idx = 0
+
+            # Initial metadata event
+            initial_metadata = {
+                "status": "ok",
+                "api_version": "v1",
+                "request_id": request_id,
+                "event": "stream_started",
+                "data": {
+                    "session_id": session_id,
+                    "user_message_id": str(uuid.uuid4()),
+                    "ai_message_id": str(uuid.uuid4()),
+                    "is_new_session": is_new_session,
+                    "agent_type": "data_analyst" if use_data_analyst else "multiagent",
+                    "timestamp": datetime.utcnow().timestamp(),
+                    "userLoginId": payload.user_login_id,
+                },
             }
-        )
+            yield f"data: {json.dumps(initial_metadata)}\n\n"
+
+            # Choose the appropriate stream
+            try:
+                if use_data_analyst:
+                    # stream_analysis yields (phase_tag, content) tuples
+                    file_paths = payload.metadata.get("file_paths", [])
+                    async for phase_tag, content in stream_analysis(payload.query, file_paths):
+                        # If client disconnected, generator will be cancelled; check here
+                        if await request.is_disconnected():
+                            logger.info("Client disconnected, cancelling stream.")
+                            break
+
+                        idx += 1
+                        if phase_tag in ("THINKING", "REASONING", "ANSWER"):
+                            chunk = create_phase_chunk(content, phase_tag.lower(), idx)
+                            yield f"data: {json.dumps(chunk)}\n\n"
+                        elif phase_tag == "CHART":
+                            chart_event = {
+                                "type": "chart:artifact",
+                                "data": {
+                                    "chart_index": idx,
+                                    "vega_spec": content,
+                                    "phase": "charts",
+                                    "timestamp": datetime.utcnow().isoformat() + "Z",
+                                },
+                            }
+                            yield f"data: {json.dumps(chart_event)}\n\n"
+                        elif phase_tag == "ERROR":
+                            err = {"status": "error", "event": "error_response", "data": {"error": {"message": content}, "request_id": request_id}}
+                            yield f"data: {json.dumps(err)}\n\n"
+                        elif phase_tag == "DONE":
+                            break
+
+                else:
+                    # Multiagent streaming: expected to yield raw chunks or control tokens
+                    async for chunk in stream_multiagent(payload.query):
+                        if await request.is_disconnected():
+                            logger.info("Client disconnected, cancelling multiagent stream.")
+                            break
+                        idx += 1
+                        if chunk == "[DONE]":
+                            break
+                        elif chunk.startswith("[ERROR]"):
+                            err = {"status": "error", "event": "error_response", "data": {"error": {"message": chunk[7:]}, "request_id": request_id}}
+                            yield f"data: {json.dumps(err)}\n\n"
+                        else:
+                            chunk_obj = create_phase_chunk(chunk, "answer", idx)
+                            yield f"data: {json.dumps(chunk_obj)}\n\n"
+
+                # Final payload: store and send completion notification
+                processing_time = (datetime.utcnow() - start_time).total_seconds()
+                # Compose final payload minimally; you can enrich and store as needed
+                completion_response = {
+                    "event": "stream_complete",
+                    "data": {
+                        "status": "finished_successfully",
+                        "timestamp": datetime.utcnow().timestamp(),
+                        "processing_time": processing_time,
+                        "convo_id": convo_ids[0],
+                    },
+                }
+                yield f"data: {json.dumps(completion_response)}\n\n"
+                yield "data: [DONE]\n\n"
+
+            except asyncio.CancelledError:
+                logger.info("Stream cancelled by client or server; performing cleanup.")
+                # Do any cleanup: close DB connections, rollback, etc.
+                raise
+            except Exception as exc:
+                logger.exception("Unhandled exception in event_generator")
+                error_payload = {"status": "error", "event": "error_response", "data": {"error": {"message": str(exc)}, "request_id": request_id}}
+                yield f"data: {json.dumps(error_payload)}\n\n"
+                yield "data: [DONE]\n\n"
 
+        # Return StreamingResponse
+        return StreamingResponse(event_generator(), media_type="text/event-stream", headers={"Cache-Control": "no-cache", "Connection": "keep-alive", "X-Accel-Buffering": "no"})
     except Exception as e:
-        raise HTTPException(status_code=500, detail=f"Error in chat stream: {str(e)}")
+        logger.exception("chat_stream top-level error")
+        raise HTTPException(status_code=500, detail=f"Error in chat stream: {str(e)}")

main_chat_v6/rough.ipynb

@@ -0,0 +1,974 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "cf14a6eb",
+   "metadata": {},
+   "source": [
+    "main_chart_router"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9abfde57",
+   "metadata": {
+    "vscode": {
+     "languageId": "plaintext"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "\"\"\"\n",
+    "Main Router with True Agent Reasoning in Thinking Phase\n",
+    "Phases: thinking (agent's reasoning), answer (analysis), charts (visualizations)\n",
+    "\"\"\"\n",
+    "\n",
+    "from fastapi import APIRouter, HTTPException\n",
+    "from fastapi.responses import StreamingResponse\n",
+    "from datetime import datetime\n",
+    "from typing import AsyncGenerator, List, Optional, Dict, Tuple\n",
+    "import json\n",
+    "import asyncio\n",
+    "import os\n",
+    "import sys\n",
+    "import uuid\n",
+    "from pathlib import Path\n",
+    "import httpx\n",
+    "import re\n",
+    "\n",
+    "PROJECT_ROOT = Path(__file__).resolve().parents[2]\n",
+    "if str(PROJECT_ROOT) not in sys.path:\n",
+    "    sys.path.insert(0, str(PROJECT_ROOT))\n",
+    "\n",
+    "from Redis.sessions_new import *\n",
+    "from main_chat_v6.data_agent import stream_analysis\n",
+    "from main_chat_v6.multiagent import stream_multiagent\n",
+    "from pydantic import BaseModel\n",
+    "from dotenv import load_dotenv\n",
+    "from retrieve_secret import *\n",
+    "\n",
+    "load_dotenv()\n",
+    "\n",
+    "# =====================================\n",
+    "# MODELS\n",
+    "# =====================================\n",
+    "\n",
+    "class Author(BaseModel):\n",
+    "    role: str\n",
+    "\n",
+    "class ChatRequest(BaseModel):\n",
+    "    user_login_id: str\n",
+    "    session_id: Optional[str] = None\n",
+    "    query: str\n",
+    "    org_id: Optional[str] = None\n",
+    "    metadata: Optional[Dict] = None\n",
+    "    list_of_files_path: Optional[List[str]] = None\n",
+    "    author: Optional[Author] = None\n",
+    "    create_time: Optional[str] = None\n",
+    "    token: Optional[str] = None\n",
+    "\n",
+    "class ChatResponse(BaseModel):\n",
+    "    session_id: str\n",
+    "    user_message: str\n",
+    "    assistant_response: Dict\n",
+    "    is_new_session: bool\n",
+    "    session_title: str\n",
+    "    timestamp: str\n",
+    "    files_processed: Optional[List[str]] = None\n",
+    "    artifacts_paths: Optional[List[str]] = None\n",
+    "\n",
+    "main_chat_router_v6 = APIRouter(prefix=\"/main_chatbot/v6\", tags=[\"main_chatbot_v6\"])\n",
+    "\n",
+    "# =====================================\n",
+    "# HELPER FUNCTIONS\n",
+    "# =====================================\n",
+    "\n",
+    "async def make_convo_id(suffix: str = \"ss\") -> List[str]:\n",
+    "    \"\"\"Generate UUID with suffix in mutable list\"\"\"\n",
+    "    base_uuid = str(uuid.uuid4())\n",
+    "    convo_id = base_uuid[:-2] + suffix\n",
+    "    return [convo_id]\n",
+    "\n",
+    "def determine_agent_type(request: ChatRequest) -> bool:\n",
+    "    \"\"\"Determine which agent to use. Returns True for data_analyst.\"\"\"\n",
+    "    if request.metadata and \"file_paths\" in request.metadata:\n",
+    "        file_paths = request.metadata[\"file_paths\"]\n",
+    "        if file_paths and len(file_paths) > 0:\n",
+    "            return True\n",
+    "    \n",
+    "    query_lower = request.query.lower()\n",
+    "    data_keywords = [\n",
+    "        \"analyze\", \"analysis\", \"dataset\", \"csv\", \"data\", \"statistics\",\n",
+    "        \"visualize\", \"visualization\", \"chart\", \"graph\", \"plot\",\n",
+    "        \"sql\", \"query\", \"select\", \"table\", \"column\", \"row\"\n",
+    "    ]\n",
+    "    \n",
+    "    return any(keyword in query_lower for keyword in data_keywords)\n",
+    "\n",
+    "def create_phase_chunk(content: str, phase: str) -> dict:\n",
+    "    \"\"\"Create standardized phase chunk\"\"\"\n",
+    "    return {\n",
+    "        \"type\": \"chat:completion\",\n",
+    "        \"data\": {\n",
+    "            \"delta_content\": content,\n",
+    "            \"phase\": phase\n",
+    "        }\n",
+    "    }\n",
+    "\n",
+    "async def store_conversation_to_db(\n",
+    "    user_login_id: str,\n",
+    "    session_id: str,\n",
+    "    user_query: str,\n",
+    "    ai_response: str,\n",
+    "    metadata: Dict[str, any],\n",
+    "    artifacts: list = None,\n",
+    "    suffix: str = \"ss\",\n",
+    "    convo_ids: List[str] = None,\n",
+    "    db_api_url: str = \"https://mr-mvp-api-dev.dev.ingenspark.com/Db_store_router/conversations\"\n",
+    ") -> Optional[Dict]:\n",
+    "    \"\"\"Store conversation to database API\"\"\"\n",
+    "    try:\n",
+    "        if convo_ids is None or not convo_ids:\n",
+    "            convo_ids = await make_convo_id(suffix)\n",
+    "        \n",
+    "        convo_id = convo_ids[0]\n",
+    "        query_id = f\"q_{uuid.uuid4()}\"\n",
+    "        \n",
+    "        payload = {\n",
+    "            \"user_id\": user_login_id,\n",
+    "            \"convo_id\": convo_id,\n",
+    "            \"data\": {\n",
+    "                \"user_query\": {\n",
+    "                    \"query_id\": query_id,\n",
+    "                    \"text\": user_query,\n",
+    "                    \"user_metadata\": {\n",
+    "                        \"location\": metadata.get(\"user\", {}).get(\"location\", \"Unknown\"),\n",
+    "                        \"language\": metadata.get(\"user\", {}).get(\"language\", \"English\")\n",
+    "                    }\n",
+    "                },\n",
+    "                \"response\": {\n",
+    "                    \"text\": ai_response,\n",
+    "                    \"status\": \"success\",\n",
+    "                    \"response_time\": datetime.now().isoformat() + \"Z\",\n",
+    "                    \"duration\": f\"{metadata.get('processing_time', 0):.2f}s\",\n",
+    "                    \"artifacts\": artifacts if artifacts else []\n",
+    "                },\n",
+    "                \"metadata\": {\n",
+    "                    \"processing_node\": {\n",
+    "                        \"model_version\": metadata.get(\"model_used\", \"gpt-4o\"),\n",
+    "                        \"api_latency_ms\": int(metadata.get(\"processing_time\", 0) * 1000),\n",
+    "                        \"search_mode\": metadata.get(\"search_mode\", \"sql_analysis\")\n",
+    "                    }\n",
+    "                }\n",
+    "            },\n",
+    "            \"is_saved\": False\n",
+    "        }\n",
+    "        \n",
+    "        async with httpx.AsyncClient(timeout=30.0) as client:\n",
+    "            response = await client.post(db_api_url, json=payload, headers={\n",
+    "                \"accept\": \"application/json\",\n",
+    "                \"Content-Type\": \"application/json\"\n",
+    "            })\n",
+    "            response.raise_for_status()\n",
+    "            return response.json()\n",
+    "            \n",
+    "    except Exception as e:\n",
+    "        print(f\"Error storing conversation to DB: {e}\")\n",
+    "        return None\n",
+    "\n",
+    "# =====================================\n",
+    "# PHASED STREAMING WITH AGENT REASONING\n",
+    "# =====================================\n",
+    "\n",
+    "async def stream_chat_response_phased(\n",
+    "    request: ChatRequest,\n",
+    "    session_id: str,\n",
+    "    is_new_session: bool,\n",
+    "    conversation_history: List,\n",
+    "    use_data_analyst: bool,\n",
+    "    convo_ids: List[str]\n",
+    ") -> AsyncGenerator[str, None]:\n",
+    "    \"\"\"\n",
+    "    Stream response with phases from agent:\n",
+    "    - thinking: Agent's actual reasoning process\n",
+    "    - answer: Analysis and narrative\n",
+    "    - charts: Visualizations\n",
+    "    \"\"\"\n",
+    "    full_response = \"\"\n",
+    "    artifacts = []\n",
+    "    user_message_id = str(uuid.uuid4())\n",
+    "    ai_message_id = str(uuid.uuid4())\n",
+    "    request_id = f\"req_{uuid.uuid4().hex[:26].upper()}\"\n",
+    "    start_time = datetime.now()\n",
+    "    \n",
+    "    thinking_buffer = \"\"\n",
+    "    answer_buffer = \"\"\n",
+    "    \n",
+    "    try:\n",
+    "        # ===== INITIAL METADATA =====\n",
+    "        initial_metadata = {\n",
+    "            \"status\": \"ok\",\n",
+    "            \"api_version\": \"v1\",\n",
+    "            \"request_id\": request_id,\n",
+    "            \"event\": \"stream_started\",\n",
+    "            \"data\": {\n",
+    "                \"session_id\": session_id,\n",
+    "                \"user_message_id\": user_message_id,\n",
+    "                \"ai_message_id\": ai_message_id,\n",
+    "                \"is_new_session\": is_new_session,\n",
+    "                \"agent_type\": \"data_analyst_gpt4o\" if use_data_analyst else \"multiagent\",\n",
+    "                \"timestamp\": datetime.now().timestamp(),\n",
+    "                \"userLoginId\": request.user_login_id\n",
+    "            }\n",
+    "        }\n",
+    "        yield f\"data: {json.dumps(initial_metadata)}\\n\\n\"\n",
+    "        await asyncio.sleep(0)\n",
+    "\n",
+    "        # ===== STREAM FROM AGENT =====\n",
+    "        if use_data_analyst:\n",
+    "            file_paths = request.metadata.get(\"file_paths\", []) if request.metadata else []\n",
+    "            \n",
+    "            if not file_paths:\n",
+    "                error_msg = \"Data analyst requires file_paths in metadata\"\n",
+    "                yield f\"data: {json.dumps({'error': error_msg})}\\n\\n\"\n",
+    "                yield \"data: [DONE]\\n\\n\"\n",
+    "                return\n",
+    "            \n",
+    "            chart_count = 0\n",
+    "            \n",
+    "            # Stream from data analyst agent\n",
+    "            async for phase, content in stream_analysis(request.query, file_paths):\n",
+    "                if phase == \"DONE\":\n",
+    "                    break\n",
+    "                \n",
+    "                elif phase == \"ERROR\":\n",
+    "                    error_response = {\n",
+    "                        \"status\": \"error\",\n",
+    "                        \"event\": \"error_response\",\n",
+    "                        \"data\": {\n",
+    "                            \"error\": {\n",
+    "                                \"code\": \"STREAM_ERROR\",\n",
+    "                                \"message\": content,\n",
+    "                                \"details\": {}\n",
+    "                            },\n",
+    "                            \"request_id\": request_id,\n",
+    "                            \"timestamp\": datetime.now().timestamp()\n",
+    "                        }\n",
+    "                    }\n",
+    "                    yield f\"data: {json.dumps(error_response)}\\n\\n\"\n",
+    "                \n",
+    "                elif phase == \"THINKING\":\n",
+    "                    # Agent's reasoning - stream as thinking phase\n",
+    "                    thinking_buffer += content\n",
+    "                    phase_chunk = create_phase_chunk(content, \"thinking\")\n",
+    "                    yield f\"data: {json.dumps(phase_chunk)}\\n\\n\"\n",
+    "                    await asyncio.sleep(0)\n",
+    "                \n",
+    "                elif phase == \"ANSWER\":\n",
+    "                    # Agent's analysis - stream as answer phase\n",
+    "                    answer_buffer += content\n",
+    "                    full_response += content\n",
+    "                    phase_chunk = create_phase_chunk(content, \"answer\")\n",
+    "                    yield f\"data: {json.dumps(phase_chunk)}\\n\\n\"\n",
+    "                    await asyncio.sleep(0)\n",
+    "                \n",
+    "                elif phase == \"CHART\":\n",
+    "                    # Visualization artifact\n",
+    "                    chart_count += 1\n",
+    "                    artifacts.append(content)\n",
+    "                    \n",
+    "                    chart_event = {\n",
+    "                        \"type\": \"chart:artifact\",\n",
+    "                        \"data\": {\n",
+    "                            \"chart_index\": chart_count,\n",
+    "                            \"vega_spec\": content,\n",
+    "                            \"phase\": \"charts\"\n",
+    "                        }\n",
+    "                    }\n",
+    "                    yield f\"data: {json.dumps(chart_event)}\\n\\n\"\n",
+    "                    await asyncio.sleep(0.1)\n",
+    "        \n",
+    "        else:\n",
+    "            # Multiagent flow (if needed)\n",
+    "            async for chunk in stream_multiagent(request.query):\n",
+    "                if chunk == \"[DONE]\":\n",
+    "                    break\n",
+    "                elif chunk.startswith(\"[ERROR]\"):\n",
+    "                    error_response = {\n",
+    "                        \"status\": \"error\",\n",
+    "                        \"event\": \"error_response\",\n",
+    "                        \"data\": {\n",
+    "                            \"error\": {\n",
+    "                                \"code\": \"STREAM_ERROR\",\n",
+    "                                \"message\": chunk[7:],\n",
+    "                                \"details\": {}\n",
+    "                            },\n",
+    "                            \"request_id\": request_id,\n",
+    "                            \"timestamp\": datetime.now().timestamp()\n",
+    "                        }\n",
+    "                    }\n",
+    "                    yield f\"data: {json.dumps(error_response)}\\n\\n\"\n",
+    "                else:\n",
+    "                    full_response += chunk\n",
+    "                    phase_chunk = create_phase_chunk(chunk, \"answer\")\n",
+    "                    yield f\"data: {json.dumps(phase_chunk)}\\n\\n\"\n",
+    "                    await asyncio.sleep(0)\n",
+    "\n",
+    "        # ===== FINAL PAYLOAD =====\n",
+    "        processing_time = (datetime.now() - start_time).total_seconds()\n",
+    "        session_data = get_session(request.user_login_id, session_id)\n",
+    "        \n",
+    "        # Combine thinking + answer for full response storage\n",
+    "        full_response_for_storage = thinking_buffer + \"\\n\\n\" + answer_buffer\n",
+    "        \n",
+    "        final_payload = {\n",
+    "            \"status\": \"ok\",\n",
+    "            \"api_version\": \"v1\",\n",
+    "            \"request_id\": request_id,\n",
+    "            \"event\": \"message_response\",\n",
+    "            \"data\": {\n",
+    "                \"type\": \"assistant_message\",\n",
+    "                \"conversation_id\": convo_ids[0],\n",
+    "                \"session_id\": session_id,\n",
+    "                \"user_message_id\": user_message_id,\n",
+    "                \"ai_message_id\": ai_message_id,\n",
+    "                \"userLoginId\": request.user_login_id,\n",
+    "                \"agent_used\": \"data_analyst_gpt4o\" if use_data_analyst else \"multiagent\",\n",
+    "                \"timestamp\": datetime.now().timestamp(),\n",
+    "                \"title\": session_data.get(\"title\", \"New Chat\"),\n",
+    "                \"total_messages\": len(get_message_history(request.user_login_id, session_id)),\n",
+    "                \"message\": {\n",
+    "                    \"id\": ai_message_id,\n",
+    "                    \"author\": {\"role\": \"assistant\", \"name\": \"DataStory AI\"},\n",
+    "                    \"metadata\": {\n",
+    "                        \"model_used\": \"gpt-4o\",\n",
+    "                        \"processing_time\": processing_time,\n",
+    "                        \"thinking_length\": len(thinking_buffer),\n",
+    "                        \"answer_length\": len(answer_buffer),\n",
+    "                        \"charts_generated\": len(artifacts)\n",
+    "                    },\n",
+    "                    \"content\": {\n",
+    "                        \"content_type\": \"text\",\n",
+    "                        \"parts\": [full_response_for_storage],\n",
+    "                        \"text\": full_response_for_storage\n",
+    "                    },\n",
+    "                    \"status\": \"finished_successfully\",\n",
+    "                    \"timestamp_\": datetime.now().timestamp()\n",
+    "                },\n",
+    "                \"artifacts\": artifacts\n",
+    "            }\n",
+    "        }\n",
+    "        \n",
+    "        yield f\"data: {json.dumps(final_payload)}\\n\\n\"\n",
+    "        \n",
+    "        # Save to Redis\n",
+    "        if full_response_for_storage:\n",
+    "            add_message(request.user_login_id, session_id, \"assistant\", full_response_for_storage)\n",
+    "            update_session_title_if_needed(request.user_login_id, session_id, is_new_session)\n",
+    "        \n",
+    "        # Store to database\n",
+    "        suffix = \"dd\" if use_data_analyst else \"ss\"\n",
+    "        db_result = await store_conversation_to_db(\n",
+    "            user_login_id=request.user_login_id,\n",
+    "            session_id=session_id,\n",
+    "            user_query=request.query,\n",
+    "            ai_response=full_response_for_storage,\n",
+    "            metadata={\n",
+    "                \"model_used\": \"gpt-4o\",\n",
+    "                \"processing_time\": processing_time,\n",
+    "                \"search_mode\": \"sql_analysis\" if use_data_analyst else \"hybrid\"\n",
+    "            },\n",
+    "            artifacts=artifacts,\n",
+    "            suffix=suffix,\n",
+    "            convo_ids=convo_ids\n",
+    "        )\n",
+    "        \n",
+    "        # Completion signal\n",
+    "        completion_response = {\n",
+    "            \"event\": \"stream_complete\",\n",
+    "            \"data\": {\n",
+    "                \"status\": \"finished_successfully\",\n",
+    "                \"timestamp\": datetime.now().timestamp(),\n",
+    "                \"db_stored\": db_result is not None,\n",
+    "                \"artifacts_count\": len(artifacts),\n",
+    "                \"phases_completed\": [\"thinking\", \"answer\", \"charts\"] if artifacts else [\"thinking\", \"answer\"]\n",
+    "            }\n",
+    "        }\n",
+    "        yield f\"data: {json.dumps(completion_response)}\\n\\n\"\n",
+    "        yield \"data: [DONE]\\n\\n\"\n",
+    "\n",
+    "    except Exception as e:\n",
+    "        error_data = {\n",
+    "            \"status\": \"error\",\n",
+    "            \"api_version\": \"v1\",\n",
+    "            \"request_id\": request_id,\n",
+    "            \"event\": \"error_response\",\n",
+    "            \"data\": {\n",
+    "                \"error\": {\n",
+    "                    \"code\": \"STREAM_ERROR\",\n",
+    "                    \"message\": str(e),\n",
+    "                    \"details\": {\"type\": \"stream_error\"}\n",
+    "                },\n",
+    "                \"timestamp\": datetime.now().timestamp()\n",
+    "            }\n",
+    "        }\n",
+    "        yield f\"data: {json.dumps(error_data)}\\n\\n\"\n",
+    "        yield \"data: [DONE]\\n\\n\"\n",
+    "\n",
+    "# =====================================\n",
+    "# ENDPOINTS\n",
+    "# =====================================\n",
+    "\n",
+    "@main_chat_router_v6.get(\"/\")\n",
+    "async def root():\n",
+    "    return {\n",
+    "        \"message\": \"AI Data Analysis Chatbot API with Agent Reasoning\",\n",
+    "        \"version\": \"2.2.0\",\n",
+    "        \"model\": \"gpt-4o\",\n",
+    "        \"features\": [\n",
+    "            \"True Agent Reasoning in Thinking Phase\",\n",
+    "            \"Phased Streaming (thinking → answer → charts)\",\n",
+    "            \"Session Management\",\n",
+    "            \"Vega-Lite Artifacts\",\n",
+    "            \"DB Storage\"\n",
+    "        ],\n",
+    "        \"phases\": {\n",
+    "            \"thinking\": \"Agent's step-by-step reasoning process\",\n",
+    "            \"answer\": \"Final analysis with specific numbers\",\n",
+    "            \"charts\": \"Vega-Lite visualizations with actual data\"\n",
+    "        }\n",
+    "    }\n",
+    "\n",
+    "@main_chat_router_v6.post(\"/chat/stream\", response_class=StreamingResponse)\n",
+    "async def chat_stream(request: ChatRequest):\n",
+    "    \"\"\"Streaming endpoint with agent reasoning phases\"\"\"\n",
+    "    try:\n",
+    "        is_new_session = False\n",
+    "        \n",
+    "        if request.session_id is None:\n",
+    "            session_data = create_session(request.user_login_id, request.org_id, request.metadata)\n",
+    "            session_id = session_data[\"session_id\"]\n",
+    "            is_new_session = True\n",
+    "            conversation_history = []\n",
+    "        else:\n",
+    "            try:\n",
+    "                session_data = get_session(request.user_login_id, request.session_id)\n",
+    "                session_id = request.session_id\n",
+    "                conversation_history = get_message_history(request.user_login_id, session_id, limit=5)\n",
+    "            except HTTPException as e:\n",
+    "                if e.status_code == 404:\n",
+    "                    session_data = create_session(request.user_login_id, request.org_id, request.metadata)\n",
+    "                    session_id = session_data[\"session_id\"]\n",
+    "                    is_new_session = True\n",
+    "                    conversation_history = []\n",
+    "                else:\n",
+    "                    raise\n",
+    "\n",
+    "        add_message(request.user_login_id, session_id, \"user\", request.query)\n",
+    "        use_data_analyst = determine_agent_type(request)\n",
+    "        convo_ids = await make_convo_id(suffix=\"dd\" if use_data_analyst else \"ss\")\n",
+    "\n",
+    "        return StreamingResponse(\n",
+    "            stream_chat_response_phased(\n",
+    "                request=request,\n",
+    "                session_id=session_id,\n",
+    "                is_new_session=is_new_session,\n",
+    "                conversation_history=conversation_history,\n",
+    "                use_data_analyst=use_data_analyst,\n",
+    "                convo_ids=convo_ids\n",
+    "            ),\n",
+    "            media_type=\"text/event-stream\",\n",
+    "            headers={\n",
+    "                \"Cache-Control\": \"no-cache\",\n",
+    "                \"Connection\": \"keep-alive\",\n",
+    "                \"X-Accel-Buffering\": \"no\",\n",
+    "            }\n",
+    "        )\n",
+    "\n",
+    "    except Exception as e:\n",
+    "        raise HTTPException(status_code=500, detail=f\"Error in chat stream: {str(e)}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e3406210",
+   "metadata": {},
+   "source": [
+    "data agent"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "809c9222",
+   "metadata": {
+    "vscode": {
+     "languageId": "plaintext"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "\"\"\"\n",
+    "Improved Data Agent with Real-time Phased Streaming\n",
+    "Phases: thinking (agent reasoning), answer (final response), charts (visualizations)\n",
+    "\"\"\"\n",
+    "\n",
+    "import asyncio\n",
+    "import json\n",
+    "import re\n",
+    "from typing import AsyncGenerator, List, Tuple\n",
+    "import duckdb\n",
+    "import pandas as pd\n",
+    "from openai.types.responses import ResponseTextDeltaEvent\n",
+    "from agents import Agent, Runner, function_tool\n",
+    "from s3.read_files import read_parquet_from_s3\n",
+    "\n",
+    "# =====================================\n",
+    "# CONFIGURATION\n",
+    "# =====================================\n",
+    "\n",
+    "ALLOWED_SQL_KEYWORDS = (\"select\", \"with\")\n",
+    "MAX_RESULT_ROWS = 10000\n",
+    "\n",
+    "_registered_datasets = {}\n",
+    "last_sql_query: str = \"\"\n",
+    "\n",
+    "# =====================================\n",
+    "# DATASET LOADING\n",
+    "# =====================================\n",
+    "\n",
+    "def generate_dataset_context(df: pd.DataFrame, table_name: str) -> str:\n",
+    "    \"\"\"Generate comprehensive dataset context\"\"\"\n",
+    "    context_parts = []\n",
+    "    context_parts.append(f\"Table: {table_name}\")\n",
+    "    context_parts.append(f\"Rows: {len(df)}, Columns: {len(df.columns)}\")\n",
+    "    context_parts.append(f\"Column Names: {', '.join(df.columns.tolist())}\")\n",
+    "    context_parts.append(\"\\nData Types:\")\n",
+    "    for col, dtype in df.dtypes.items():\n",
+    "        context_parts.append(f\"  - {col}: {dtype}\")\n",
+    "    \n",
+    "    context_parts.append(\"\\nSample Data (first 3 rows):\")\n",
+    "    sample_str = df.head(3).to_string(index=False, max_cols=10)\n",
+    "    context_parts.append(sample_str)\n",
+    "    \n",
+    "    numeric_cols = df.select_dtypes(include=['number']).columns.tolist()\n",
+    "    if numeric_cols:\n",
+    "        context_parts.append(\"\\nNumeric Column Statistics:\")\n",
+    "        desc = df[numeric_cols].describe().transpose()\n",
+    "        for col in numeric_cols[:5]:\n",
+    "            if col in desc.index:\n",
+    "                context_parts.append(\n",
+    "                    f\"  - {col}: mean={desc.loc[col, 'mean']:.2f}, \"\n",
+    "                    f\"min={desc.loc[col, 'min']:.2f}, max={desc.loc[col, 'max']:.2f}\"\n",
+    "                )\n",
+    "    \n",
+    "    return \"\\n\".join(context_parts)\n",
+    "\n",
+    "def load_and_register_datasets(file_paths: List[str]) -> str:\n",
+    "    \"\"\"Load datasets and register them for SQL queries\"\"\"\n",
+    "    global _registered_datasets\n",
+    "    _registered_datasets.clear()\n",
+    "    context_parts = [\"Available datasets for SQL queries:\\n\"]\n",
+    "    \n",
+    "    for idx, file_path in enumerate(file_paths, start=1):\n",
+    "        table_name = f\"CSV{idx}\"\n",
+    "        try:\n",
+    "            print(f\"Loading {file_path} as {table_name}\")\n",
+    "            df = read_parquet_from_s3(file_path)\n",
+    "            _registered_datasets[table_name] = df\n",
+    "            dataset_context = generate_dataset_context(df, table_name)\n",
+    "            context_parts.append(f\"\\n{dataset_context}\\n\")\n",
+    "            context_parts.append(\"=\"*70)\n",
+    "            print(f\"Loaded {table_name}: {len(df)} rows, {len(df.columns)} columns\")\n",
+    "        except Exception as e:\n",
+    "            print(f\"Error loading {file_path}: {str(e)}\")\n",
+    "            raise\n",
+    "    \n",
+    "    return \"\\n\".join(context_parts)\n",
+    "\n",
+    "# =====================================\n",
+    "# SQL TOOL\n",
+    "# =====================================\n",
+    "\n",
+    "@function_tool\n",
+    "async def sql_tool(sql_query: str, user_query: str = \"\") -> dict:\n",
+    "    \"\"\"Execute SQL query and return results\"\"\"\n",
+    "    global _registered_datasets, last_sql_query\n",
+    "    last_sql_query = sql_query\n",
+    "\n",
+    "    print(f\"\\n{'='*70}\")\n",
+    "    print(f\"Executing SQL:\\n{sql_query}\")\n",
+    "    print(f\"{'='*70}\\n\")\n",
+    "\n",
+    "    normalized_query = sql_query.strip().lower()\n",
+    "    if not normalized_query.startswith(ALLOWED_SQL_KEYWORDS):\n",
+    "        raise ValueError(\n",
+    "            f\"Only {', '.join(k.upper() for k in ALLOWED_SQL_KEYWORDS)} queries allowed.\"\n",
+    "        )\n",
+    "\n",
+    "    if not _registered_datasets:\n",
+    "        raise ValueError(\"No datasets loaded.\")\n",
+    "\n",
+    "    conn = duckdb.connect(\":memory:\")\n",
+    "    try:\n",
+    "        for name, df in _registered_datasets.items():\n",
+    "            conn.register(name, df)\n",
+    "        result_df = conn.execute(sql_query).df()\n",
+    "    except Exception as e:\n",
+    "        conn.close()\n",
+    "        raise ValueError(f\"SQL execution failed: {str(e)}\")\n",
+    "    finally:\n",
+    "        conn.close()\n",
+    "\n",
+    "    if len(result_df) > MAX_RESULT_ROWS:\n",
+    "        result_df = result_df.head(MAX_RESULT_ROWS)\n",
+    "\n",
+    "    summary_lines = [\n",
+    "        f\"Returned {len(result_df)} rows, {len(result_df.columns)} columns\"\n",
+    "    ]\n",
+    "    if not result_df.empty:\n",
+    "        summary_lines.append(\"\\nPreview (first 10 rows):\")\n",
+    "        summary_lines.append(result_df.head(10).to_string(index=False, max_rows=10))\n",
+    "    \n",
+    "    result_summary = \"\\n\".join(summary_lines)\n",
+    "\n",
+    "    print(f\"SQL executed successfully\\n\")\n",
+    "    return {\n",
+    "        \"sql_query\": sql_query,\n",
+    "        \"result_summary\": result_summary,\n",
+    "        \"columns\": result_df.columns.tolist(),\n",
+    "        \"row_count\": len(result_df)\n",
+    "    }\n",
+    "\n",
+    "# =====================================\n",
+    "# DATA ANALYST AGENT WITH STRUCTURED OUTPUT\n",
+    "# =====================================\n",
+    "\n",
+    "def create_data_analyst_agent(dataset_context: str) -> Agent:\n",
+    "    \"\"\"Create data analyst agent with explicit phase instructions\"\"\"\n",
+    "    return Agent(\n",
+    "        name=\"DataStory_AI\",\n",
+    "        model=\"gpt-4o\",\n",
+    "        tools=[sql_tool],\n",
+    "        instructions=f\"\"\"You are **DataStory AI**, an elite data analyst who combines deep analytical reasoning with compelling storytelling.\n",
+    "\n",
+    "DATASET CONTEXT:\n",
+    "{dataset_context}\n",
+    "\n",
+    "===== CRITICAL: STRUCTURED RESPONSE FORMAT =====\n",
+    "\n",
+    "You MUST structure your response in THREE distinct phases using XML tags:\n",
+    "\n",
+    "**PHASE 1: THINKING (Your Reasoning Process)**\n",
+    "<thinking>\n",
+    "- Explain your analytical approach step-by-step\n",
+    "- Describe what SQL queries you'll run and WHY\n",
+    "- Outline the insights you're looking for\n",
+    "- Discuss any data quality considerations\n",
+    "- Map out your visualization strategy\n",
+    "Example:\n",
+    "\"To answer this question, I need to:\n",
+    "1. First examine the distribution of sales across regions by querying CSV1\n",
+    "2. Calculate year-over-year growth rates to identify trends\n",
+    "3. Identify top performers and outliers\n",
+    "4. Create visualizations showing: (a) regional comparison, (b) time series trend, (c) top 10 performers\"\n",
+    "</thinking>\n",
+    "\n",
+    "**PHASE 2: ANSWER (Your Analysis & Narrative)**\n",
+    "<answer>\n",
+    "Write your complete data story here:\n",
+    "- Present key findings with specific numbers\n",
+    "- Provide context and business implications  \n",
+    "- Explain patterns, trends, and anomalies\n",
+    "- Use clear section headers\n",
+    "- Be specific: \"Sales increased by 23.5% in Q3\" not \"Sales increased significantly\"\n",
+    "</answer>\n",
+    "\n",
+    "**PHASE 3: CHARTS (Visualizations)**\n",
+    "For each visualization, use this EXACT format:\n",
+    "\n",
+    "<datastory_chart>\n",
+    "<chart_title>Descriptive Title</chart_title>\n",
+    "<chart_description>What this chart shows with specific numbers</chart_description>\n",
+    "<vega_spec>\n",
+    "{{\n",
+    "  \"$schema\": \"https://vega.github.io/schema/vega-lite/v5.json\",\n",
+    "  \"title\": {{\"text\": \"Chart Title\", \"fontSize\": 18, \"fontWeight\": \"bold\"}},\n",
+    "  \"width\": 700,\n",
+    "  \"height\": 400,\n",
+    "  \"data\": {{\"values\": [\n",
+    "    {{\"category\": \"A\", \"value\": 100}},\n",
+    "    {{\"category\": \"B\", \"value\": 150}}\n",
+    "  ]}},\n",
+    "  \"mark\": {{\"type\": \"bar\", \"tooltip\": true, \"cornerRadius\": 4}},\n",
+    "  \"encoding\": {{\n",
+    "    \"x\": {{\"field\": \"category\", \"type\": \"nominal\", \"axis\": {{\"labelAngle\": 0}}}},\n",
+    "    \"y\": {{\"field\": \"value\", \"type\": \"quantitative\", \"axis\": {{\"title\": \"Value\"}}}}\n",
+    "  }},\n",
+    "  \"config\": {{\n",
+    "    \"axis\": {{\"labelFontSize\": 12, \"titleFontSize\": 14}},\n",
+    "    \"legend\": {{\"labelFontSize\": 12, \"titleFontSize\": 14}}\n",
+    "  }}\n",
+    "}}\n",
+    "</vega_spec>\n",
+    "</datastory_chart>\n",
+    "\n",
+    "===== WORKFLOW =====\n",
+    "1. **START WITH THINKING**: Always begin with <thinking> tags explaining your analytical plan\n",
+    "2. **Execute SQL queries**: Use sql_tool(sql_query=\"...\", user_query=\"...\")\n",
+    "3. **WRITE ANSWER**: Present insights in <answer> tags with specific numbers\n",
+    "4. **CREATE CHARTS**: Generate 2-3 visualizations wrapped in <datastory_chart> tags\n",
+    "\n",
+    "===== SQL QUERY RULES =====\n",
+    "- Table names: CSV1, CSV2, CSV3, etc. (from dataset context above)\n",
+    "- Only SELECT and WITH statements allowed\n",
+    "- Always include user_query parameter in sql_tool calls\n",
+    "- Use meaningful column aliases\n",
+    "- Limit results appropriately for visualizations\n",
+    "\n",
+    "===== VISUALIZATION BEST PRACTICES =====\n",
+    "- Include ACTUAL data in \"values\" array (not placeholders)\n",
+    "- Use appropriate chart types: bar, line, point, area, arc (pie)\n",
+    "- Add tooltips: \"mark\": {{\"type\": \"bar\", \"tooltip\": true}}\n",
+    "- Use clear axis labels and titles\n",
+    "- Color schemes: use \"color\" encoding for categorical data\n",
+    "- For time series: parse dates with \"timeUnit\"\n",
+    "\n",
+    "===== QUALITY STANDARDS =====\n",
+    "✓ Be specific with numbers: \"Revenue increased by $2.3M (18.7%)\" \n",
+    "✓ Provide context: Compare to benchmarks, previous periods, or goals\n",
+    "✓ Explain WHY patterns exist, not just WHAT they are\n",
+    "✓ Structure with clear headers: ## Key Finding 1, ## Key Finding 2\n",
+    "✓ Generate 2-3 complementary visualizations\n",
+    "✓ End with actionable recommendations\n",
+    "\n",
+    "===== EXAMPLE RESPONSE STRUCTURE =====\n",
+    "\n",
+    "<thinking>\n",
+    "Let me analyze the sales data systematically:\n",
+    "\n",
+    "1. **Data Exploration**: I'll query CSV1 to understand sales distribution across regions and time periods\n",
+    "2. **Trend Analysis**: Calculate monthly growth rates to identify seasonal patterns  \n",
+    "3. **Performance Ranking**: Identify top 10 products by revenue\n",
+    "4. **Visualization Plan**:\n",
+    "   - Chart 1: Regional sales comparison (bar chart)\n",
+    "   - Chart 2: Monthly trend with growth rate (line chart)\n",
+    "   - Chart 3: Top 10 products (horizontal bar)\n",
+    "\n",
+    "SQL Strategy: First, aggregate sales by region, then by month, finally rank products.\n",
+    "</thinking>\n",
+    "\n",
+    "<answer>\n",
+    "## Executive Summary\n",
+    "Analysis of 50,000 transactions reveals strong regional disparities and seasonal trends...\n",
+    "\n",
+    "## Regional Performance\n",
+    "The West region dominates with $4.2M in sales (42% of total), followed by East at $3.1M (31%)...\n",
+    "\n",
+    "[Continue with detailed findings, specific numbers, and business implications]\n",
+    "</answer>\n",
+    "\n",
+    "<datastory_chart>\n",
+    "<chart_title>Sales by Region - West Region Leads with 42% Share</chart_title>\n",
+    "<chart_description>West region generated $4.2M, 35% ahead of second-place East region ($3.1M)</chart_description>\n",
+    "<vega_spec>\n",
+    "{{\n",
+    "  \"$schema\": \"https://vega.github.io/schema/vega-lite/v5.json\",\n",
+    "  \"title\": {{\"text\": \"Sales by Region ($ Millions)\", \"fontSize\": 18}},\n",
+    "  \"width\": 700,\n",
+    "  \"height\": 400,\n",
+    "  \"data\": {{\"values\": [\n",
+    "    {{\"region\": \"West\", \"sales\": 4.2}},\n",
+    "    {{\"region\": \"East\", \"sales\": 3.1}},\n",
+    "    {{\"region\": \"North\", \"sales\": 1.8}},\n",
+    "    {{\"region\": \"South\", \"sales\": 1.5}}\n",
+    "  ]}},\n",
+    "  \"mark\": {{\"type\": \"bar\", \"tooltip\": true, \"color\": \"#4C78A8\"}},\n",
+    "  \"encoding\": {{\n",
+    "    \"x\": {{\"field\": \"region\", \"type\": \"nominal\", \"sort\": \"-y\"}},\n",
+    "    \"y\": {{\"field\": \"sales\", \"type\": \"quantitative\", \"title\": \"Sales ($M)\"}}\n",
+    "  }}\n",
+    "}}\n",
+    "</vega_spec>\n",
+    "</datastory_chart>\n",
+    "\n",
+    "REMEMBER: Always use the three-phase structure with XML tags. Your reasoning goes in <thinking>, analysis in <answer>, and charts in <datastory_chart> blocks.\n",
+    "\"\"\"\n",
+    "    )\n",
+    "\n",
+    "# =====================================\n",
+    "# PHASE-AWARE CHART BUFFER\n",
+    "# =====================================\n",
+    "\n",
+    "class PhaseAwareBuffer:\n",
+    "    \"\"\"Buffer that separates content into thinking, answer, and chart phases\"\"\"\n",
+    "    \n",
+    "    def __init__(self):\n",
+    "        self.full_buffer = \"\"\n",
+    "        self.thinking_complete = False\n",
+    "        self.answer_complete = False\n",
+    "        self.extracted_charts = []\n",
+    "    \n",
+    "    def add_text(self, text: str) -> List[Tuple[str, str]]:\n",
+    "        \"\"\"\n",
+    "        Add text and extract phase-separated content.\n",
+    "        Returns: List of (phase, content) tuples\n",
+    "        \"\"\"\n",
+    "        self.full_buffer += text\n",
+    "        emissions = []\n",
+    "        \n",
+    "        # Extract thinking phase\n",
+    "        if not self.thinking_complete:\n",
+    "            thinking_match = re.search(r'<thinking>(.*?)</thinking>', self.full_buffer, re.DOTALL)\n",
+    "            if thinking_match:\n",
+    "                thinking_content = thinking_match.group(1).strip()\n",
+    "                emissions.append((\"thinking\", thinking_content))\n",
+    "                self.thinking_complete = True\n",
+    "                # Remove from buffer\n",
+    "                self.full_buffer = self.full_buffer[thinking_match.end():]\n",
+    "        \n",
+    "        # Extract answer phase\n",
+    "        if self.thinking_complete and not self.answer_complete:\n",
+    "            answer_match = re.search(r'<answer>(.*?)</answer>', self.full_buffer, re.DOTALL)\n",
+    "            if answer_match:\n",
+    "                answer_content = answer_match.group(1).strip()\n",
+    "                emissions.append((\"answer\", answer_content))\n",
+    "                self.answer_complete = True\n",
+    "                # Remove from buffer\n",
+    "                self.full_buffer = self.full_buffer[answer_match.end():]\n",
+    "        \n",
+    "        # Extract charts (can be multiple)\n",
+    "        if self.answer_complete:\n",
+    "            chart_pattern = re.compile(r'<datastory_chart>(.*?)</datastory_chart>', re.DOTALL)\n",
+    "            matches = list(chart_pattern.finditer(self.full_buffer))\n",
+    "            \n",
+    "            for match in matches:\n",
+    "                chart_content = match.group(1)\n",
+    "                \n",
+    "                # Parse Vega spec\n",
+    "                vega_pattern = r'<vega_spec>(.*?)</vega_spec>'\n",
+    "                vega_match = re.search(vega_pattern, chart_content, re.DOTALL)\n",
+    "                \n",
+    "                if vega_match:\n",
+    "                    try:\n",
+    "                        vega_json_str = vega_match.group(1).strip()\n",
+    "                        vega_spec = json.loads(vega_json_str)\n",
+    "                        \n",
+    "                        # Only emit if not already extracted\n",
+    "                        if vega_spec not in self.extracted_charts:\n",
+    "                            self.extracted_charts.append(vega_spec)\n",
+    "                            emissions.append((\"charts\", vega_spec))\n",
+    "                    except json.JSONDecodeError as e:\n",
+    "                        print(f\"Failed to parse chart JSON: {e}\")\n",
+    "                \n",
+    "                # Remove from buffer\n",
+    "                self.full_buffer = self.full_buffer[match.end():]\n",
+    "        \n",
+    "        return emissions\n",
+    "\n",
+    "# =====================================\n",
+    "# PHASED STREAMING WITH AGENT REASONING\n",
+    "# =====================================\n",
+    "\n",
+    "async def stream_analysis(query: str, file_paths: List[str]) -> AsyncGenerator[Tuple[str, str], None]:\n",
+    "    \"\"\"\n",
+    "    Stream analysis with agent's actual reasoning in thinking phase.\n",
+    "    Yields: (phase, content) tuples where phase is:\n",
+    "    - THINKING: Agent's reasoning process\n",
+    "    - ANSWER: Final analysis and narrative\n",
+    "    - CHART: Vega-Lite chart specification\n",
+    "    - ERROR, DONE: Control signals\n",
+    "    \"\"\"\n",
+    "    print(f\"\\n{'='*70}\")\n",
+    "    print(\"ANALYSIS REQUEST (phased streaming with agent reasoning)\")\n",
+    "    print(f\"Files: {', '.join(file_paths)}\")\n",
+    "    print(f\"Query: {query}\")\n",
+    "    print(f\"{'='*70}\\n\")\n",
+    "\n",
+    "    # Load datasets\n",
+    "    try:\n",
+    "        dataset_context = load_and_register_datasets(file_paths)\n",
+    "        print(f\"\\nDataset Context loaded: {len(dataset_context)} chars\\n\")\n",
+    "    except Exception as e:\n",
+    "        yield (\"ERROR\", f\"Failed to load datasets: {str(e)}\")\n",
+    "        yield (\"DONE\", \"\")\n",
+    "        return\n",
+    "\n",
+    "    # Create agent\n",
+    "    agent = create_data_analyst_agent(dataset_context)\n",
+    "    \n",
+    "    # Initialize phase-aware buffer\n",
+    "    phase_buffer = PhaseAwareBuffer()\n",
+    "    \n",
+    "    try:\n",
+    "        result = Runner.run_streamed(agent, input=query)\n",
+    "        \n",
+    "        async for event in result.stream_events():\n",
+    "            if (\n",
+    "                event.type == \"raw_response_event\"\n",
+    "                and isinstance(event.data, ResponseTextDeltaEvent)\n",
+    "            ):\n",
+    "                delta: str = event.data.delta\n",
+    "                \n",
+    "                # Add to buffer and extract phase-separated content\n",
+    "                emissions = phase_buffer.add_text(delta)\n",
+    "                \n",
+    "                for phase, content in emissions:\n",
+    "                    if phase == \"thinking\":\n",
+    "                        # Stream thinking in chunks for better UX\n",
+    "                        sentences = content.split('\\n')\n",
+    "                        for sentence in sentences:\n",
+    "                            if sentence.strip():\n",
+    "                                yield (\"THINKING\", sentence.strip() + \"\\n\")\n",
+    "                                await asyncio.sleep(0.05)\n",
+    "                    \n",
+    "                    elif phase == \"answer\":\n",
+    "                        # Stream answer in chunks\n",
+    "                        paragraphs = content.split('\\n\\n')\n",
+    "                        for para in paragraphs:\n",
+    "                            if para.strip():\n",
+    "                                yield (\"ANSWER\", para.strip() + \"\\n\\n\")\n",
+    "                                await asyncio.sleep(0.05)\n",
+    "                    \n",
+    "                    elif phase == \"charts\":\n",
+    "                        # Emit complete chart\n",
+    "                        yield (\"CHART\", content)\n",
+    "                        await asyncio.sleep(0.1)\n",
+    "\n",
+    "    except Exception as e:\n",
+    "        yield (\"ERROR\", f\"Streaming error: {str(e)}\")\n",
+    "    \n",
+    "    yield (\"DONE\", \"\")\n",
+    "\n",
+    "# =====================================\n",
+    "# LEGACY STREAMING (BACKWARD COMPATIBILITY)\n",
+    "# =====================================\n",
+    "\n",
+    "async def stream_analysis_legacy(query: str, file_paths: List[str]) -> AsyncGenerator[str, None]:\n",
+    "    \"\"\"\n",
+    "    Original streaming without phases - for backward compatibility\n",
+    "    \"\"\"\n",
+    "    dataset_context = load_and_register_datasets(file_paths)\n",
+    "    agent = create_data_analyst_agent(dataset_context)\n",
+    "    result = Runner.run_streamed(agent, input=query)\n",
+    "    \n",
+    "    async for event in result.stream_events():\n",
+    "        if (\n",
+    "            event.type == \"raw_response_event\"\n",
+    "            and isinstance(event.data, ResponseTextDeltaEvent)\n",
+    "        ):\n",
+    "            yield event.data.delta\n",
+    "    \n",
+    "    yield \"[DONE]\""
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

report_generation/pdf_v3.py

@@ -1,394 +1,394 @@
-
-from fastapi import FastAPI, HTTPException
-from fastapi.responses import HTMLResponse
-from pydantic import BaseModel
-from typing import List, Dict
-import os
-import json
-import httpx
-import asyncio
-from datetime import datetime
-import google.generativeai as genai
-from fastapi import APIRouter 
-
-import base64
-import altair as alt
-import re
-
-# #app = FastAPI(title="Gemini Report Generator API")
-# Report_Generation_Router_ppt_v2 = APIRouter(prefix="/report_generation_ppt_v2", tags=["Report_Generation_ppt_v2"])
-# # ================= CONFIG =================
-# GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "AIzaSyBWACJwKQVwEwACVcoVANgYOXXinwuPNFw")
-# genai.configure(api_key=GEMINI_API_KEY)
-
-
-# === CONFIG ===
-genai.configure(api_key="AIzaSyBWACJwKQVwEwACVcoVANgYOXXinwuPNFw")
-MODEL_NAME = "gemini-2.5-flash"  # CORRECT MODEL
-
-# app = FastAPI(
-#     title="AI Stroke Report Generator",
-#     description="Receive user input → Extract Q&A → Generate 9-page HTML report",
-#     version="1.0"
-# )
-Report_Generation_Router_pdf_v2 = APIRouter(prefix="/report_generation_pdf_v2", tags=["Report_Generation_pdf_v2"])
-# Dynamic model: Get current valid one (fixes 404; adapts to 2025+)
-def get_valid_model():
-    try:
-        models = [m.name for m in genai.list_models() if 'generate_content' in m.supported_generation_methods]
-        flash_model = next((m for m in models if 'gemini-2.0-flash' in m or 'gemini-2.5-flash' in m), models[0] if models else 'gemini-2.0-flash')
-        print(f"✅ Using model: {flash_model}")
-        return genai.GenerativeModel(flash_model)
-    except Exception as e:
-        print(f"⚠️ Model fetch failed ({e}); fallback to gemini-2.0-flash")
-        return genai.GenerativeModel('gemini-2.0-flash')
-
-MODEL = get_valid_model()
-
-OUTPUT_DIR = "pdf_reports"
-CHARTS_DIR = "temp_charts"
-os.makedirs(OUTPUT_DIR, exist_ok=True)
-os.makedirs(CHARTS_DIR, exist_ok=True)
-
-# Themes (minimal)
-THEMES = {
-    "execBlue": {"name": "Executive Blue", "h": "#1B56A6", "a": "#06B6D4", "g1": "#1B56A6", "g2": "#06B6D4"},
-    "sunsetOrange": {"name": "Sunset Orange", "h": "#FF7A1A", "a": "#F59E0B", "g1": "#FF7A1A", "g2": "#F59E0B"},
-    "indigoExec": {"name": "Indigo Executive", "h": "#4C1D95", "a": "#A78BFA", "g1": "#4C1D95", "g2": "#A78BFA"},
-    "healthTeal": {"name": "Healthcare Teal", "h": "#0F766E", "a": "#14B8A6", "g1": "#0F766E", "g2": "#14B8A6"},
-}
-DEFAULT_THEME = "execBlue"
-
-# ================= MODELS =================
-class ReportRequest(BaseModel):
-    format_type: str
-    reportname: str
-    include_citations: bool
-    user_id: str
-    success: bool
-    list_of_queries: List[str]
-    theme: str = "execBlue"
-    
-    # Description is now mandatory for prompt generation.
-    # If the user does not provide it, this default text will be used.
-    Description: str = "Please ensure the report provides a comprehensive analysis of the data, highlighting key findings and actionable recommendations."
-
-# ================= HELPERS =================
-async def fetch_convo(client: httpx.AsyncClient, user_id: str, convo_id: str):
-    url = f"https://mr-mvp-api-dev.dev.ingenspark.com/Db_store_router/conversations/{convo_id}"
-    try:
-        r = await client.get(url, params={"user_id": user_id}, timeout=30)
-        r.raise_for_status()
-        data = r.json().get("conversation") or (r.json().get("conversations") or [None])[0]
-        if not data: return None
-        resp = data.get("response", {})
-        return {"query": data.get("user_query", {}).get("text", ""), "response": resp.get("text", "")}
-    except Exception as e:
-        print(f"Error fetching conversation {convo_id}: {e}")
-        return None
-
-async def fetch_all(user_id: str, convo_ids: List[str]):
-    async with httpx.AsyncClient() as client:
-        tasks = [fetch_convo(client, user_id, cid) for cid in convo_ids]
-        results = await asyncio.gather(*tasks)
-        return [r for r in results if r]
-
-def render_vega(spec: dict) -> str | None:
-    try:
-        chart = alt.Chart.from_dict(spec)
-        path = os.path.join(CHARTS_DIR, f"tmp_{datetime.now().timestamp()}.png")
-        chart.save(path)
-        with open(path, "rb") as f:
-            b64 = base64.b64encode(f.read()).decode()
-        os.remove(path)
-        return f"data:image/png;base64,{b64}"
-    except Exception as e:
-        print(f"Error rendering Vega chart: {e}")
-        return None
-
-# ================= GEMINI (All via one prompt) =================
-def ask_gemini(system: str, user: str) -> str:
-    try:
-        resp = MODEL.generate_content([system, user], generation_config={"temperature": 0.2, "max_output_tokens": 8192})
-        return resp.text.strip()
-    except Exception as e:
-        print(f"Gemini error: {e}")
-        return '{"error": "Gemini failed"}'
-
-def generate_report_content(conversations: List[dict], title: str, theme_id: str, description: str):
-    # Get the theme configuration, fallback to default if theme_id is invalid
-    theme_cfg = THEMES.get(theme_id, THEMES[DEFAULT_THEME])
-    
-    # Always add the description to the prompt, as it's now guaranteed to have a value.
-    description_text = f"\nUser's specific instructions/description: {description}\n"
-
-    prompt = f"""
-You are an expert analyst. Generate a full professional report titled "{title}" from these conversations.{description_text}
-
-Data:
-{json.dumps([{'query': c['query'], 'response': c['response']} for c in conversations], indent=2)}
-
-Do ALL: Analyze data, generate overview/summary/TOC, clean sections (remove narration/tags/suggestions, convert markdown to HTML).
-If the user description mentions specific sections to *focus on* (e.g., 'only graphs and overview'), provide rich and detailed content for those sections. For sections implicitly or explicitly excluded, provide minimal, generic, or empty content but ensure the JSON structure remains valid.
-
-Output ONLY valid JSON (no extra text/markdown):
-{{
-  "overview": "3-4 specific paragraphs (use real numbers/categories from data; formal tone)",
-  "summary": "3-4 paragraphs with actionable recommendations (specific, data-driven)",
-  "toc": [{{"title": "string", "page": int}}],  // Start pages at 4
-  "sections": [  // One per conversation
-    {{
-      "title": "Extracted chart title or query summary",
-      "content_html": "Cleaned <p>text</p><strong>bold</strong><ul><li>bullets</li></ul> (no AI chit-chat)",
-      "chart": {{"title": "string or null", "vega_spec": {{...}} or null, "caption": "string or null"}}
-    }}
-  ]
-}}
-
-Rules: Specific (e.g., '68% in Engineering'); clean aggressively; if no vega_spec, use null.
-"""
-
-    raw = ask_gemini("Precise JSON generator only. No explanations.", prompt)
-
-    # Robust JSON parse
-    try:
-        start = raw.find("{")
-        end = raw.rfind("}") + 1
-        if start == -1: raise ValueError("No JSON")
-        data = json.loads(raw[start:end])
-        required = ["overview", "summary", "toc", "sections"]
-        if not all(k in data for k in required):
-            raise ValueError("Missing keys")
-        return data
-    except Exception as e:
-        print(f"JSON failed: {e}\nRaw: {raw[:1000]}")
-        # Return a fallback structure with minimal content to prevent further errors
-        return {
-            "overview": "Analysis overview based on provided data.",
-            "summary": "Key findings and recommendations.",
-            "toc": [{"title": "Overview", "page": 4}],
-            "sections": [{"title": "Section 1", "content_html": "<p>Sample content.</p>", "chart": {"title": None, "vega_spec": None, "caption": None}}]
-        }
-
-# ================= HTML TEMPLATE (Unchanged Structure) =================
-HTML_TEMPLATE = """<!DOCTYPE html>
-<html><head><meta charset="UTF-8"><title>{title}</title>
-<style>
-body {{font-family:Georgia,serif;margin:0;background:#fff;color:#374151;line-height:1.6;}}
-.page {{width:210mm;min-height:297mm;padding:20mm;background:white;box-shadow:0 4px 8px rgba(0,0,0,0.1);margin:20px auto;position:relative;}}
-.cover-page {{background:linear-gradient(135deg,{g1} 0%,{g2} 100%);color:white;text-align:center;display:flex;justify-content:center;align-items:center;flex-direction:column;}}
-.cover-title {{font-size:36px;font-weight:bold;margin-bottom:20px;text-transform:uppercase;letter-spacing:2px;}}
-.cover-meta {{font-size:16px;margin-top:60px;padding-top:40px;border-top:2px solid rgba(255,255,255,0.3);}}
-.section-title {{font-size:28px;color:{h};margin-bottom:30px;padding-bottom:15px;border-bottom:3px solid {a};}}
-.chart-container {{margin:30px 0;text-align:center;}}
-.chart-container img {{max-width:100%;height:auto;border:1px solid #ddd;border-radius:8px;box-shadow:0 2px 4px rgba(0,0,0,0.1);}}
-.chart-caption {{margin-top:15px;font-style:italic;color:#555;font-size:14px;}}
-.toc-list {{list-style:none;padding:0;margin-top:40px;}}
-.toc-item {{display:flex;justify-content:space-between;padding:15px 0;border-bottom:1px dotted #bdc3c7;font-size:18px;}}
-.page-footer {{position:absolute;bottom:15mm;left:20mm;right:20mm;display:flex;justify-content:space-between;align-items:center;font-size:12px;color:#7f8c8d;border-top:1px solid #ecf0f1;padding-top:10px;}}
-@media print {{body {{background:white;}}.page {{box-shadow:none;margin:0;page-break-after:always;}}}}
-</style></head><body><div class="report-container" style="display:flex;flex-direction:column;gap:20px;padding:20px;max-width:1200px;margin:0 auto;">{body}</div></body></html>"""
-
-# ================= ENDPOINT =================
-@Report_Generation_Router_pdf_v2.post("/generate_report")
-async def generate_report(req: ReportRequest):
-    try:
-        print(f"🚀 Report: {req.reportname} | User: {req.user_id} | Convos: {len(req.list_of_queries)} | Citations: {req.include_citations} | Success: {req.success} | Description: {req.Description}")
-
-        data = await fetch_all(req.user_id, req.list_of_queries)
-        if not data:
-            raise HTTPException(404, "No valid conversations found")
-
-        print(f"✅ Fetched {len(data)} conversations")
-
-        # Normalize the theme from the request here, before passing it around
-        normalized_theme_id = req.theme
-        if "orang" in req.theme.lower():
-            normalized_theme_id = "sunsetOrange"
-        elif normalized_theme_id not in THEMES:
-            normalized_theme_id = DEFAULT_THEME
-        
-        print(f"🎨 Applying theme: {normalized_theme_id}")
-
-        # Pass the normalized theme_id AND the description to generate_report_content
-        content = generate_report_content(data, req.reportname, normalized_theme_id, req.Description)
-        
-        # Use the normalized theme for HTML formatting
-        selected_theme_config = THEMES.get(normalized_theme_id, THEMES[DEFAULT_THEME])
-
-        # Render charts
-        for sec in content.get("sections", []):
-            # Ensure 'sec' is a dictionary before attempting to access its keys
-            if not isinstance(sec, dict):
-                print(f"⚠️ Warning: Non-dictionary item found in 'sections' array, skipping: {sec}")
-                continue # Skip to the next item if it's not a dictionary
-
-            # Retrieve the 'chart' data. If 'chart' key is missing or its value is not a dict (e.g., null),
-            # default it to an empty dictionary to safely call .get() on it.
-            chart_config = sec.get("chart")
-            if not isinstance(chart_config, dict):
-                chart_config = {} # Ensure it's always a dict for safe .get() calls
-
-            if chart_config.get("vega_spec"):
-                b64 = render_vega(chart_config["vega_spec"])
-                chart_config["image_b64"] = b64 if b64 else ""
-                sec["chart"] = chart_config # Update the 'chart' entry in 'sec' with the modified config
-
-        # --- NEW: Parse description to determine what sections to include ---
-        description_lower = req.Description.lower()
-        
-        # Default inclusions (before explicit "no need of" or "only" directives)
-        include_coverpage = True
-        include_toc = True
-        include_overview = True
-        include_sections = True
-        include_summary = True
-
-        # Check for explicit exclusions
-        if "no need of coverpage" in description_lower or "no coverpage" in description_lower:
-            include_coverpage = False
-        if "no need of toc" in description_lower or "no table of contents" in description_lower:
-            include_toc = False
-        if "no need of overview" in description_lower or "no executive overview" in description_lower:
-            include_overview = False
-        if "no need of graphs" in description_lower or "no need of sections" in description_lower: # "graphs" implies sections for charts
-            include_sections = False
-        if "no need of summary" in description_lower or "no need of conclusion" in description_lower:
-            include_summary = False
-
-        # If "only" is present, it overrides previous inclusions to keep only specified parts.
-        # This uses a regex to find "only " followed by a keyword, allowing for more flexible phrasing.
-        only_match = re.search(r"only\s+(.*?)(?:in report|\.|$)", description_lower)
-        if only_match:
-            # Set all to False first, then selectively enable based on "only" keywords found
-            include_coverpage = False
-            include_toc = False
-            include_overview = False
-            include_sections = False
-            include_summary = False
-            
-            only_keywords = only_match.group(1)
-            
-            if "coverpage" in only_keywords:
-                include_coverpage = True
-            if "toc" in only_keywords or "table of contents" in only_keywords:
-                include_toc = True
-            if "overview" in only_keywords or "executive overview" in only_keywords:
-                include_overview = True
-            if "graphs" in only_keywords or "sections" in only_keywords: # "graphs" implies sections for charts
-                include_sections = True
-            if "summary" in only_keywords or "conclusion" in only_keywords:
-                include_summary = True
-
-
-        # Build pages conditionally
-        body_parts = []
-        current_page_number = 1 # Start page numbering from 1
-
-        # Cover
-        if include_coverpage:
-            current_date = datetime.now().strftime('%B %Y')
-            body_parts.append(f'''
-            <div class="page cover-page">
-                <h1 class="cover-title">{req.reportname}</h1>
-                <p class="cover-subtitle">Comprehensive Data Analysis Report</p>
-                <div class="cover-meta">
-                    <div class="cover-company">ACCUSAGA</div>
-                    <div>{current_date}</div>
-                </div>
-            </div>''')
-            current_page_number += 1
-
-        # TOC
-        if include_toc:
-            # Note: TOC page numbers in content["toc"] might not align with actual rendered page numbers.
-            # This is a limitation when dynamically building HTML pages and relying on fixed page numbers from LLM.
-            toc_html = ''.join(f'<li class="toc-item"><span>{item["title"]}</span><span>Page {item["page"]}</span></li>' for item in content.get("toc", []))
-            body_parts.append(f'''
-            <div class="page">
-                <h2 class="section-title">Table of Contents</h2>
-                <ul class="toc-list">{toc_html}</ul>
-                <div class="page-footer"><div>contact@accusaga.com</div><div>Page {current_page_number}</div></div>
-            </div>''')
-            current_page_number += 1
-
-        # Overview
-        if include_overview:
-            body_parts.append(f'''
-            <div class="page">
-                <h2 class="section-title">Executive Overview</h2>
-                <div class="content-area" style="margin-top:20px;line-height:1.8;">{content["overview"]}</div>
-                <div class="page-footer"><div>contact@accusaga.com</div><div>Page {current_page_number}</div></div>
-            </div>''')
-            current_page_number += 1
-
-        # Sections
-        if include_sections:
-            for sec in content.get("sections", []):
-                # Ensure 'sec' is a dictionary before processing
-                if not isinstance(sec, dict):
-                    continue 
-
-                chart_html = ''
-                # Retrieve chart configuration robustly
-                ch = sec.get("chart")
-                if not isinstance(ch, dict):
-                    ch = {} # Ensure ch is a dictionary for safe access
-                    
-                if ch.get("image_b64"):
-                    chart_html = f'''
-                    <div class="chart-container">
-                        <h3>{ch.get("title", "Visualization")}</h3>
-                        <img src="{ch["image_b64"]}" alt="{ch.get("title")}">
-                        <p class="chart-caption">{ch.get("caption", "")}</p>
-                    </div>'''
-                body_parts.append(f'''
-                <div class="page">
-                    <h2 class="section-title">{sec["title"]}</h2>
-                    <div class="content-area" style="margin-top:20px;line-height:1.8;">{sec["content_html"]}{chart_html}</div>
-                    <div class="page-footer"><div>contact@accusaga.com</div><div>Page {current_page_number}</div></div>
-                </div>''')
-                current_page_number += 1
-
-        # Summary
-        if include_summary:
-            body_parts.append(f'''
-            <div class="page">
-                <h2 class="section-title">Summary and Conclusions</h2>
-                <div class="content-area" style="margin-top:20px;line-height:1.8;">{content["summary"]}</div>
-                <div class="page-footer"><div>contact@accusaga.com</div><div>Page {current_page_number}</div></div>
-            </div>''')
-            current_page_number += 1 
-
-        # If no parts are included, return a minimal HTML response
-        if not body_parts:
-            return HTMLResponse(content="<html><head><title>No Report Content</title></head><body><h1>No report content generated or included based on your description.</h1></body></html>")
-
-        html = HTML_TEMPLATE.format(title=req.reportname, 
-                                    g1=selected_theme_config["g1"], 
-                                    g2=selected_theme_config["g2"], 
-                                    h=selected_theme_config["h"], 
-                                    a=selected_theme_config["a"], 
-                                    body=''.join(body_parts))
-
-        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
-        filename = f"report_{req.user_id}_{timestamp}.html"
-        path = os.path.join(OUTPUT_DIR, filename)
-        with open(path, 'w', encoding='utf-8') as f:
-            f.write(html)
-
-        print(f"✅ Report saved: {path}")
-        return HTMLResponse(content=html)
-
-    except HTTPException:
-        raise
-    except Exception as e:
-        print(f"❌ Error: {e}")
-        import traceback
-        traceback.print_exc()
-        raise HTTPException(500, str(e))
-
-@Report_Generation_Router_pdf_v2.get("/")
-async def root():
-    return {"message": "Gemini Report Generator API v3 (Fixed for 2025)", "model": MODEL._model_name, "themes": list(THEMES.keys())}
-
+
+from fastapi import FastAPI, HTTPException
+from fastapi.responses import HTMLResponse
+from pydantic import BaseModel
+from typing import List, Dict
+import os
+import json
+import httpx
+import asyncio
+from datetime import datetime
+import google.generativeai as genai
+from fastapi import APIRouter 
+
+import base64
+import altair as alt
+import re
+
+# #app = FastAPI(title="Gemini Report Generator API")
+# Report_Generation_Router_ppt_v2 = APIRouter(prefix="/report_generation_ppt_v2", tags=["Report_Generation_ppt_v2"])
+# # ================= CONFIG =================
+# GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "AIzaSyBWACJwKQVwEwACVcoVANgYOXXinwuPNFw")
+# genai.configure(api_key=GEMINI_API_KEY)
+
+
+# === CONFIG ===
+genai.configure(api_key="AIzaSyBWACJwKQVwEwACVcoVANgYOXXinwuPNFw")
+MODEL_NAME = "gemini-2.5-flash"  # CORRECT MODEL
+
+# app = FastAPI(
+#     title="AI Stroke Report Generator",
+#     description="Receive user input → Extract Q&A → Generate 9-page HTML report",
+#     version="1.0"
+# )
+Report_Generation_Router_pdf_v2 = APIRouter(prefix="/report_generation_pdf_v2", tags=["Report_Generation_pdf_v2"])
+# Dynamic model: Get current valid one (fixes 404; adapts to 2025+)
+def get_valid_model():
+    try:
+        models = [m.name for m in genai.list_models() if 'generate_content' in m.supported_generation_methods]
+        flash_model = next((m for m in models if 'gemini-2.0-flash' in m or 'gemini-2.5-flash' in m), models[0] if models else 'gemini-2.0-flash')
+        print(f"✅ Using model: {flash_model}")
+        return genai.GenerativeModel(flash_model)
+    except Exception as e:
+        print(f"⚠️ Model fetch failed ({e}); fallback to gemini-2.0-flash")
+        return genai.GenerativeModel('gemini-2.0-flash')
+
+MODEL = get_valid_model()
+
+OUTPUT_DIR = "pdf_reports"
+CHARTS_DIR = "temp_charts"
+os.makedirs(OUTPUT_DIR, exist_ok=True)
+os.makedirs(CHARTS_DIR, exist_ok=True)
+
+# Themes (minimal)
+THEMES = {
+    "execBlue": {"name": "Executive Blue", "h": "#1B56A6", "a": "#06B6D4", "g1": "#1B56A6", "g2": "#06B6D4"},
+    "sunsetOrange": {"name": "Sunset Orange", "h": "#FF7A1A", "a": "#F59E0B", "g1": "#FF7A1A", "g2": "#F59E0B"},
+    "indigoExec": {"name": "Indigo Executive", "h": "#4C1D95", "a": "#A78BFA", "g1": "#4C1D95", "g2": "#A78BFA"},
+    "healthTeal": {"name": "Healthcare Teal", "h": "#0F766E", "a": "#14B8A6", "g1": "#0F766E", "g2": "#14B8A6"},
+}
+DEFAULT_THEME = "execBlue"
+
+# ================= MODELS =================
+class ReportRequest(BaseModel):
+    format_type: str
+    reportname: str
+    include_citations: bool
+    user_id: str
+    success: bool
+    list_of_queries: List[str]
+    theme: str = "execBlue"
+    
+    # Description is now mandatory for prompt generation.
+    # If the user does not provide it, this default text will be used.
+    Description: str = "Please ensure the report provides a comprehensive analysis of the data, highlighting key findings and actionable recommendations."
+
+# ================= HELPERS =================
+async def fetch_convo(client: httpx.AsyncClient, user_id: str, convo_id: str):
+    url = f"https://mr-mvp-api-dev.dev.ingenspark.com/Db_store_router/conversations/{convo_id}"
+    try:
+        r = await client.get(url, params={"user_id": user_id}, timeout=30)
+        r.raise_for_status()
+        data = r.json().get("conversation") or (r.json().get("conversations") or [None])[0]
+        if not data: return None
+        resp = data.get("response", {})
+        return {"query": data.get("user_query", {}).get("text", ""), "response": resp.get("text", "")}
+    except Exception as e:
+        print(f"Error fetching conversation {convo_id}: {e}")
+        return None
+
+async def fetch_all(user_id: str, convo_ids: List[str]):
+    async with httpx.AsyncClient() as client:
+        tasks = [fetch_convo(client, user_id, cid) for cid in convo_ids]
+        results = await asyncio.gather(*tasks)
+        return [r for r in results if r]
+
+def render_vega(spec: dict) -> str | None:
+    try:
+        chart = alt.Chart.from_dict(spec)
+        path = os.path.join(CHARTS_DIR, f"tmp_{datetime.now().timestamp()}.png")
+        chart.save(path)
+        with open(path, "rb") as f:
+            b64 = base64.b64encode(f.read()).decode()
+        os.remove(path)
+        return f"data:image/png;base64,{b64}"
+    except Exception as e:
+        print(f"Error rendering Vega chart: {e}")
+        return None
+
+# ================= GEMINI (All via one prompt) =================
+def ask_gemini(system: str, user: str) -> str:
+    try:
+        resp = MODEL.generate_content([system, user], generation_config={"temperature": 0.2, "max_output_tokens": 8192})
+        return resp.text.strip()
+    except Exception as e:
+        print(f"Gemini error: {e}")
+        return '{"error": "Gemini failed"}'
+
+def generate_report_content(conversations: List[dict], title: str, theme_id: str, description: str):
+    # Get the theme configuration, fallback to default if theme_id is invalid
+    theme_cfg = THEMES.get(theme_id, THEMES[DEFAULT_THEME])
+    
+    # Always add the description to the prompt, as it's now guaranteed to have a value.
+    description_text = f"\nUser's specific instructions/description: {description}\n"
+
+    prompt = f"""
+You are an expert analyst. Generate a full professional report titled "{title}" from these conversations.{description_text}
+
+Data:
+{json.dumps([{'query': c['query'], 'response': c['response']} for c in conversations], indent=2)}
+
+Do ALL: Analyze data, generate overview/summary/TOC, clean sections (remove narration/tags/suggestions, convert markdown to HTML).
+If the user description mentions specific sections to *focus on* (e.g., 'only graphs and overview'), provide rich and detailed content for those sections. For sections implicitly or explicitly excluded, provide minimal, generic, or empty content but ensure the JSON structure remains valid.
+
+Output ONLY valid JSON (no extra text/markdown):
+{{
+  "overview": "3-4 specific paragraphs (use real numbers/categories from data; formal tone)",
+  "summary": "3-4 paragraphs with actionable recommendations (specific, data-driven)",
+  "toc": [{{"title": "string", "page": int}}],  // Start pages at 4
+  "sections": [  // One per conversation
+    {{
+      "title": "Extracted chart title or query summary",
+      "content_html": "Cleaned <p>text</p><strong>bold</strong><ul><li>bullets</li></ul> (no AI chit-chat)",
+      "chart": {{"title": "string or null", "vega_spec": {{...}} or null, "caption": "string or null"}}
+    }}
+  ]
+}}
+
+Rules: Specific (e.g., '68% in Engineering'); clean aggressively; if no vega_spec, use null.
+"""
+
+    raw = ask_gemini("Precise JSON generator only. No explanations.", prompt)
+
+    # Robust JSON parse
+    try:
+        start = raw.find("{")
+        end = raw.rfind("}") + 1
+        if start == -1: raise ValueError("No JSON")
+        data = json.loads(raw[start:end])
+        required = ["overview", "summary", "toc", "sections"]
+        if not all(k in data for k in required):
+            raise ValueError("Missing keys")
+        return data
+    except Exception as e:
+        print(f"JSON failed: {e}\nRaw: {raw[:1000]}")
+        # Return a fallback structure with minimal content to prevent further errors
+        return {
+            "overview": "Analysis overview based on provided data.",
+            "summary": "Key findings and recommendations.",
+            "toc": [{"title": "Overview", "page": 4}],
+            "sections": [{"title": "Section 1", "content_html": "<p>Sample content.</p>", "chart": {"title": None, "vega_spec": None, "caption": None}}]
+        }
+
+# ================= HTML TEMPLATE (Unchanged Structure) =================
+HTML_TEMPLATE = """<!DOCTYPE html>
+<html><head><meta charset="UTF-8"><title>{title}</title>
+<style>
+body {{font-family:Georgia,serif;margin:0;background:#fff;color:#374151;line-height:1.6;}}
+.page {{width:210mm;min-height:297mm;padding:20mm;background:white;box-shadow:0 4px 8px rgba(0,0,0,0.1);margin:20px auto;position:relative;}}
+.cover-page {{background:linear-gradient(135deg,{g1} 0%,{g2} 100%);color:white;text-align:center;display:flex;justify-content:center;align-items:center;flex-direction:column;}}
+.cover-title {{font-size:36px;font-weight:bold;margin-bottom:20px;text-transform:uppercase;letter-spacing:2px;}}
+.cover-meta {{font-size:16px;margin-top:60px;padding-top:40px;border-top:2px solid rgba(255,255,255,0.3);}}
+.section-title {{font-size:28px;color:{h};margin-bottom:30px;padding-bottom:15px;border-bottom:3px solid {a};}}
+.chart-container {{margin:30px 0;text-align:center;}}
+.chart-container img {{max-width:100%;height:auto;border:1px solid #ddd;border-radius:8px;box-shadow:0 2px 4px rgba(0,0,0,0.1);}}
+.chart-caption {{margin-top:15px;font-style:italic;color:#555;font-size:14px;}}
+.toc-list {{list-style:none;padding:0;margin-top:40px;}}
+.toc-item {{display:flex;justify-content:space-between;padding:15px 0;border-bottom:1px dotted #bdc3c7;font-size:18px;}}
+.page-footer {{position:absolute;bottom:15mm;left:20mm;right:20mm;display:flex;justify-content:space-between;align-items:center;font-size:12px;color:#7f8c8d;border-top:1px solid #ecf0f1;padding-top:10px;}}
+@media print {{body {{background:white;}}.page {{box-shadow:none;margin:0;page-break-after:always;}}}}
+</style></head><body><div class="report-container" style="display:flex;flex-direction:column;gap:20px;padding:20px;max-width:1200px;margin:0 auto;">{body}</div></body></html>"""
+
+# ================= ENDPOINT =================
+@Report_Generation_Router_pdf_v2.post("/generate_report")
+async def generate_report(req: ReportRequest):
+    try:
+        print(f"🚀 Report: {req.reportname} | User: {req.user_id} | Convos: {len(req.list_of_queries)} | Citations: {req.include_citations} | Success: {req.success} | Description: {req.Description}")
+
+        data = await fetch_all(req.user_id, req.list_of_queries)
+        if not data:
+            raise HTTPException(404, "No valid conversations found")
+
+        print(f"✅ Fetched {len(data)} conversations")
+
+        # Normalize the theme from the request here, before passing it around
+        normalized_theme_id = req.theme
+        if "orang" in req.theme.lower():
+            normalized_theme_id = "sunsetOrange"
+        elif normalized_theme_id not in THEMES:
+            normalized_theme_id = DEFAULT_THEME
+        
+        print(f"🎨 Applying theme: {normalized_theme_id}")
+
+        # Pass the normalized theme_id AND the description to generate_report_content
+        content = generate_report_content(data, req.reportname, normalized_theme_id, req.Description)
+        
+        # Use the normalized theme for HTML formatting
+        selected_theme_config = THEMES.get(normalized_theme_id, THEMES[DEFAULT_THEME])
+
+        # Render charts
+        for sec in content.get("sections", []):
+            # Ensure 'sec' is a dictionary before attempting to access its keys
+            if not isinstance(sec, dict):
+                print(f"⚠️ Warning: Non-dictionary item found in 'sections' array, skipping: {sec}")
+                continue # Skip to the next item if it's not a dictionary
+
+            # Retrieve the 'chart' data. If 'chart' key is missing or its value is not a dict (e.g., null),
+            # default it to an empty dictionary to safely call .get() on it.
+            chart_config = sec.get("chart")
+            if not isinstance(chart_config, dict):
+                chart_config = {} # Ensure it's always a dict for safe .get() calls
+
+            if chart_config.get("vega_spec"):
+                b64 = render_vega(chart_config["vega_spec"])
+                chart_config["image_b64"] = b64 if b64 else ""
+                sec["chart"] = chart_config # Update the 'chart' entry in 'sec' with the modified config
+
+        # --- NEW: Parse description to determine what sections to include ---
+        description_lower = req.Description.lower()
+        
+        # Default inclusions (before explicit "no need of" or "only" directives)
+        include_coverpage = True
+        include_toc = True
+        include_overview = True
+        include_sections = True
+        include_summary = True
+
+        # Check for explicit exclusions
+        if "no need of coverpage" in description_lower or "no coverpage" in description_lower:
+            include_coverpage = False
+        if "no need of toc" in description_lower or "no table of contents" in description_lower:
+            include_toc = False
+        if "no need of overview" in description_lower or "no executive overview" in description_lower:
+            include_overview = False
+        if "no need of graphs" in description_lower or "no need of sections" in description_lower: # "graphs" implies sections for charts
+            include_sections = False
+        if "no need of summary" in description_lower or "no need of conclusion" in description_lower:
+            include_summary = False
+
+        # If "only" is present, it overrides previous inclusions to keep only specified parts.
+        # This uses a regex to find "only " followed by a keyword, allowing for more flexible phrasing.
+        only_match = re.search(r"only\s+(.*?)(?:in report|\.|$)", description_lower)
+        if only_match:
+            # Set all to False first, then selectively enable based on "only" keywords found
+            include_coverpage = False
+            include_toc = False
+            include_overview = False
+            include_sections = False
+            include_summary = False
+            
+            only_keywords = only_match.group(1)
+            
+            if "coverpage" in only_keywords:
+                include_coverpage = True
+            if "toc" in only_keywords or "table of contents" in only_keywords:
+                include_toc = True
+            if "overview" in only_keywords or "executive overview" in only_keywords:
+                include_overview = True
+            if "graphs" in only_keywords or "sections" in only_keywords: # "graphs" implies sections for charts
+                include_sections = True
+            if "summary" in only_keywords or "conclusion" in only_keywords:
+                include_summary = True
+
+
+        # Build pages conditionally
+        body_parts = []
+        current_page_number = 1 # Start page numbering from 1
+
+        # Cover
+        if include_coverpage:
+            current_date = datetime.now().strftime('%B %Y')
+            body_parts.append(f'''
+            <div class="page cover-page">
+                <h1 class="cover-title">{req.reportname}</h1>
+                <p class="cover-subtitle">Comprehensive Data Analysis Report</p>
+                <div class="cover-meta">
+                    <div class="cover-company">ACCUSAGA</div>
+                    <div>{current_date}</div>
+                </div>
+            </div>''')
+            current_page_number += 1
+
+        # TOC
+        if include_toc:
+            # Note: TOC page numbers in content["toc"] might not align with actual rendered page numbers.
+            # This is a limitation when dynamically building HTML pages and relying on fixed page numbers from LLM.
+            toc_html = ''.join(f'<li class="toc-item"><span>{item["title"]}</span><span>Page {item["page"]}</span></li>' for item in content.get("toc", []))
+            body_parts.append(f'''
+            <div class="page">
+                <h2 class="section-title">Table of Contents</h2>
+                <ul class="toc-list">{toc_html}</ul>
+                <div class="page-footer"><div>contact@accusaga.com</div><div>Page {current_page_number}</div></div>
+            </div>''')
+            current_page_number += 1
+
+        # Overview
+        if include_overview:
+            body_parts.append(f'''
+            <div class="page">
+                <h2 class="section-title">Executive Overview</h2>
+                <div class="content-area" style="margin-top:20px;line-height:1.8;">{content["overview"]}</div>
+                <div class="page-footer"><div>contact@accusaga.com</div><div>Page {current_page_number}</div></div>
+            </div>''')
+            current_page_number += 1
+
+        # Sections
+        if include_sections:
+            for sec in content.get("sections", []):
+                # Ensure 'sec' is a dictionary before processing
+                if not isinstance(sec, dict):
+                    continue 
+
+                chart_html = ''
+                # Retrieve chart configuration robustly
+                ch = sec.get("chart")
+                if not isinstance(ch, dict):
+                    ch = {} # Ensure ch is a dictionary for safe access
+                    
+                if ch.get("image_b64"):
+                    chart_html = f'''
+                    <div class="chart-container">
+                        <h3>{ch.get("title", "Visualization")}</h3>
+                        <img src="{ch["image_b64"]}" alt="{ch.get("title")}">
+                        <p class="chart-caption">{ch.get("caption", "")}</p>
+                    </div>'''
+                body_parts.append(f'''
+                <div class="page">
+                    <h2 class="section-title">{sec["title"]}</h2>
+                    <div class="content-area" style="margin-top:20px;line-height:1.8;">{sec["content_html"]}{chart_html}</div>
+                    <div class="page-footer"><div>contact@accusaga.com</div><div>Page {current_page_number}</div></div>
+                </div>''')
+                current_page_number += 1
+
+        # Summary
+        if include_summary:
+            body_parts.append(f'''
+            <div class="page">
+                <h2 class="section-title">Summary and Conclusions</h2>
+                <div class="content-area" style="margin-top:20px;line-height:1.8;">{content["summary"]}</div>
+                <div class="page-footer"><div>contact@accusaga.com</div><div>Page {current_page_number}</div></div>
+            </div>''')
+            current_page_number += 1 
+
+        # If no parts are included, return a minimal HTML response
+        if not body_parts:
+            return HTMLResponse(content="<html><head><title>No Report Content</title></head><body><h1>No report content generated or included based on your description.</h1></body></html>")
+
+        html = HTML_TEMPLATE.format(title=req.reportname, 
+                                    g1=selected_theme_config["g1"], 
+                                    g2=selected_theme_config["g2"], 
+                                    h=selected_theme_config["h"], 
+                                    a=selected_theme_config["a"], 
+                                    body=''.join(body_parts))
+
+        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+        filename = f"report_{req.user_id}_{timestamp}.html"
+        path = os.path.join(OUTPUT_DIR, filename)
+        with open(path, 'w', encoding='utf-8') as f:
+            f.write(html)
+
+        print(f"✅ Report saved: {path}")
+        return HTMLResponse(content=html)
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        print(f"❌ Error: {e}")
+        import traceback
+        traceback.print_exc()
+        raise HTTPException(500, str(e))
+
+@Report_Generation_Router_pdf_v2.get("/")
+async def root():
+    return {"message": "Gemini Report Generator API v3 (Fixed for 2025)", "model": MODEL._model_name, "themes": list(THEMES.keys())}
+

requirements.txt

@@ -1,91 +1,91 @@
-fastapi
-uvicorn
-pandas
-sqlparse
-duckdb
-ipykernel
-numpy
-openai
-holoviews 
-nbformat
-nbclient
-panel
-bokeh
-plotly
-altair
-redis
-IPython
-langchain_experimental
-google-generativeai
-tabulate
-redis_client
-langchain
-boto3
-python-multipart
-dotenv
-psycopg2-binary
-langchain_qdrant
-langchain_openai
-langchain_community
-langextract
-fastembed-gpu
-redis
-autoviz
-openai-agents
-# openpyxl
-# xlrd
-# odfpy
-openpyxl>=3.0.0
-xlrd>=2.0.1
-odfpy>=1.4.1
-# cairosvg 
-fastapi
-uvicorn
-pyarrow
-pandas
-duckdb
-sqlparse
-pydantic
-requests
-plotly>=5.0.0
-boto3
-python-dotenv
-jupyter
-nbformat
-nbconvert
-
-
-
-
-#==================login==============
-annotated-types==0.7.0
-anyio==4.11.0
-bcrypt==4.2.0
-cffi==2.0.0
-click==8.3.0
-colorama==0.4.6
-cryptography==46.0.3
-dnspython==2.8.0
-ecdsa==0.19.1
-email_validator==2.2.0
-fastapi==0.119.1
-greenlet==3.2.4
-h11==0.16.0
-idna==3.11
-passlib==1.7.4
-psycopg2-binary==2.9.11
-pyasn1==0.6.1
-pycparser==2.23
-pydantic==2.12.3
-pydantic_core==2.41.4
-python-dotenv==1.1.1
-python-jose==3.3.0
-python-multipart==0.0.20
-rsa==4.9.1
-six==1.17.0
-sniffio==1.3.1
-SQLAlchemy==2.0.44
-starlette==0.48.0
-typing-inspection==0.4.2
-typing_extensions==4.15.0
-uvicorn==0.38.0
+fastapi
+uvicorn
+pandas
+sqlparse
+duckdb
+ipykernel
+numpy
+openai
+holoviews 
+nbformat
+nbclient
+panel
+bokeh
+plotly
+redis
+IPython
+langchain_experimental
+google-generativeai
+tabulate
+redis_client
+langchain
+boto3
+python-multipart
+dotenv
+psycopg2-binary
+langchain_qdrant
+langchain_openai
+langchain_community
+langextract
+fastembed-gpu
+redis
+autoviz
+openai-agents
+# openpyxl
+# xlrd
+# odfpy
+openpyxl>=3.0.0
+xlrd>=2.0.1
+odfpy>=1.4.1
+# cairosvg 
+fastapi
+uvicorn
+altair
+pyarrow
+pandas
+duckdb
+sqlparse
+pydantic
+requests
+plotly>=5.0.0
+boto3
+python-dotenv
+jupyter
+nbformat
+nbconvert
+
+
+
+
+#==================login==============
+annotated-types==0.7.0
+anyio==4.11.0
+bcrypt==4.2.0
+cffi==2.0.0
+click==8.3.0
+colorama==0.4.6
+cryptography==46.0.3
+dnspython==2.8.0
+ecdsa==0.19.1
+email_validator==2.2.0
+fastapi==0.119.1
+greenlet==3.2.4
+h11==0.16.0
+idna==3.11
+passlib==1.7.4
+psycopg2-binary==2.9.11
+pyasn1==0.6.1
+pycparser==2.23
+pydantic==2.12.3
+pydantic_core==2.41.4
+python-dotenv==1.1.1
+python-jose==3.3.0
+python-multipart==0.0.20
+rsa==4.9.1
+six==1.17.0
+sniffio==1.3.1
+SQLAlchemy==2.0.44
+starlette==0.48.0
+typing-inspection==0.4.2
+typing_extensions==4.15.0
+uvicorn==0.38.0

s3/create_dataset_graphs.py

@@ -1,295 +1,295 @@
-import pandas as pd
-import numpy as np
-import json
-from typing import Dict, List, Any, Union
-
-def convert_to_serializable(obj):
-    """Convert non-serializable objects to JSON-compatible types"""
-    if pd.isna(obj):
-        return None
-    elif isinstance(obj, (pd.Timestamp, pd.DatetimeTZDtype)):
-        return obj.isoformat()
-    elif isinstance(obj, (np.integer, np.int64, np.int32, np.int16, np.int8)):
-        return int(obj)
-    elif isinstance(obj, (np.floating, np.float64, np.float32, np.float16)):
-        return float(obj)
-    elif isinstance(obj, np.ndarray):
-        return obj.tolist()
-    elif isinstance(obj, (np.bool_, bool)):
-        return bool(obj)
-    elif hasattr(obj, 'isoformat'):
-        return obj.isoformat()
-    else:
-        return str(obj)
-
-
-def analyze_column(series: pd.Series, max_rows: int = 200) -> Dict[str, Any]:
-    """
-    Analyze a single column and return chart-ready data
-    
-    Args:
-        series: Pandas Series to analyze
-        max_rows: Maximum rows to analyze (default: 200)
-        
-    Returns:
-        Dictionary with column analysis including type and visualization data
-    """
-    series = series.head(max_rows).dropna()
-    
-    if len(series) == 0:
-        return {
-            'name': series.name,
-            'type': 'empty',
-            'sampleCount': 0,
-            'distinctCount': 0,
-            'sampleValues': []
-        }
-    
-    # Try numeric analysis
-    numeric_series = pd.to_numeric(series, errors='coerce')
-    valid_nums = numeric_series.dropna()
-    numeric_ratio = len(valid_nums) / len(series) if len(series) > 0 else 0
-    
-    if numeric_ratio >= 0.6 and len(valid_nums) > 2:
-        # Numeric column - compute histogram data
-        min_val = float(valid_nums.min())
-        max_val = float(valid_nums.max())
-        range_val = max_val - min_val if max_val != min_val else 1
-        
-        # Create 12 bins
-        bins = np.zeros(12, dtype=int)
-        bin_edges = np.linspace(min_val, max_val, 13)
-        
-        for val in valid_nums:
-            bin_idx = int((val - min_val) / range_val * 12)
-            if bin_idx >= 12:
-                bin_idx = 11
-            bins[bin_idx] += 1
-        
-        bin_ranges = [[float(bin_edges[i]), float(bin_edges[i+1])] for i in range(12)]
-        
-        return {
-            'name': series.name,
-            'type': 'numeric',
-            'sampleCount': int(len(series)),
-            'distinctCount': int(series.nunique()),
-            'sampleValues': [convert_to_serializable(v) for v in series.head(5).tolist()],
-            'numericStats': {
-                'min': min_val,
-                'max': max_val,
-                'avg': float(valid_nums.mean()),
-                'median': float(valid_nums.median()),
-                'std': float(valid_nums.std()),
-                'bins': bins.tolist(),  # For histogram bars
-                'binRanges': bin_ranges  # For x-axis labels
-            }
-        }
-    
-    # Check if categorical
-    unique_ratio = series.nunique() / len(series)
-    if unique_ratio <= 0.5:
-        # Categorical column
-        value_counts = series.value_counts()
-        top_values = value_counts.head(2)  # Top 2 for the cards
-        
-        categorical_stats = []
-        for value, count in top_values.items():
-            percentage = (count / len(series)) * 100
-            categorical_stats.append({
-                'value': convert_to_serializable(value),
-                'count': int(count),
-                'percentage': round(float(percentage), 1)
-            })
-        
-        # Calculate "Other"
-        top_sum = sum(top_values.values)
-        other_count = len(series) - top_sum
-        other_distinct = series.nunique() - len(top_values)
-        other_percentage = (other_count / len(series)) * 100 if len(series) > 0 else 0
-        
-        categorical_other = None
-        if other_count > 0:
-            categorical_other = {
-                'count': int(other_count),
-                'distinct': int(other_distinct),
-                'percentage': round(float(other_percentage), 1)
-            }
-        
-        return {
-            'name': series.name,
-            'type': 'categorical',
-            'sampleCount': int(len(series)),
-            'distinctCount': int(series.nunique()),
-            'sampleValues': [convert_to_serializable(v) for v in series.head(5).tolist()],
-            'categoricalStats': categorical_stats,  # For bar charts
-            'categoricalOther': categorical_other
-        }
-    
-    # Text column
-    return {
-        'name': series.name,
-        'type': 'text',
-        'sampleCount': int(len(series)),
-        'distinctCount': int(series.nunique()),
-        'sampleValues': [convert_to_serializable(v) for v in series.head(5).tolist()],
-        'textStats': {
-            'avgLength': round(float(series.astype(str).str.len().mean()), 1),
-            'maxLength': int(series.astype(str).str.len().max()),
-            'minLength': int(series.astype(str).str.len().min())
-        }
-    }
-
-
-def create_data_set_graphs(df: pd.DataFrame, max_rows: int = 200) -> str:
-    """
-    Analyze a DataFrame and generate JSON for frontend column visualization cards
-    
-    Args:
-        df: Pandas DataFrame to analyze
-        max_rows: Maximum rows to analyze per column (default: 200)
-        
-    Returns:
-        JSON string containing column summaries with visualization data
-        
-    Example:
-        >>> df = pd.read_csv('data.csv')
-        >>> json_output = create_data_set_graphs(df)
-        >>> print(json_output)
-        
-    JSON Structure:
-        {
-            "columnSummaries": [
-                {
-                    "name": "column_name",
-                    "type": "numeric|categorical|text|empty",
-                    "sampleCount": 1000,
-                    "distinctCount": 45,
-                    "sampleValues": [...],
-                    "numericStats": {...},      // For numeric columns
-                    "categoricalStats": [...],  // For categorical columns
-                    "textStats": {...}          // For text columns
-                }
-            ]
-        }
-    """
-    try:
-        # Analyze all columns
-        column_summaries = []
-        for col in df.columns:
-            analysis = analyze_column(df[col], max_rows=max_rows)
-            column_summaries.append(analysis)
-        
-        # Create the JSON output
-        json_output = {
-            "columnSummaries": column_summaries
-        }
-        
-        # Convert to JSON string
-        return json.dumps(json_output, indent=2)
-    
-    except Exception as e:
-        # Return error as JSON
-        error_output = {
-            "error": str(e),
-            "columnSummaries": []
-        }
-        return json.dumps(error_output, indent=2)
-
-
-def create_data_set_graphs_dict(df: pd.DataFrame, max_rows: int = 200) -> Dict[str, Any]:
-    """
-    Analyze a DataFrame and generate dictionary for frontend column visualization cards
-    
-    Args:
-        df: Pandas DataFrame to analyze
-        max_rows: Maximum rows to analyze per column (default: 200)
-        
-    Returns:
-        Dictionary containing column summaries with visualization data
-    """
-    try:
-        # Analyze all columns
-        column_summaries = []
-        for col in df.columns:
-            analysis = analyze_column(df[col], max_rows=max_rows)
-            column_summaries.append(analysis)
-        
-        # Return as dictionary
-        return {
-            "columnSummaries": column_summaries
-        }
-    
-    except Exception as e:
-        # Return error as dict
-        return {
-            "error": str(e),
-            "columnSummaries": []
-        }
-
-
-# # Example usage
-# if __name__ == "__main__":
-#     # Example 1: Create sample data
-#     sample_df = pd.DataFrame({
-#         'id': range(1, 101),
-#         'name': [f'Person {i}' for i in range(1, 101)],
-#         'age': np.random.randint(20, 60, 100),
-#         'salary': np.random.uniform(30000, 150000, 100),
-#         'department': np.random.choice(['Sales', 'Engineering', 'Marketing', 'HR'], 100),
-#         'email': [f'person{i}@example.com' for i in range(1, 101)],
-#         'join_date': pd.date_range('2020-01-01', periods=100, freq='3D')
-#     })
-    
-#     print("=" * 80)
-#     print("Example 1: Generate JSON string")
-#     print("=" * 80)
-    
-#     # Generate JSON string
-#     json_output = create_data_set_graphs(sample_df)
-#     print(json_output)
-    
-#     # Save to file
-#     with open('column_insights.json', 'w') as f:
-#         f.write(json_output)
-#     print("\n✅ JSON saved to 'column_insights.json'")
-    
-#     print("\n" + "=" * 80)
-#     print("Example 2: Generate dictionary")
-#     print("=" * 80)
-    
-#     # Generate dictionary
-#     dict_output = create_data_set_graphs_dict(sample_df)
-#     print(f"Found {len(dict_output['columnSummaries'])} columns")
-    
-#     # Print first column summary
-#     if dict_output['columnSummaries']:
-#         print("\nFirst column summary:")
-#         print(json.dumps(dict_output['columnSummaries'][0], indent=2))
-    
-#     print("\n" + "=" * 80)
-#     print("Example 3: Load from CSV file")
-#     print("=" * 80)
-    
-#     # If you have a CSV file
-#     # df = pd.read_csv('your_file.csv')
-#     # json_output = create_data_set_graphs(df)
-#     # print(json_output)
-    
-#     print("""
-#     Usage in your code:
-    
-#     # From CSV
-#     df = pd.read_csv('data.csv')
-#     json_result = create_data_set_graphs(df)
-    
-#     # From Excel
-#     df = pd.read_excel('data.xlsx')
-#     json_result = create_data_set_graphs(df)
-    
-#     # As dictionary (for further processing)
-#     dict_result = create_data_set_graphs_dict(df)
-    
-#     # Access column summaries
-#     for column in dict_result['columnSummaries']:
-#         print(f"Column: {column['name']}, Type: {column['type']}")
+import pandas as pd
+import numpy as np
+import json
+from typing import Dict, List, Any, Union
+
+def convert_to_serializable(obj):
+    """Convert non-serializable objects to JSON-compatible types"""
+    if pd.isna(obj):
+        return None
+    elif isinstance(obj, (pd.Timestamp, pd.DatetimeTZDtype)):
+        return obj.isoformat()
+    elif isinstance(obj, (np.integer, np.int64, np.int32, np.int16, np.int8)):
+        return int(obj)
+    elif isinstance(obj, (np.floating, np.float64, np.float32, np.float16)):
+        return float(obj)
+    elif isinstance(obj, np.ndarray):
+        return obj.tolist()
+    elif isinstance(obj, (np.bool_, bool)):
+        return bool(obj)
+    elif hasattr(obj, 'isoformat'):
+        return obj.isoformat()
+    else:
+        return str(obj)
+
+
+def analyze_column(series: pd.Series, max_rows: int = 200) -> Dict[str, Any]:
+    """
+    Analyze a single column and return chart-ready data
+    
+    Args:
+        series: Pandas Series to analyze
+        max_rows: Maximum rows to analyze (default: 200)
+        
+    Returns:
+        Dictionary with column analysis including type and visualization data
+    """
+    series = series.head(max_rows).dropna()
+    
+    if len(series) == 0:
+        return {
+            'name': series.name,
+            'type': 'empty',
+            'sampleCount': 0,
+            'distinctCount': 0,
+            'sampleValues': []
+        }
+    
+    # Try numeric analysis
+    numeric_series = pd.to_numeric(series, errors='coerce')
+    valid_nums = numeric_series.dropna()
+    numeric_ratio = len(valid_nums) / len(series) if len(series) > 0 else 0
+    
+    if numeric_ratio >= 0.6 and len(valid_nums) > 2:
+        # Numeric column - compute histogram data
+        min_val = float(valid_nums.min())
+        max_val = float(valid_nums.max())
+        range_val = max_val - min_val if max_val != min_val else 1
+        
+        # Create 12 bins
+        bins = np.zeros(12, dtype=int)
+        bin_edges = np.linspace(min_val, max_val, 13)
+        
+        for val in valid_nums:
+            bin_idx = int((val - min_val) / range_val * 12)
+            if bin_idx >= 12:
+                bin_idx = 11
+            bins[bin_idx] += 1
+        
+        bin_ranges = [[float(bin_edges[i]), float(bin_edges[i+1])] for i in range(12)]
+        
+        return {
+            'name': series.name,
+            'type': 'numeric',
+            'sampleCount': int(len(series)),
+            'distinctCount': int(series.nunique()),
+            'sampleValues': [convert_to_serializable(v) for v in series.head(5).tolist()],
+            'numericStats': {
+                'min': min_val,
+                'max': max_val,
+                'avg': float(valid_nums.mean()),
+                'median': float(valid_nums.median()),
+                'std': float(valid_nums.std()),
+                'bins': bins.tolist(),  # For histogram bars
+                'binRanges': bin_ranges  # For x-axis labels
+            }
+        }
+    
+    # Check if categorical
+    unique_ratio = series.nunique() / len(series)
+    if unique_ratio <= 0.5:
+        # Categorical column
+        value_counts = series.value_counts()
+        top_values = value_counts.head(2)  # Top 2 for the cards
+        
+        categorical_stats = []
+        for value, count in top_values.items():
+            percentage = (count / len(series)) * 100
+            categorical_stats.append({
+                'value': convert_to_serializable(value),
+                'count': int(count),
+                'percentage': round(float(percentage), 1)
+            })
+        
+        # Calculate "Other"
+        top_sum = sum(top_values.values)
+        other_count = len(series) - top_sum
+        other_distinct = series.nunique() - len(top_values)
+        other_percentage = (other_count / len(series)) * 100 if len(series) > 0 else 0
+        
+        categorical_other = None
+        if other_count > 0:
+            categorical_other = {
+                'count': int(other_count),
+                'distinct': int(other_distinct),
+                'percentage': round(float(other_percentage), 1)
+            }
+        
+        return {
+            'name': series.name,
+            'type': 'categorical',
+            'sampleCount': int(len(series)),
+            'distinctCount': int(series.nunique()),
+            'sampleValues': [convert_to_serializable(v) for v in series.head(5).tolist()],
+            'categoricalStats': categorical_stats,  # For bar charts
+            'categoricalOther': categorical_other
+        }
+    
+    # Text column
+    return {
+        'name': series.name,
+        'type': 'text',
+        'sampleCount': int(len(series)),
+        'distinctCount': int(series.nunique()),
+        'sampleValues': [convert_to_serializable(v) for v in series.head(5).tolist()],
+        'textStats': {
+            'avgLength': round(float(series.astype(str).str.len().mean()), 1),
+            'maxLength': int(series.astype(str).str.len().max()),
+            'minLength': int(series.astype(str).str.len().min())
+        }
+    }
+
+
+def create_data_set_graphs(df: pd.DataFrame, max_rows: int = 200) -> str:
+    """
+    Analyze a DataFrame and generate JSON for frontend column visualization cards
+    
+    Args:
+        df: Pandas DataFrame to analyze
+        max_rows: Maximum rows to analyze per column (default: 200)
+        
+    Returns:
+        JSON string containing column summaries with visualization data
+        
+    Example:
+        >>> df = pd.read_csv('data.csv')
+        >>> json_output = create_data_set_graphs(df)
+        >>> print(json_output)
+        
+    JSON Structure:
+        {
+            "columnSummaries": [
+                {
+                    "name": "column_name",
+                    "type": "numeric|categorical|text|empty",
+                    "sampleCount": 1000,
+                    "distinctCount": 45,
+                    "sampleValues": [...],
+                    "numericStats": {...},      // For numeric columns
+                    "categoricalStats": [...],  // For categorical columns
+                    "textStats": {...}          // For text columns
+                }
+            ]
+        }
+    """
+    try:
+        # Analyze all columns
+        column_summaries = []
+        for col in df.columns:
+            analysis = analyze_column(df[col], max_rows=max_rows)
+            column_summaries.append(analysis)
+        
+        # Create the JSON output
+        json_output = {
+            "columnSummaries": column_summaries
+        }
+        
+        # Convert to JSON string
+        return json.dumps(json_output, indent=2)
+    
+    except Exception as e:
+        # Return error as JSON
+        error_output = {
+            "error": str(e),
+            "columnSummaries": []
+        }
+        return json.dumps(error_output, indent=2)
+
+
+def create_data_set_graphs_dict(df: pd.DataFrame, max_rows: int = 200) -> Dict[str, Any]:
+    """
+    Analyze a DataFrame and generate dictionary for frontend column visualization cards
+    
+    Args:
+        df: Pandas DataFrame to analyze
+        max_rows: Maximum rows to analyze per column (default: 200)
+        
+    Returns:
+        Dictionary containing column summaries with visualization data
+    """
+    try:
+        # Analyze all columns
+        column_summaries = []
+        for col in df.columns:
+            analysis = analyze_column(df[col], max_rows=max_rows)
+            column_summaries.append(analysis)
+        
+        # Return as dictionary
+        return {
+            "columnSummaries": column_summaries
+        }
+    
+    except Exception as e:
+        # Return error as dict
+        return {
+            "error": str(e),
+            "columnSummaries": []
+        }
+
+
+# # Example usage
+# if __name__ == "__main__":
+#     # Example 1: Create sample data
+#     sample_df = pd.DataFrame({
+#         'id': range(1, 101),
+#         'name': [f'Person {i}' for i in range(1, 101)],
+#         'age': np.random.randint(20, 60, 100),
+#         'salary': np.random.uniform(30000, 150000, 100),
+#         'department': np.random.choice(['Sales', 'Engineering', 'Marketing', 'HR'], 100),
+#         'email': [f'person{i}@example.com' for i in range(1, 101)],
+#         'join_date': pd.date_range('2020-01-01', periods=100, freq='3D')
+#     })
+    
+#     print("=" * 80)
+#     print("Example 1: Generate JSON string")
+#     print("=" * 80)
+    
+#     # Generate JSON string
+#     json_output = create_data_set_graphs(sample_df)
+#     print(json_output)
+    
+#     # Save to file
+#     with open('column_insights.json', 'w') as f:
+#         f.write(json_output)
+#     print("\n✅ JSON saved to 'column_insights.json'")
+    
+#     print("\n" + "=" * 80)
+#     print("Example 2: Generate dictionary")
+#     print("=" * 80)
+    
+#     # Generate dictionary
+#     dict_output = create_data_set_graphs_dict(sample_df)
+#     print(f"Found {len(dict_output['columnSummaries'])} columns")
+    
+#     # Print first column summary
+#     if dict_output['columnSummaries']:
+#         print("\nFirst column summary:")
+#         print(json.dumps(dict_output['columnSummaries'][0], indent=2))
+    
+#     print("\n" + "=" * 80)
+#     print("Example 3: Load from CSV file")
+#     print("=" * 80)
+    
+#     # If you have a CSV file
+#     # df = pd.read_csv('your_file.csv')
+#     # json_output = create_data_set_graphs(df)
+#     # print(json_output)
+    
+#     print("""
+#     Usage in your code:
+    
+#     # From CSV
+#     df = pd.read_csv('data.csv')
+#     json_result = create_data_set_graphs(df)
+    
+#     # From Excel
+#     df = pd.read_excel('data.xlsx')
+#     json_result = create_data_set_graphs(df)
+    
+#     # As dictionary (for further processing)
+#     dict_result = create_data_set_graphs_dict(df)
+    
+#     # Access column summaries
+#     for column in dict_result['columnSummaries']:
+#         print(f"Column: {column['name']}, Type: {column['type']}")
 #     """)

s3/r4.py

@@ -1,1057 +1,853 @@
-# from fastapi import FastAPI, UploadFile, File, HTTPException, Query, APIRouter
-# from fastapi.responses import JSONResponse
-# import pandas as pd
-# from autoviz.AutoViz_Class import AutoViz_Class
-# import io, os, boto3, tempfile, glob, matplotlib, json, hashlib, shutil
-# matplotlib.use('Agg')
-# import matplotlib.pyplot as plt
-# import sys
-# from pathlib import Path
-# from typing import List
-# import httpx
-
-# # --- Project Root Setup ---
-# PROJECT_ROOT = Path(__file__).resolve().parents[1]
-# if str(PROJECT_ROOT) not in sys.path:
-#     sys.path.insert(0, str(PROJECT_ROOT))
-
-# from retrieve_secret import *
-# from s3.meta_data_creation_from_s3 import create_file_metadata_from_df
-# from s3.create_dataset_graphs import create_data_set_graphs_dict
-
-# # --- File Validation ---
-# MAX_FILE_SIZE_BYTES = 100 * 1024 * 1024  # 100 MiB
-# MAX_ROWS_ALLOWED = 1_000_000
-# ALLOWED_EXTENSIONS = {".csv", ".xlsx", ".xls", ".ods"}
-
-# # --- AWS S3 Config ---
-# print("AWS S3 config:", AWS_S3_CREDS_KEY_ID, AWS_S3_CREDS_SECRET_KEY, BUCKET_NAME)
-# ACCESS_KEY = AWS_S3_CREDS_KEY_ID
-# SECRET_KEY = AWS_S3_CREDS_SECRET_KEY
-# BUCKET_NAME = BUCKET_NAME
-# REGION_NAME = "us-east-1"
-
-# s3 = boto3.client(
-#     "s3",
-#     aws_access_key_id=ACCESS_KEY,
-#     aws_secret_access_key=SECRET_KEY,
-#     region_name=REGION_NAME
-# )
-
-# ENDPOINT_URL = f"https://s3.{REGION_NAME}.amazonaws.com"
-
-# # --- FastAPI Router ---
-# s3_bucket_router1 = APIRouter(prefix="/s3/v3", tags=["s3_v3"])
-
-
-# # --- Helper: S3 Key ---
-# def make_key(path: str, filename: str) -> str:
-#     return f"{path.strip('/')}/{filename}" if path else filename
-
-
-# # --- Sanitize for JSON ---
-# def sanitize_for_json(obj):
-#     if isinstance(obj, dict):
-#         return {k: sanitize_for_json(v) for k, v in obj.items()}
-#     elif isinstance(obj, list):
-#         return [sanitize_for_json(item) for item in obj]
-#     elif isinstance(obj, (pd.Timestamp, pd.DatetimeTZDtype)):
-#         return str(obj)
-#     elif pd.isna(obj):
-#         return None
-#     elif isinstance(obj, (int, float, str, bool, type(None))):
-#         return obj
-#     else:
-#         return str(obj)
-
-
-# # --- Vector DB Placeholders ---
-# def check_vdb(user_id: str):
-#     print(f"Checking VDB for user: {user_id}")
-
-# async def add_metadata_only(collection_name: str, metadata: dict):
-#     print(f"Adding metadata to collection: {collection_name}")
-#     return {"status": "success", "collection": collection_name}
-
-
-# # --- AutoViz HTML ---
-# def run_autoviz_html(
-#     dfte, depVar="", sep=",", header=0, verbose=0, lowess=False,
-#     max_rows_analyzed=150000, max_cols_analyzed=30
-# ):
-#     tmp_dir = tempfile.mkdtemp()
-#     AV = AutoViz_Class()
-#     AV.AutoViz(
-#         filename="", sep=sep, depVar=depVar, dfte=dfte, header=header,
-#         verbose=verbose, lowess=lowess, chart_format="html",
-#         max_rows_analyzed=max_rows_analyzed, max_cols_analyzed=max_cols_analyzed,
-#         save_plot_dir=tmp_dir
-#     )
-#     print(f"HTML plots saved to: {tmp_dir}")
-#     return tmp_dir
-
-
-# # --- AutoViz Bokeh ---
-# def run_autoviz_bokeh(
-#     dfte, depVar="", sep=",", header=0, verbose=0, lowess=False,
-#     max_rows_analyzed=150000, max_cols_analyzed=30
-# ):
-#     tmp_dir = tempfile.mkdtemp()
-#     print(f"Bokeh temp directory: {tmp_dir}")
-#     try:
-#         matplotlib.use('Agg')
-#         plt.ioff()
-#         AV = AutoViz_Class()
-#         AV.AutoViz(
-#             filename="", sep=sep, depVar=depVar, dfte=dfte, header=header,
-#             verbose=verbose, lowess=lowess, chart_format="bokeh",
-#             max_rows_analyzed=max_rows_analyzed, max_cols_analyzed=max_cols_analyzed,
-#             save_plot_dir=tmp_dir
-#         )
-#         plt.close('all')
-#         print(f"Bokeh plots saved to: {tmp_dir}")
-#         return tmp_dir
-#     except Exception as e:
-#         print(f"Error in Bokeh: {e}")
-#         import traceback
-#         traceback.print_exc()
-#         raise
-
-
-# # --- Fallback Matplotlib ---
-# def generate_matplotlib_plots(df, user_id, base_filename):
-#     print("\nGenerating matplotlib fallback plots...")
-#     tmp_dir = tempfile.mkdtemp()
-#     plot_metadata = []
-#     try:
-#         matplotlib.use('Agg')
-#         numeric_cols = df.select_dtypes(include=['number']).columns.tolist()
-#         if len(numeric_cols) == 0:
-#             print("No numeric columns for plotting")
-#             return []
-
-#         # Correlation heatmap
-#         if len(numeric_cols) > 1:
-#             plt.figure(figsize=(10, 8))
-#             import seaborn as sns
-#             corr = df[numeric_cols].corr()
-#             sns.heatmap(corr, annot=True, cmap='coolwarm', center=0)
-#             plt.title('Correlation Heatmap')
-#             plt.tight_layout()
-#             png_path = os.path.join(tmp_dir, 'correlation_heatmap.png')
-#             plt.savefig(png_path, dpi=100, bbox_inches='tight')
-#             plt.close()
-
-#             s3_key = f"{user_id}/files/datasets/{base_filename}/pngs/correlation_heatmap.png"
-#             with open(png_path, 'rb') as f:
-#                 s3.put_object(Bucket=BUCKET_NAME, Key=s3_key, Body=f.read(), ContentType='image/png')
-#             plot_metadata.append({
-#                 "file_name": "correlation_heatmap.png",
-#                 "s3_path": s3_key,
-#                 "url": f"{ENDPOINT_URL}/{BUCKET_NAME}/{s3_key}",
-#                 "type": "png",
-#                 "plot_type": "heatmap"
-#             })
-
-#         # Histograms
-#         for col in numeric_cols[:5]:
-#             plt.figure(figsize=(10, 6))
-#             df[col].hist(bins=30, edgecolor='black')
-#             plt.title(f'Distribution of {col}')
-#             plt.xlabel(col)
-#             plt.ylabel('Frequency')
-#             plt.tight_layout()
-#             png_path = os.path.join(tmp_dir, f'distribution_{col}.png')
-#             plt.savefig(png_path, dpi=100, bbox_inches='tight')
-#             plt.close()
-
-#             s3_key = f"{user_id}/files/datasets/{base_filename}/pngs/distribution_{col}.png"
-#             with open(png_path, 'rb') as f:
-#                 s3.put_object(Bucket=BUCKET_NAME, Key=s3_key, Body=f.read(), ContentType='image/png')
-#             plot_metadata.append({
-#                 "file_name": f"distribution_{col}.png",
-#                 "s3_path": s3_key,
-#                 "url": f"{ENDPOINT_URL}/{BUCKET_NAME}/{s3_key}",
-#                 "type": "png",
-#                 "plot_type": "histogram"
-#             })
-#         return plot_metadata
-#     except Exception as e:
-#         print(f"Matplotlib failed: {e}")
-#         return []
-#     finally:
-#         shutil.rmtree(tmp_dir, ignore_errors=True)
-
-
-# # --- Upload Viz Files ---
-# def upload_viz_files_to_s3(tmp_dir, file_pattern, user_id, base_filename, viz_type):
-#     patterns = [file_pattern, f"*.{viz_type}", f"**/*.{viz_type}"]
-#     files = []
-#     for p in patterns:
-#         files.extend(glob.glob(os.path.join(tmp_dir, p), recursive=True))
-#     files = list(set(files))
-#     if not files:
-#         print(f"No {viz_type} files found")
-#         return []
-
-#     folder = "htmls" if viz_type == "html" else "svgs" if viz_type == "svg" else "pngs"
-#     metadata = []
-#     content_type_map = {
-#         'html': 'text/html', 'svg': 'image/svg+xml', 'png': 'image/png',
-#         'jpg': 'image/jpeg', 'jpeg': 'image/jpeg'
-#     }
-
-#     for file_path in files:
-#         name = os.path.basename(file_path)
-#         ext = os.path.splitext(name)[1][1:]
-#         s3_key = f"{user_id}/files/datasets/{base_filename}/{folder}/{name}"
-#         with open(file_path, "rb") as f:
-#             body = f.read()
-#         s3.put_object(
-#             Bucket=BUCKET_NAME, Key=s3_key, Body=body,
-#             ContentType=content_type_map.get(ext, 'application/octet-stream')
-#         )
-#         metadata.append({
-#             "file_name": name, "s3_path": s3_key,
-#             "url": f"{ENDPOINT_URL}/{BUCKET_NAME}/{s3_key}",
-#             "type": viz_type, "size": len(body)
-#         })
-#         print(f"{viz_type.upper()} uploaded: {s3_key}")
-#     return metadata
-
-
-# # --- Convert to Parquet ---
-# def convert_df_to_parquet(df: pd.DataFrame) -> io.BytesIO:
-#     buffer = io.BytesIO()
-#     df.to_parquet(buffer, engine='pyarrow', compression='snappy', index=False)
-#     buffer.seek(0)
-#     return buffer
-
-
-# # --- Check File Hash Exists (FIXED) ---
-# async def check_file_hash_exists(user_id: str, file_hash: str) -> dict:
-#     """
-#     Check if a file hash already exists for a user.
-#     Returns a dict with 'success', 'exists', and optional 'data' keys.
-#     """
-#     url = f"https://mr-mvp-api-dev.dev.ingenspark.com/auth/UserMetadata/{user_id}/check_file_hash?file_hash={file_hash}"
-#     headers = {"accept": "application/json"}
-#     async with httpx.AsyncClient(timeout=10.0) as client:
-#         try:
-#             r = await client.get(url, headers=headers)
-#             r.raise_for_status()
-#             result = r.json()
-            
-#             # Log the actual response for debugging
-#             print(f"Hash check API response: {result}")
-            
-#             # Check for 'exists' field first, then parse message if not present
-#             exists = result.get("exists", None)
-            
-#             # If 'exists' field is not present, parse the message
-#             if exists is None and "message" in result:
-#                 message_lower = result["message"].lower()
-#                 # Check if message indicates file already exists
-#                 exists = "already existed" in message_lower or "already exists" in message_lower or "duplicate" in message_lower
-            
-#             # Default to False if still None
-#             if exists is None:
-#                 exists = False
-            
-#             return {
-#                 "success": True,
-#                 "exists": exists,
-#                 "data": result
-#             }
-#         except httpx.HTTPStatusError as e:
-#             print(f"Hash check HTTP error: {e.response.status_code} - {e.response.text}")
-#             return {
-#                 "success": False, 
-#                 "exists": False,  # Assume not exists on error to be safe
-#                 "message": f"HTTP {e.response.status_code}",
-#                 "error_detail": e.response.text
-#             }
-#         except Exception as e:
-#             print(f"Hash check exception: {str(e)}")
-#             return {
-#                 "success": False, 
-#                 "exists": False,  # Assume not exists on error
-#                 "message": f"Request failed: {str(e)}"
-#             }
-
-
-# # --- PostgreSQL Metadata Upload with file_hash ---
-# async def user_metadata_upload_pg(
-#     user_id: str,
-#     user_metadata: str,
-#     path: str,
-#     url: str,
-#     filename: str,
-#     file_type: str,
-#     file_size_bytes: int,
-#     file_hash: str,
-#     timeout: float = 10.0
-# ):
-#     payload = {
-#         "user_id": user_id,
-#         "user_metadata": user_metadata,
-#         "path": path,
-#         "url": url,
-#         "filename": filename,
-#         "file_type": file_type,
-#         "file_size_bytes": file_size_bytes,
-#         "file_hash": file_hash
-#     }
-#     print(f"PostgreSQL payload file_hash: {payload['file_hash']}")
-#     async with httpx.AsyncClient() as client:
-#         try:
-#             r = await client.post(
-#                 "https://mr-mvp-api-dev.dev.ingenspark.com/auth/UserMetadataCreate",
-#                 json=payload,
-#                 timeout=timeout
-#             )
-#             r.raise_for_status()
-#             result = r.json()
-#             result["file_hash"] = file_hash
-#             print(f"PostgreSQL result file_hash: {result['file_hash']}")
-#             return {"success": True, "data": result}
-#         except httpx.HTTPStatusError as e:
-#             return {
-#                 "success": False,
-#                 "error": "HTTP error",
-#                 "status_code": e.response.status_code,
-#                 "detail": e.response.text,
-#                 "file_hash": file_hash
-#             }
-#         except Exception as e:
-#             return {
-#                 "success": False,
-#                 "error": "Request failed",
-#                 "detail": str(e),
-#                 "file_hash": file_hash
-#             }
-
-
-# # --- DEBUG ENDPOINT (Optional - for troubleshooting) ---
-# @s3_bucket_router1.get("/debug/check_hash/{user_id}/{file_hash}")
-# async def debug_check_hash(user_id: str, file_hash: str):
-#     """Debug endpoint to test hash checking"""
-#     result = await check_file_hash_exists(user_id, file_hash)
-#     return {
-#         "raw_result": result,
-#         "exists_value": result.get("exists"),
-#         "exists_type": type(result.get("exists")).__name__,
-#         "success_value": result.get("success"),
-#         "interpretation": "File exists" if result.get("exists") is True else "File does not exist or check failed"
-#     }
-
-
-# # --- MAIN ENDPOINT WITH FIXED HASH CHECK AND DATASET GRAPHS ---
-# @s3_bucket_router1.post("/upload_datasets_v3/")
-# async def upload_file(
-#     file: UploadFile = File(...),
-#     user_id: str = Query(..., description="User ID"),
-#     path: str = Query("", description="Optional subpath")
-# ):
-#     html_tmp_dir = None
-#     bokeh_tmp_dir = None
-#     file_content = None
-
-#     try:
-#         # 1. Validate extension
-#         file_ext = os.path.splitext(file.filename)[1].lower()
-#         if file_ext not in ALLOWED_EXTENSIONS:
-#             raise HTTPException(status_code=400, detail=f"Unsupported file type: {file_ext}")
-
-#         # 2. Read file
-#         file_content = await file.read()
-#         if not file_content:
-#             raise HTTPException(status_code=400, detail="Empty file")
-#         if len(file_content) > MAX_FILE_SIZE_BYTES:
-#             raise HTTPException(status_code=413, detail="File exceeds 100 MiB limit")
-
-#         # 3. Generate hash
-#         file_hash = hashlib.sha256(file_content).hexdigest()
-#         print(f"Generated file hash: {file_hash}")
-
-#         # 4. Check hash via API (FIXED LOGIC)
-#         hash_result = await check_file_hash_exists(user_id, file_hash)
-        
-#         # Enhanced logging
-#         print(f"Hash check result: {hash_result}")
-        
-#         # Check if the API call was successful first
-#         if not hash_result.get("success", False):
-#             print(f"⚠️ Warning: Hash check API failed: {hash_result.get('message')}")
-#             # You can decide to fail here if hash check is critical
-#             # raise HTTPException(status_code=503, detail="Hash check service unavailable")
-        
-#         # Check if file exists
-#         if hash_result.get("exists") is True:
-#             print(f"🚫 Duplicate file detected: {file_hash}")
-#             return JSONResponse(
-#                 status_code=409,
-#                 content={
-#                     "message": "File already uploaded.",
-#                     "reason": "Duplicate file detected via SHA-256 hash.",
-#                     "file_hash": file_hash,
-#                     "user_id": user_id,
-#                     "filename": file.filename,
-#                     "action": "skipped",
-#                     "existing_file_info": hash_result.get("data")
-#                 }
-#             )
-        
-#         print("✅ Hash check passed. New file - proceeding with upload.")
-
-#         # 5. Load DataFrame
-#         try:
-#             if file_ext == ".csv":
-#                 df = pd.read_csv(io.BytesIO(file_content))
-#             elif file_ext in {".xlsx", ".xls"}:
-#                 engine = 'openpyxl' if file_ext == ".xlsx" else 'xlrd'
-#                 df = pd.read_excel(io.BytesIO(file_content), engine=engine)
-#             elif file_ext == ".ods":
-#                 df = pd.read_excel(io.BytesIO(file_content), engine='odf')
-#         except Exception as e:
-#             raise HTTPException(status_code=400, detail=f"Failed to parse file: {str(e)}")
-
-#         if len(df) > MAX_ROWS_ALLOWED:
-#             raise HTTPException(status_code=413, detail=f"Too many rows: {len(df):,} > {MAX_ROWS_ALLOWED:,}")
-
-#         # 6. Convert to Parquet
-#         parquet_buffer = convert_df_to_parquet(df)
-#         parquet_size = parquet_buffer.getbuffer().nbytes
-
-#         # 7. Upload Parquet to S3
-#         base_filename = os.path.splitext(file.filename)[0]
-#         parquet_filename = f"{base_filename}.parquet"
-#         file_key = f"{user_id}/files/datasets/{parquet_filename}"
-#         file_url = f"{ENDPOINT_URL}/{BUCKET_NAME}/{file_key}"
-
-#         s3.upload_fileobj(parquet_buffer, BUCKET_NAME, file_key,
-#                           ExtraArgs={'ContentType': 'application/octet-stream'})
-#         print(f"Uploaded Parquet: {file_url}")
-
-#         # 8. Generate metadata
-#         metadata = create_file_metadata_from_df(df, parquet_filename, file_key)
-#         metadata.update({
-#             "user_id": user_id,
-#             "s3_path": file_key,
-#             "s3_url": file_url,
-#             "source_file": file.filename,
-#             "source_file_type": file_ext[1:],
-#             "file_type": "parquet",
-#             "original_file_size_bytes": len(file_content),
-#             "parquet_file_size_bytes": parquet_size,
-#             "compression_ratio": f"{(1 - parquet_size/len(file_content))*100:.1f}%",
-#             "file_hash": file_hash
-#         })
-
-#         # 8.5 Generate dataset preview graphs
-#         print("Generating dataset preview graphs...")
-#         try:
-#             dataset_graphs = create_data_set_graphs_dict(df, max_rows=200)
-#             metadata["data_sets_preview_graph"] = dataset_graphs
-#             print(f"✅ Generated graphs for {len(dataset_graphs.get('columnSummaries', []))} columns")
-#         except Exception as e:
-#             print(f"⚠️ Failed to generate dataset graphs: {e}")
-#             import traceback
-#             traceback.print_exc()
-#             metadata["data_sets_preview_graph"] = {
-#                 "error": str(e),
-#                 "columnSummaries": []
-#             }
-
-#         safe_metadata = sanitize_for_json(metadata)
-
-#         # 9. Vector DB
-#         check_vdb(user_id)
-#         vdb_res = await add_metadata_only("sri_1_files_&_files_metadata", metadata)
-#         vdb_success = vdb_res.get("status") == "success"
-#         print(f"VDB upload success: {vdb_success}")
-
-#         # 10. PostgreSQL Metadata + file_hash
-#         pg_result = await user_metadata_upload_pg(
-#             user_id=user_id,
-#             user_metadata=json.dumps(safe_metadata),
-#             path=file_key,
-#             url=file_url,
-#             filename=parquet_filename,
-#             file_type="parquet",
-#             file_size_bytes=parquet_size,
-#             file_hash=file_hash
-#         )
-#         print(f"PostgreSQL upload result: {pg_result}")
-#         pg_success = pg_result.get("success", False)
-        
-#         graphs_count = len(safe_metadata.get("data_sets_preview_graph", {}).get("columnSummaries", []))
-#         print(f"Graphs generated: {graphs_count}")
-
-#         # 11. Return success
-#         return {
-#             "message": "Upload successful.",
-#             "filename": parquet_filename,
-#             "original_filename": file.filename,
-#             "user_id": user_id,
-#             "file_path": file_key,
-#             "file_url": file_url,
-#             "file_hash": file_hash,
-#             "source_file_type": file_ext[1:],
-#             "file_type": "parquet",
-#             "original_file_size_bytes": len(file_content),
-#             "parquet_file_size_bytes": parquet_size,
-#             "compression_ratio": f"{(1 - parquet_size/len(file_content))*100:.1f}%",
-#             "rows": len(df),
-#             "columns": len(df.columns),
-#             "metadata": safe_metadata,
-#             "upload_dataset_vdb": vdb_success,
-#             "upload_dataset_pg": pg_success,
-#             "pg_details": pg_result,
-#             "graphs_generated": graphs_count
-#         }
-
-#     except HTTPException:
-#         raise
-#     except Exception as e:
-#         print(f"Unexpected error: {e}")
-#         import traceback
-#         traceback.print_exc()
-#         raise HTTPException(status_code=500, detail=str(e))
-#     finally:
-#         # Clean up temp directories
-#         for d in (html_tmp_dir, bokeh_tmp_dir):
-#             if d and os.path.exists(d):
-#                 shutil.rmtree(d, ignore_errors=True)
-
-
-
-
-
-from fastapi import FastAPI, UploadFile, File, HTTPException, Query, APIRouter
-from fastapi.responses import JSONResponse
-import pandas as pd
-from autoviz.AutoViz_Class import AutoViz_Class
-import io, os, boto3, tempfile, glob, matplotlib, json, hashlib, shutil
-matplotlib.use('Agg')
-import matplotlib.pyplot as plt
-import sys
-from pathlib import Path
-from typing import List
-import httpx
-
-# --- Project Root Setup ---
-PROJECT_ROOT = Path(__file__).resolve().parents[1]
-if str(PROJECT_ROOT) not in sys.path:
-    sys.path.insert(0, str(PROJECT_ROOT))
-
-from retrieve_secret import *
-from s3.meta_data_creation_from_s3 import create_file_metadata_from_df
-from s3.create_dataset_graphs import create_data_set_graphs_dict
-
-# --- File Validation ---
-MAX_FILE_SIZE_BYTES = 100 * 1024 * 1024  # 100 MiB
-MAX_ROWS_ALLOWED = 1_000_000
-ALLOWED_EXTENSIONS = {".csv", ".xlsx", ".xls", ".ods"}
-
-# --- AWS S3 Config ---
-print("AWS S3 config:", AWS_S3_CREDS_KEY_ID, AWS_S3_CREDS_SECRET_KEY, BUCKET_NAME)
-ACCESS_KEY = AWS_S3_CREDS_KEY_ID
-SECRET_KEY = AWS_S3_CREDS_SECRET_KEY
-BUCKET_NAME = BUCKET_NAME
-REGION_NAME = "us-east-1"
-
-s3 = boto3.client(
-    "s3",
-    aws_access_key_id=ACCESS_KEY,
-    aws_secret_access_key=SECRET_KEY,
-    region_name=REGION_NAME
-)
-
-ENDPOINT_URL = f"https://s3.{REGION_NAME}.amazonaws.com"
-
-# --- FastAPI Router ---
-s3_bucket_router1 = APIRouter(prefix="/s3/v3", tags=["s3_v3"])
-
-
-# --- Helper: S3 Key ---
-def make_key(path: str, filename: str) -> str:
-    return f"{path.strip('/')}/{filename}" if path else filename
-
-
-# --- Sanitize for JSON ---
-def sanitize_for_json(obj):
-    if isinstance(obj, dict):
-        return {k: sanitize_for_json(v) for k, v in obj.items()}
-    elif isinstance(obj, list):
-        return [sanitize_for_json(item) for item in obj]
-    elif isinstance(obj, (pd.Timestamp, pd.DatetimeTZDtype)):
-        return str(obj)
-    elif pd.isna(obj):
-        return None
-    elif isinstance(obj, (int, float, str, bool, type(None))):
-        return obj
-    else:
-        return str(obj)
-
-
-# --- Vector DB Placeholders ---
-def check_vdb(user_id: str):
-    print(f"Checking VDB for user: {user_id}")
-
-async def add_metadata_only(collection_name: str, metadata: dict):
-    print(f"Adding metadata to collection: {collection_name}")
-    return {"status": "success", "collection": collection_name}
-
-
-# --- AutoViz HTML ---
-def run_autoviz_html(
-    dfte, depVar="", sep=",", header=0, verbose=0, lowess=False,
-    max_rows_analyzed=150000, max_cols_analyzed=30
-):
-    tmp_dir = tempfile.mkdtemp()
-    AV = AutoViz_Class()
-    AV.AutoViz(
-        filename="", sep=sep, depVar=depVar, dfte=dfte, header=header,
-        verbose=verbose, lowess=lowess, chart_format="html",
-        max_rows_analyzed=max_rows_analyzed, max_cols_analyzed=max_cols_analyzed,
-        save_plot_dir=tmp_dir
-    )
-    print(f"HTML plots saved to: {tmp_dir}")
-    return tmp_dir
-
-
-# --- AutoViz Bokeh ---
-def run_autoviz_bokeh(
-    dfte, depVar="", sep=",", header=0, verbose=0, lowess=False,
-    max_rows_analyzed=150000, max_cols_analyzed=30
-):
-    tmp_dir = tempfile.mkdtemp()
-    print(f"Bokeh temp directory: {tmp_dir}")
-    try:
-        matplotlib.use('Agg')
-        plt.ioff()
-        AV = AutoViz_Class()
-        AV.AutoViz(
-            filename="", sep=sep, depVar=depVar, dfte=dfte, header=header,
-            verbose=verbose, lowess=lowess, chart_format="bokeh",
-            max_rows_analyzed=max_rows_analyzed, max_cols_analyzed=max_cols_analyzed,
-            save_plot_dir=tmp_dir
-        )
-        plt.close('all')
-        print(f"Bokeh plots saved to: {tmp_dir}")
-        return tmp_dir
-    except Exception as e:
-        print(f"Error in Bokeh: {e}")
-        import traceback
-        traceback.print_exc()
-        raise
-
-
-# --- Fallback Matplotlib ---
-def generate_matplotlib_plots(df, user_id, base_filename):
-    print("\nGenerating matplotlib fallback plots...")
-    tmp_dir = tempfile.mkdtemp()
-    plot_metadata = []
-    try:
-        matplotlib.use('Agg')
-        numeric_cols = df.select_dtypes(include=['number']).columns.tolist()
-        if len(numeric_cols) == 0:
-            print("No numeric columns for plotting")
-            return []
-
-        # Correlation heatmap
-        if len(numeric_cols) > 1:
-            plt.figure(figsize=(10, 8))
-            import seaborn as sns
-            corr = df[numeric_cols].corr()
-            sns.heatmap(corr, annot=True, cmap='coolwarm', center=0)
-            plt.title('Correlation Heatmap')
-            plt.tight_layout()
-            png_path = os.path.join(tmp_dir, 'correlation_heatmap.png')
-            plt.savefig(png_path, dpi=100, bbox_inches='tight')
-            plt.close()
-
-            s3_key = f"{user_id}/files/datasets/{base_filename}/pngs/correlation_heatmap.png"
-            with open(png_path, 'rb') as f:
-                s3.put_object(Bucket=BUCKET_NAME, Key=s3_key, Body=f.read(), ContentType='image/png')
-            plot_metadata.append({
-                "file_name": "correlation_heatmap.png",
-                "s3_path": s3_key,
-                "url": f"{ENDPOINT_URL}/{BUCKET_NAME}/{s3_key}",
-                "type": "png",
-                "plot_type": "heatmap"
-            })
-
-        # Histograms
-        for col in numeric_cols[:5]:
-            plt.figure(figsize=(10, 6))
-            df[col].hist(bins=30, edgecolor='black')
-            plt.title(f'Distribution of {col}')
-            plt.xlabel(col)
-            plt.ylabel('Frequency')
-            plt.tight_layout()
-            png_path = os.path.join(tmp_dir, f'distribution_{col}.png')
-            plt.savefig(png_path, dpi=100, bbox_inches='tight')
-            plt.close()
-
-            s3_key = f"{user_id}/files/datasets/{base_filename}/pngs/distribution_{col}.png"
-            with open(png_path, 'rb') as f:
-                s3.put_object(Bucket=BUCKET_NAME, Key=s3_key, Body=f.read(), ContentType='image/png')
-            plot_metadata.append({
-                "file_name": f"distribution_{col}.png",
-                "s3_path": s3_key,
-                "url": f"{ENDPOINT_URL}/{BUCKET_NAME}/{s3_key}",
-                "type": "png",
-                "plot_type": "histogram"
-            })
-        return plot_metadata
-    except Exception as e:
-        print(f"Matplotlib failed: {e}")
-        return []
-    finally:
-        shutil.rmtree(tmp_dir, ignore_errors=True)
-
-
-# --- Upload Viz Files ---
-def upload_viz_files_to_s3(tmp_dir, file_pattern, user_id, base_filename, viz_type):
-    patterns = [file_pattern, f"*.{viz_type}", f"**/*.{viz_type}"]
-    files = []
-    for p in patterns:
-        files.extend(glob.glob(os.path.join(tmp_dir, p), recursive=True))
-    files = list(set(files))
-    if not files:
-        print(f"No {viz_type} files found")
-        return []
-
-    folder = "htmls" if viz_type == "html" else "svgs" if viz_type == "svg" else "pngs"
-    metadata = []
-    content_type_map = {
-        'html': 'text/html', 'svg': 'image/svg+xml', 'png': 'image/png',
-        'jpg': 'image/jpeg', 'jpeg': 'image/jpeg'
-    }
-
-    for file_path in files:
-        name = os.path.basename(file_path)
-        ext = os.path.splitext(name)[1][1:]
-        s3_key = f"{user_id}/files/datasets/{base_filename}/{folder}/{name}"
-        with open(file_path, "rb") as f:
-            body = f.read()
-        s3.put_object(
-            Bucket=BUCKET_NAME, Key=s3_key, Body=body,
-            ContentType=content_type_map.get(ext, 'application/octet-stream')
-        )
-        metadata.append({
-            "file_name": name, "s3_path": s3_key,
-            "url": f"{ENDPOINT_URL}/{BUCKET_NAME}/{s3_key}",
-            "type": viz_type, "size": len(body)
-        })
-        print(f"{viz_type.upper()} uploaded: {s3_key}")
-    return metadata
-
-
-# --- Convert to Parquet ---
-def convert_df_to_parquet(df: pd.DataFrame) -> io.BytesIO:
-    buffer = io.BytesIO()
-    df.to_parquet(buffer, engine='pyarrow', compression='snappy', index=False)
-    buffer.seek(0)
-    return buffer
-
-
-# --- Check File Hash Exists (FIXED) ---
-async def check_file_hash_exists(user_id: str, file_hash: str) -> dict:
-    """
-    Check if a file hash already exists for a user.
-    Returns a dict with 'success', 'exists', and optional 'data' keys.
-    """
-    url = f"https://mr-mvp-api-dev.dev.ingenspark.com/auth/UserMetadata/{user_id}/check_file_hash?file_hash={file_hash}"
-    headers = {"accept": "application/json"}
-    async with httpx.AsyncClient(timeout=10.0) as client:
-        try:
-            r = await client.get(url, headers=headers)
-            r.raise_for_status()
-            result = r.json()
-            
-            # Log the actual response for debugging
-            print(f"Hash check API response: {result}")
-            
-            # Check for 'exists' field first, then parse message if not present
-            exists = result.get("exists", None)
-            
-            # If 'exists' field is not present, parse the message
-            if exists is None and "message" in result:
-                message_lower = result["message"].lower()
-                # Check if message indicates file already exists
-                exists = "already existed" in message_lower or "already exists" in message_lower or "duplicate" in message_lower
-            
-            # Default to False if still None
-            if exists is None:
-                exists = False
-            
-            return {
-                "success": True,
-                "exists": exists,
-                "data": result
-            }
-        except httpx.HTTPStatusError as e:
-            print(f"Hash check HTTP error: {e.response.status_code} - {e.response.text}")
-            return {
-                "success": False, 
-                "exists": False,  # Assume not exists on error to be safe
-                "message": f"HTTP {e.response.status_code}",
-                "error_detail": e.response.text
-            }
-        except Exception as e:
-            print(f"Hash check exception: {str(e)}")
-            return {
-                "success": False, 
-                "exists": False,  # Assume not exists on error
-                "message": f"Request failed: {str(e)}"
-            }
-
-
-# --- PostgreSQL Metadata Upload with file_hash ---
-async def user_metadata_upload_pg(
-    user_id: str,
-    user_metadata: str,
-    path: str,
-    url: str,
-    filename: str,
-    file_type: str,
-    file_size_bytes: int,
-    file_hash: str,
-    timeout: float = 10.0
-):
-    payload = {
-        "user_id": user_id,
-        "user_metadata": user_metadata,
-        "path": path,
-        "url": url,
-        "filename": filename,
-        "file_type": file_type,
-        "file_size_bytes": file_size_bytes,
-        "file_hash": file_hash
-    }
-    print(f"PostgreSQL payload file_hash: {payload['file_hash']}")
-    async with httpx.AsyncClient() as client:
-        try:
-            r = await client.post(
-                "https://mr-mvp-api-dev.dev.ingenspark.com/auth/UserMetadataCreate",
-                json=payload,
-                timeout=timeout
-            )
-            r.raise_for_status()
-            result = r.json()
-            result["file_hash"] = file_hash
-            print(f"PostgreSQL result file_hash: {result['file_hash']}")
-            return {"success": True, "data": result}
-        except httpx.HTTPStatusError as e:
-            return {
-                "success": False,
-                "error": "HTTP error",
-                "status_code": e.response.status_code,
-                "detail": e.response.text,
-                "file_hash": file_hash
-            }
-        except Exception as e:
-            return {
-                "success": False,
-                "error": "Request failed",
-                "detail": str(e),
-                "file_hash": file_hash
-            }
-
-
-# --- DEBUG ENDPOINT (Optional - for troubleshooting) ---
-@s3_bucket_router1.get("/debug/check_hash/{user_id}/{file_hash}")
-async def debug_check_hash(user_id: str, file_hash: str):
-    """Debug endpoint to test hash checking"""
-    result = await check_file_hash_exists(user_id, file_hash)
-    return {
-        "raw_result": result,
-        "exists_value": result.get("exists"),
-        "exists_type": type(result.get("exists")).__name__,
-        "success_value": result.get("success"),
-        "interpretation": "File exists" if result.get("exists") is True else "File does not exist or check failed"
-    }
-
-
-# --- MAIN ENDPOINT WITH FIXED HASH CHECK AND DATASET GRAPHS ---
-@s3_bucket_router1.post("/upload_datasets_v3/")
-async def upload_file(
-    file: UploadFile = File(...),
-    user_id: str = Query(..., description="User ID"),
-    path: str = Query("", description="Optional subpath")
-):
-    html_tmp_dir = None
-    bokeh_tmp_dir = None
-    file_content = None
-
-    try:
-        # 1. Validate extension
-        file_ext = os.path.splitext(file.filename)[1].lower()
-        if file_ext not in ALLOWED_EXTENSIONS:
-            raise HTTPException(status_code=400, detail=f"Unsupported file type: {file_ext}")
-
-        # 2. Read file
-        file_content = await file.read()
-        if not file_content:
-            raise HTTPException(status_code=400, detail="Empty file")
-        if len(file_content) > MAX_FILE_SIZE_BYTES:
-            raise HTTPException(status_code=413, detail="File exceeds 100 MiB limit")
-
-        # 3. Generate hash
-        file_hash = hashlib.sha256(file_content).hexdigest()
-        print(f"Generated file hash: {file_hash}")
-
-        # 4. Check hash via API (FIXED LOGIC)
-        hash_result = await check_file_hash_exists(user_id, file_hash)
-        
-        # Enhanced logging
-        print(f"Hash check result: {hash_result}")
-        
-        # Check if the API call was successful first
-        if not hash_result.get("success", False):
-            print(f"⚠️ Warning: Hash check API failed: {hash_result.get('message')}")
-            # You can decide to fail here if hash check is critical
-            # raise HTTPException(status_code=503, detail="Hash check service unavailable")
-        
-        # Check if file exists
-        if hash_result.get("exists") is True:
-            print(f"🚫 Duplicate file detected: {file_hash}")
-            return JSONResponse(
-                status_code=409,
-                content={
-                    "message": "File already uploaded.",
-                    "reason": "Duplicate file detected via SHA-256 hash.",
-                    "file_hash": file_hash,
-                    "user_id": user_id,
-                    "filename": file.filename,
-                    "action": "skipped",
-                    "existing_file_info": hash_result.get("data")
-                }
-            )
-        
-        print("✅ Hash check passed. New file - proceeding with upload.")
-
-        # 5. Load DataFrame
-        try:
-            if file_ext == ".csv":
-                df = pd.read_csv(io.BytesIO(file_content))
-            elif file_ext in {".xlsx", ".xls"}:
-                engine = 'openpyxl' if file_ext == ".xlsx" else 'xlrd'
-                df = pd.read_excel(io.BytesIO(file_content), engine=engine)
-            elif file_ext == ".ods":
-                df = pd.read_excel(io.BytesIO(file_content), engine='odf')
-        except Exception as e:
-            raise HTTPException(status_code=400, detail=f"Failed to parse file: {str(e)}")
-
-        if len(df) > MAX_ROWS_ALLOWED:
-            raise HTTPException(status_code=413, detail=f"Too many rows: {len(df):,} > {MAX_ROWS_ALLOWED:,}")
-
-        # 6. Convert to Parquet
-        parquet_buffer = convert_df_to_parquet(df)
-        parquet_size = parquet_buffer.getbuffer().nbytes
-
-        # 7. Upload Parquet to S3
-        base_filename = os.path.splitext(file.filename)[0]
-        parquet_filename = f"{base_filename}.parquet"
-        file_key = f"{user_id}/files/datasets/{parquet_filename}"
-        file_url = f"{ENDPOINT_URL}/{BUCKET_NAME}/{file_key}"
-
-        s3.upload_fileobj(parquet_buffer, BUCKET_NAME, file_key,
-                          ExtraArgs={'ContentType': 'application/octet-stream'})
-        print(f"Uploaded Parquet: {file_url}")
-
-        # 8. Generate metadata
-        metadata = create_file_metadata_from_df(df, parquet_filename, file_key)
-        metadata.update({
-            "user_id": user_id,
-            "s3_path": file_key,
-            "s3_url": file_url,
-            "source_file": file.filename,
-            "source_file_type": file_ext[1:],
-            "file_type": "parquet",
-            "original_file_size_bytes": len(file_content),
-            "parquet_file_size_bytes": parquet_size,
-            "compression_ratio": f"{(1 - parquet_size/len(file_content))*100:.1f}%",
-            "file_hash": file_hash
-        })
-
-        # 8.5 Generate dataset preview graphs (separate from metadata)
-        print("Generating dataset preview graphs...")
-        dataset_graphs = None
-        try:
-            dataset_graphs = create_data_set_graphs_dict(df, max_rows=200)
-            print(f"✅ Generated graphs for {len(dataset_graphs.get('columnSummaries', []))} columns")
-        except Exception as e:
-            print(f"⚠️ Failed to generate dataset graphs: {e}")
-            import traceback
-            traceback.print_exc()
-            dataset_graphs = {
-                "error": str(e),
-                "columnSummaries": []
-            }
-
-        safe_metadata = sanitize_for_json(metadata)
-        safe_dataset_graphs = sanitize_for_json(dataset_graphs) if dataset_graphs else {"columnSummaries": []}
-
-        # 9. Vector DB
-        check_vdb(user_id)
-        vdb_res = await add_metadata_only("sri_1_files_&_files_metadata", metadata)
-        vdb_success = vdb_res.get("status") == "success"
-        print(f"VDB upload success: {vdb_success}")
-
-        # 10. PostgreSQL Metadata + file_hash
-        pg_result = await user_metadata_upload_pg(
-            user_id=user_id,
-            user_metadata=json.dumps(safe_metadata),
-            path=file_key,
-            url=file_url,
-            filename=parquet_filename,
-            file_type="parquet",
-            file_size_bytes=parquet_size,
-            file_hash=file_hash
-        )
-        print(f"PostgreSQL upload result: {pg_result}")
-        pg_success = pg_result.get("success", False)
-        
-        graphs_count = len(safe_dataset_graphs.get("columnSummaries", []))
-        print(f"Graphs generated: {graphs_count}")
-
-        # 11. Return success
-        return {
-            "message": "Upload successful.",
-            "filename": parquet_filename,
-            "original_filename": file.filename,
-            "user_id": user_id,
-            "file_path": file_key,
-            "file_url": file_url,
-            "file_hash": file_hash,
-            "source_file_type": file_ext[1:],
-            "file_type": "parquet",
-            "original_file_size_bytes": len(file_content),
-            "parquet_file_size_bytes": parquet_size,
-            "compression_ratio": f"{(1 - parquet_size/len(file_content))*100:.1f}%",
-            "rows": len(df),
-            "columns": len(df.columns),
-            "metadata": safe_metadata,
-            "data_sets_preview_graph": safe_dataset_graphs,  # Separate key at root level
-            "upload_dataset_vdb": vdb_success,
-            "upload_dataset_pg": pg_success,
-            "pg_details": pg_result,
-            "graphs_generated": graphs_count
-        }
-
-    except HTTPException:
-        raise
-    except Exception as e:
-        print(f"Unexpected error: {e}")
-        import traceback
-        traceback.print_exc()
-        raise HTTPException(status_code=500, detail=str(e))
-    finally:
-        # Clean up temp directories
-        for d in (html_tmp_dir, bokeh_tmp_dir):
-            if d and os.path.exists(d):
+# from fastapi import FastAPI, UploadFile, File, HTTPException, Query, APIRouter
+# from fastapi.responses import JSONResponse
+# import pandas as pd
+# from autoviz.AutoViz_Class import AutoViz_Class
+# import io, os, boto3, tempfile, glob, matplotlib, json, hashlib, shutil
+# matplotlib.use('Agg')
+# import matplotlib.pyplot as plt
+# import sys
+# from pathlib import Path
+# from typing import List
+
+# # --- Project Root Setup ---
+# PROJECT_ROOT = Path(__file__).resolve().parents[1]
+# if str(PROJECT_ROOT) not in sys.path:
+#     sys.path.insert(0, str(PROJECT_ROOT))
+
+# from retrieve_secret import *
+# import httpx
+
+# # --- File Validation ---
+# MAX_FILE_SIZE_BYTES = 100 * 1024 * 1024  # 100 MiB
+# MAX_ROWS_ALLOWED = 1_000_000
+# ALLOWED_EXTENSIONS = {".csv", ".xlsx", ".xls", ".ods"}
+
+# # --- AWS S3 Config ---
+# print("AWS S3 config:", AWS_S3_CREDS_KEY_ID, AWS_S3_CREDS_SECRET_KEY, BUCKET_NAME)
+# ACCESS_KEY = AWS_S3_CREDS_KEY_ID
+# SECRET_KEY = AWS_S3_CREDS_SECRET_KEY
+# BUCKET_NAME = BUCKET_NAME
+# REGION_NAME = "us-east-1"
+
+# s3 = boto3.client(
+#     "s3",
+#     aws_access_key_id=ACCESS_KEY,
+#     aws_secret_access_key=SECRET_KEY,
+#     region_name=REGION_NAME
+# )
+
+# ENDPOINT_URL = f"https://s3.{REGION_NAME}.amazonaws.com"
+
+# # --- FastAPI Router ---
+# s3_bucket_router1 = APIRouter(prefix="/s3/v3", tags=["s3_v3"])
+
+
+# # --- Helper: S3 Key ---
+# def make_key(path: str, filename: str) -> str:
+#     return f"{path.strip('/')}/{filename}" if path else filename
+
+
+# # --- Sanitize for JSON ---
+# def sanitize_for_json(obj):
+#     if isinstance(obj, dict):
+#         return {k: sanitize_for_json(v) for k, v in obj.items()}
+#     elif isinstance(obj, list):
+#         return [sanitize_for_json(item) for item in obj]
+#     elif isinstance(obj, (pd.Timestamp, pd.DatetimeTZDtype)):
+#         return str(obj)
+#     elif pd.isna(obj):
+#         return None
+#     elif isinstance(obj, (int, float, str, bool, type(None))):
+#         return obj
+#     else:
+#         return str(obj)
+
+
+# # --- Read CSV from S3 (optional) ---
+# def read_csv_from_s3(key: str) -> pd.DataFrame:
+#     obj = s3.get_object(Bucket=BUCKET_NAME, Key=key)
+#     return pd.read_csv(io.BytesIO(obj['Body'].read()))
+
+
+# # --- Vector DB Placeholders ---
+# def check_vdb(user_id: str):
+#     print(f"Checking VDB for user: {user_id}")
+
+# async def add_metadata_only(collection_name: str, metadata: dict):
+#     print(f"Adding metadata to collection: {collection_name}")
+#     return {"status": "success", "collection": collection_name}
+
+
+# # --- AutoViz HTML ---
+# def run_autoviz_html(
+#     dfte, depVar="", sep=",", header=0, verbose=0, lowess=False,
+#     max_rows_analyzed=150000, max_cols_analyzed=30
+# ):
+#     tmp_dir = tempfile.mkdtemp()
+#     AV = AutoViz_Class()
+#     AV.AutoViz(
+#         filename="", sep=sep, depVar=depVar, dfte=dfte, header=header,
+#         verbose=verbose, lowess=lowess, chart_format="html",
+#         max_rows_analyzed=max_rows_analyzed, max_cols_analyzed=max_cols_analyzed,
+#         save_plot_dir=tmp_dir
+#     )
+#     print(f"HTML plots saved to: {tmp_dir}")
+#     return tmp_dir
+
+
+# # --- AutoViz Bokeh ---
+# def run_autoviz_bokeh(
+#     dfte, depVar="", sep=",", header=0, verbose=0, lowess=False,
+#     max_rows_analyzed=150000, max_cols_analyzed=30
+# ):
+#     tmp_dir = tempfile.mkdtemp()
+#     print(f"Bokeh temp directory: {tmp_dir}")
+#     try:
+#         matplotlib.use('Agg')
+#         plt.ioff()
+#         AV = AutoViz_Class()
+#         AV.AutoViz(
+#             filename="", sep=sep, depVar=depVar, dfte=dfte, header=header,
+#             verbose=verbose, lowess=lowess, chart_format="bokeh",
+#             max_rows_analyzed=max_rows_analyzed, max_cols_analyzed=max_cols_analyzed,
+#             save_plot_dir=tmp_dir
+#         )
+#         plt.close('all')
+#         print(f"Bokeh plots saved to: {tmp_dir}")
+#         return tmp_dir
+#     except Exception as e:
+#         print(f"Error in Bokeh: {e}")
+#         import traceback
+#         traceback.print_exc()
+#         raise
+
+
+# # --- Fallback Matplotlib ---
+# def generate_matplotlib_plots(df, user_id, base_filename):
+#     print("\nGenerating matplotlib fallback plots...")
+#     tmp_dir = tempfile.mkdtemp()
+#     plot_metadata = []
+#     try:
+#         matplotlib.use('Agg')
+#         numeric_cols = df.select_dtypes(include=['number']).columns.tolist()
+#         if len(numeric_cols) == 0:
+#             print("No numeric columns for plotting")
+#             return []
+
+#         # Correlation heatmap
+#         if len(numeric_cols) > 1:
+#             plt.figure(figsize=(10, 8))
+#             import seaborn as sns
+#             corr = df[numeric_cols].corr()
+#             sns.heatmap(corr, annot=True, cmap='coolwarm', center=0)
+#             plt.title('Correlation Heatmap')
+#             plt.tight_layout()
+#             png_path = os.path.join(tmp_dir, 'correlation_heatmap.png')
+#             plt.savefig(png_path, dpi=100, bbox_inches='tight')
+#             plt.close()
+
+#             s3_key = f"{user_id}/files/datasets/{base_filename}/pngs/correlation_heatmap.png"
+#             with open(png_path, 'rb') as f:
+#                 s3.put_object(Bucket=BUCKET_NAME, Key=s3_key, Body=f.read(), ContentType='image/png')
+#             plot_metadata.append({
+#                 "file_name": "correlation_heatmap.png",
+#                 "s3_path": s3_key,
+#                 "url": f"{ENDPOINT_URL}/{BUCKET_NAME}/{s3_key}",
+#                 "type": "png",
+#                 "plot_type": "heatmap"
+#             })
+
+#         # Histograms
+#         for col in numeric_cols[:5]:
+#             plt.figure(figsize=(10, 6))
+#             df[col].hist(bins=30, edgecolor='black')
+#             plt.title(f'Distribution of {col}')
+#             plt.xlabel(col)
+#             plt.ylabel('Frequency')
+#             plt.tight_layout()
+#             png_path = os.path.join(tmp_dir, f'distribution_{col}.png')
+#             plt.savefig(png_path, dpi=100, bbox_inches='tight')
+#             plt.close()
+
+#             s3_key = f"{user_id}/files/datasets/{base_filename}/pngs/distribution_{col}.png"
+#             with open(png_path, 'rb') as f:
+#                 s3.put_object(Bucket=BUCKET_NAME, Key=s3_key, Body=f.read(), ContentType='image/png')
+#             plot_metadata.append({
+#                 "file_name": f"distribution_{col}.png",
+#                 "s3_path": s3_key,
+#                 "url": f"{ENDPOINT_URL}/{BUCKET_NAME}/{s3_key}",
+#                 "type": "png",
+#                 "plot_type": "histogram"
+#             })
+#         return plot_metadata
+#     except Exception as e:
+#         print(f"Matplotlib failed: {e}")
+#         return []
+#     finally:
+#         shutil.rmtree(tmp_dir, ignore_errors=True)
+
+
+# # --- Upload Viz Files ---
+# def upload_viz_files_to_s3(tmp_dir, file_pattern, user_id, base_filename, viz_type):
+#     patterns = [file_pattern, f"*.{viz_type}", f"**/*.{viz_type}"]
+#     files = []
+#     for p in patterns:
+#         files.extend(glob.glob(os.path.join(tmp_dir, p), recursive=True))
+#     files = list(set(files))
+#     if not files:
+#         print(f"No {viz_type} files found")
+#         return []
+
+#     folder = "htmls" if viz_type == "html" else "svgs" if viz_type == "svg" else "pngs"
+#     metadata = []
+#     content_type_map = {
+#         'html': 'text/html', 'svg': 'image/svg+xml', 'png': 'image/png',
+#         'jpg': 'image/jpeg', 'jpeg': 'image/jpeg'
+#     }
+
+#     for file_path in files:
+#         name = os.path.basename(file_path)
+#         ext = os.path.splitext(name)[1][1:]
+#         s3_key = f"{user_id}/files/datasets/{base_filename}/{folder}/{name}"
+#         with open(file_path, "rb") as f:
+#             body = f.read()
+#         s3.put_object(
+#             Bucket=BUCKET_NAME, Key=s3_key, Body=body,
+#             ContentType=content_type_map.get(ext, 'application/octet-stream')
+#         )
+#         metadata.append({
+#             "file_name": name, "s3_path": s3_key,
+#             "url": f"{ENDPOINT_URL}/{BUCKET_NAME}/{s3_key}",
+#             "type": viz_type, "size": len(body)
+#         })
+#         print(f"{viz_type.upper()} uploaded: {s3_key}")
+#     return metadata
+
+
+# # --- Convert to Parquet ---
+# def convert_df_to_parquet(df: pd.DataFrame) -> io.BytesIO:
+#     buffer = io.BytesIO()
+#     df.to_parquet(buffer, engine='pyarrow', compression='snappy', index=False)
+#     buffer.seek(0)
+#     return buffer
+
+
+# # --- NEW: Check File Hash ---
+# async def check_file_hash_exists(user_id: str, file_hash: str) -> dict:
+#     url = f"https://mr-mvp-api-dev.dev.ingenspark.com/auth/UserMetadata/{user_id}/check_file_hash?file_hash={file_hash}"
+#     headers = {"accept": "application/json"}
+#     async with httpx.AsyncClient(timeout=10.0) as client:
+#         try:
+#             r = await client.get(url, headers=headers)
+#             r.raise_for_status()
+#             return r.json()
+#         except httpx.HTTPStatusError as e:
+#             return {"success": False, "message": f"HTTP {e.response.status_code}"}
+#         except Exception as e:
+#             return {"success": False, "message": f"Request failed: {str(e)}"}
+
+
+# # --- PostgreSQL Metadata Upload ---
+# async def user_metadata_upload_pg(
+#     user_id: str, user_metadata: str, path: str, url: str,
+#     filename: str, file_type: str, file_size_bytes: int, timeout: float = 10.0
+# ):
+#     payload = {
+#         "user_id": user_id, "user_metadata": user_metadata, "path": path,
+#         "url": url, "filename": filename, "file_type": file_type,
+#         "file_size_bytes": file_size_bytes
+#     }
+#     async with httpx.AsyncClient() as client:
+#         try:
+#             r = await client.post(
+#                 "https://mr-mvp-api-dev.dev.ingenspark.com/auth/UserMetadataCreate",
+#                 json=payload, timeout=timeout
+#             )
+#             r.raise_for_status()
+#             return {"success": True, "data": r.json()}
+#         except httpx.HTTPStatusError as e:
+#             return {"success": False, "error": "HTTP error", "status_code": e.response.status_code, "detail": e.response.text}
+#         except Exception as e:
+#             return {"success": False, "error": "Request failed", "detail": str(e)}
+
+
+# # --- MAIN ENDPOINT ---
+# @s3_bucket_router1.post("/upload_datasets_v3/")
+# async def upload_file(
+#     file: UploadFile = File(...),
+#     user_id: str = Query(..., description="User ID"),
+#     path: str = Query("", description="Optional subpath")
+# ):
+#     html_tmp_dir = bokeh_tmp_dir = None
+#     file_content = None
+
+#     try:
+#         # 1. Validate extension
+#         file_ext = os.path.splitext(file.filename)[1].lower()
+#         if file_ext not in ALLOWED_EXTENSIONS:
+#             raise HTTPException(status_code=400, detail=f"Unsupported: {file_ext}")
+
+#         # 2. Read file
+#         file_content = await file.read()
+#         if not file_content:
+#             raise HTTPException(status_code=400, detail="Empty file")
+#         if len(file_content) > MAX_FILE_SIZE_BYTES:
+#             raise HTTPException(status_code=413, detail=f"File > 100 MiB")
+
+#         # 3. Generate hash
+#         file_hash = hashlib.sha256(file_content).hexdigest()
+#         print(f"Generated hash: {file_hash}")
+
+#         # 4. Check hash via API
+#         hash_result = await check_file_hash_exists(user_id, file_hash)
+#         if not hash_result.get("success", False):
+#             return JSONResponse(
+#                 status_code=409,
+#                 content={
+#                     "message": "File already uploaded.",
+#                     "reason": hash_result.get("message", "Hash exists."),
+#                     "file_hash": file_hash,
+#                     "user_id": user_id,
+#                     "action": "skipped"
+#                 }
+#             )
+#         print("Hash check passed.")
+
+#         # 5. Load DataFrame
+#         try:
+#             if file_ext == ".csv":
+#                 df = pd.read_csv(io.BytesIO(file_content))
+#             elif file_ext in {".xlsx", ".xls"}:
+#                 engine = 'openpyxl' if file_ext == ".xlsx" else 'xlrd'
+#                 df = pd.read_excel(io.BytesIO(file_content), engine=engine)
+#             elif file_ext == ".ods":
+#                 df = pd.read_excel(io.BytesIO(file_content), engine='odf')
+#         except Exception as e:
+#             raise HTTPException(status_code=400, detail=f"Parse error: {e}")
+
+#         if len(df) > MAX_ROWS_ALLOWED:
+#             raise HTTPException(status_code=413, detail=f"Rows > {MAX_ROWS_ALLOWED:,}")
+
+#         # 6. Convert to Parquet
+#         parquet_buffer = convert_df_to_parquet(df)
+#         parquet_size = parquet_buffer.getbuffer().nbytes
+
+#         # 7. Upload to S3
+#         base_filename = os.path.splitext(file.filename)[0]
+#         parquet_filename = f"{base_filename}.parquet"
+#         file_key = f"{user_id}/files/datasets/{parquet_filename}"
+#         file_url = f"{ENDPOINT_URL}/{BUCKET_NAME}/{file_key}"
+
+#         s3.upload_fileobj(parquet_buffer, BUCKET_NAME, file_key,
+#                           ExtraArgs={'ContentType': 'application/octet-stream'})
+#         print(f"Uploaded: {file_url}")
+
+#         # 8. Metadata
+#         from s3.meta_data_creation_from_s3 import create_file_metadata_from_df
+#         metadata = create_file_metadata_from_df(df, parquet_filename, file_key)
+#         metadata.update({
+#             "user_id": user_id, "s3_path": file_key, "s3_url": file_url,
+#             "source_file": file.filename, "source_file_type": file_ext[1:],
+#             "file_type": "parquet", "original_file_size_bytes": len(file_content),
+#             "parquet_file_size_bytes": parquet_size,
+#             "compression_ratio": f"{(1 - parquet_size/len(file_content))*100:.1f}%",
+#             "file_hash": file_hash
+#         })
+#         print(f"Metadata: {metadata}")
+#         # 9. Vector DB
+#         check_vdb(user_id)
+#         vdb_res = await add_metadata_only("sri_1_files_&_files_metadata", metadata)
+#         vdb_success = vdb_res.get("status") == "success"
+#         #safe_metadata 
+#         safe_metadata = sanitize_for_json(metadata)
+#         # 10. PostgreSQL
+#         # pg_result = await user_metadata_upload_pg(
+#         #     user_id=user_id, user_metadata=json.dumps(metadata),
+#         #     path=file_key, url=file_url, filename=parquet_filename,
+#         #     file_type="parquet", file_size_bytes=parquet_size
+#         # )
+#         pg_result = await user_metadata_upload_pg(
+#         user_id=user_id, user_metadata=json.dumps(safe_metadata),
+#         path=file_key, url=file_url, filename=parquet_filename,
+#         file_type="parquet", file_size_bytes=parquet_size
+#     )
+#         # Inject hash into pg_result
+#         if isinstance(pg_result, dict):
+#             pg_result["file_hash"] = file_hash
+#         pg_success = pg_result.get("success", False)
+
+#         # 11. Return
+#         return {
+#             "message": "Upload successful.",
+#             "filename": parquet_filename,
+#             "original_filename": file.filename,
+#             "user_id": user_id,
+#             "file_path": file_key,
+#             "file_url": file_url,
+#             "file_hash": file_hash,
+#             "source_file_type": file_ext[1:],
+#             "file_type": "parquet",
+#             "original_file_size_bytes": len(file_content),
+#             "parquet_file_size_bytes": parquet_size,
+#             "rows": len(df), "columns": len(df.columns),
+#             # "metadata": sanitize_for_json(metadata),
+#             "metadata": safe_metadata,
+#             "upload_dataset_vdb": vdb_success,
+#             "upload_dataset_pg": pg_success,
+#             "pg_details": pg_result
+#         }
+     
+#     except HTTPException:
+#         raise
+#     except Exception as e:
+#         print(f"Error: {e}")
+#         import traceback; traceback.print_exc()
+#         raise HTTPException(status_code=500, detail=str(e))
+#     finally:
+#         for d in (html_tmp_dir, bokeh_tmp_dir):
+#             if d and os.path.exists(d):
+#                 shutil.rmtree(d, ignore_errors=True)
+
+
+
+from fastapi import FastAPI, UploadFile, File, HTTPException, Query, APIRouter
+from fastapi.responses import JSONResponse
+import pandas as pd
+from autoviz.AutoViz_Class import AutoViz_Class
+import io, os, boto3, tempfile, glob, matplotlib, json, hashlib, shutil
+matplotlib.use('Agg')
+import matplotlib.pyplot as plt
+import sys
+from pathlib import Path
+from typing import List
+import httpx
+
+# --- Project Root Setup ---
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from retrieve_secret import *
+from s3.meta_data_creation_from_s3 import create_file_metadata_from_df
+
+# --- File Validation ---
+MAX_FILE_SIZE_BYTES = 100 * 1024 * 1024  # 100 MiB
+MAX_ROWS_ALLOWED = 1_000_000
+ALLOWED_EXTENSIONS = {".csv", ".xlsx", ".xls", ".ods"}
+
+# --- AWS S3 Config ---
+print("AWS S3 config:", AWS_S3_CREDS_KEY_ID, AWS_S3_CREDS_SECRET_KEY, BUCKET_NAME)
+ACCESS_KEY = AWS_S3_CREDS_KEY_ID
+SECRET_KEY = AWS_S3_CREDS_SECRET_KEY
+BUCKET_NAME = BUCKET_NAME
+REGION_NAME = "us-east-1"
+
+s3 = boto3.client(
+    "s3",
+    aws_access_key_id=ACCESS_KEY,
+    aws_secret_access_key=SECRET_KEY,
+    region_name=REGION_NAME
+)
+
+ENDPOINT_URL = f"https://s3.{REGION_NAME}.amazonaws.com"
+
+# --- FastAPI Router ---
+s3_bucket_router1 = APIRouter(prefix="/s3/v3", tags=["s3_v3"])
+
+
+# --- Helper: S3 Key ---
+def make_key(path: str, filename: str) -> str:
+    return f"{path.strip('/')}/{filename}" if path else filename
+
+
+# --- Sanitize for JSON ---
+def sanitize_for_json(obj):
+    if isinstance(obj, dict):
+        return {k: sanitize_for_json(v) for k, v in obj.items()}
+    elif isinstance(obj, list):
+        return [sanitize_for_json(item) for item in obj]
+    elif isinstance(obj, (pd.Timestamp, pd.DatetimeTZDtype)):
+        return str(obj)
+    elif pd.isna(obj):
+        return None
+    elif isinstance(obj, (int, float, str, bool, type(None))):
+        return obj
+    else:
+        return str(obj)
+
+
+# --- Vector DB Placeholders ---
+def check_vdb(user_id: str):
+    print(f"Checking VDB for user: {user_id}")
+
+async def add_metadata_only(collection_name: str, metadata: dict):
+    print(f"Adding metadata to collection: {collection_name}")
+    return {"status": "success", "collection": collection_name}
+
+
+# --- AutoViz HTML ---
+def run_autoviz_html(
+    dfte, depVar="", sep=",", header=0, verbose=0, lowess=False,
+    max_rows_analyzed=150000, max_cols_analyzed=30
+):
+    tmp_dir = tempfile.mkdtemp()
+    AV = AutoViz_Class()
+    AV.AutoViz(
+        filename="", sep=sep, depVar=depVar, dfte=dfte, header=header,
+        verbose=verbose, lowess=lowess, chart_format="html",
+        max_rows_analyzed=max_rows_analyzed, max_cols_analyzed=max_cols_analyzed,
+        save_plot_dir=tmp_dir
+    )
+    print(f"HTML plots saved to: {tmp_dir}")
+    return tmp_dir
+
+
+# --- AutoViz Bokeh ---
+def run_autoviz_bokeh(
+    dfte, depVar="", sep=",", header=0, verbose=0, lowess=False,
+    max_rows_analyzed=150000, max_cols_analyzed=30
+):
+    tmp_dir = tempfile.mkdtemp()
+    print(f"Bokeh temp directory: {tmp_dir}")
+    try:
+        matplotlib.use('Agg')
+        plt.ioff()
+        AV = AutoViz_Class()
+        AV.AutoViz(
+            filename="", sep=sep, depVar=depVar, dfte=dfte, header=header,
+            verbose=verbose, lowess=lowess, chart_format="bokeh",
+            max_rows_analyzed=max_rows_analyzed, max_cols_analyzed=max_cols_analyzed,
+            save_plot_dir=tmp_dir
+        )
+        plt.close('all')
+        print(f"Bokeh plots saved to: {tmp_dir}")
+        return tmp_dir
+    except Exception as e:
+        print(f"Error in Bokeh: {e}")
+        import traceback
+        traceback.print_exc()
+        raise
+
+
+# --- Fallback Matplotlib ---
+def generate_matplotlib_plots(df, user_id, base_filename):
+    print("\nGenerating matplotlib fallback plots...")
+    tmp_dir = tempfile.mkdtemp()
+    plot_metadata = []
+    try:
+        matplotlib.use('Agg')
+        numeric_cols = df.select_dtypes(include=['number']).columns.tolist()
+        if len(numeric_cols) == 0:
+            print("No numeric columns for plotting")
+            return []
+
+        # Correlation heatmap
+        if len(numeric_cols) > 1:
+            plt.figure(figsize=(10, 8))
+            import seaborn as sns
+            corr = df[numeric_cols].corr()
+            sns.heatmap(corr, annot=True, cmap='coolwarm', center=0)
+            plt.title('Correlation Heatmap')
+            plt.tight_layout()
+            png_path = os.path.join(tmp_dir, 'correlation_heatmap.png')
+            plt.savefig(png_path, dpi=100, bbox_inches='tight')
+            plt.close()
+
+            s3_key = f"{user_id}/files/datasets/{base_filename}/pngs/correlation_heatmap.png"
+            with open(png_path, 'rb') as f:
+                s3.put_object(Bucket=BUCKET_NAME, Key=s3_key, Body=f.read(), ContentType='image/png')
+            plot_metadata.append({
+                "file_name": "correlation_heatmap.png",
+                "s3_path": s3_key,
+                "url": f"{ENDPOINT_URL}/{BUCKET_NAME}/{s3_key}",
+                "type": "png",
+                "plot_type": "heatmap"
+            })
+
+        # Histograms
+        for col in numeric_cols[:5]:
+            plt.figure(figsize=(10, 6))
+            df[col].hist(bins=30, edgecolor='black')
+            plt.title(f'Distribution of {col}')
+            plt.xlabel(col)
+            plt.ylabel('Frequency')
+            plt.tight_layout()
+            png_path = os.path.join(tmp_dir, f'distribution_{col}.png')
+            plt.savefig(png_path, dpi=100, bbox_inches='tight')
+            plt.close()
+
+            s3_key = f"{user_id}/files/datasets/{base_filename}/pngs/distribution_{col}.png"
+            with open(png_path, 'rb') as f:
+                s3.put_object(Bucket=BUCKET_NAME, Key=s3_key, Body=f.read(), ContentType='image/png')
+            plot_metadata.append({
+                "file_name": f"distribution_{col}.png",
+                "s3_path": s3_key,
+                "url": f"{ENDPOINT_URL}/{BUCKET_NAME}/{s3_key}",
+                "type": "png",
+                "plot_type": "histogram"
+            })
+        return plot_metadata
+    except Exception as e:
+        print(f"Matplotlib failed: {e}")
+        return []
+    finally:
+        shutil.rmtree(tmp_dir, ignore_errors=True)
+
+
+# --- Upload Viz Files ---
+def upload_viz_files_to_s3(tmp_dir, file_pattern, user_id, base_filename, viz_type):
+    patterns = [file_pattern, f"*.{viz_type}", f"**/*.{viz_type}"]
+    files = []
+    for p in patterns:
+        files.extend(glob.glob(os.path.join(tmp_dir, p), recursive=True))
+    files = list(set(files))
+    if not files:
+        print(f"No {viz_type} files found")
+        return []
+
+    folder = "htmls" if viz_type == "html" else "svgs" if viz_type == "svg" else "pngs"
+    metadata = []
+    content_type_map = {
+        'html': 'text/html', 'svg': 'image/svg+xml', 'png': 'image/png',
+        'jpg': 'image/jpeg', 'jpeg': 'image/jpeg'
+    }
+
+    for file_path in files:
+        name = os.path.basename(file_path)
+        ext = os.path.splitext(name)[1][1:]
+        s3_key = f"{user_id}/files/datasets/{base_filename}/{folder}/{name}"
+        with open(file_path, "rb") as f:
+            body = f.read()
+        s3.put_object(
+            Bucket=BUCKET_NAME, Key=s3_key, Body=body,
+            ContentType=content_type_map.get(ext, 'application/octet-stream')
+        )
+        metadata.append({
+            "file_name": name, "s3_path": s3_key,
+            "url": f"{ENDPOINT_URL}/{BUCKET_NAME}/{s3_key}",
+            "type": viz_type, "size": len(body)
+        })
+        print(f"{viz_type.upper()} uploaded: {s3_key}")
+    return metadata
+
+
+# --- Convert to Parquet ---
+def convert_df_to_parquet(df: pd.DataFrame) -> io.BytesIO:
+    buffer = io.BytesIO()
+    df.to_parquet(buffer, engine='pyarrow', compression='snappy', index=False)
+    buffer.seek(0)
+    return buffer
+
+
+# --- Check File Hash Exists ---
+async def check_file_hash_exists(user_id: str, file_hash: str) -> dict:
+    url = f"https://mr-mvp-api-dev.dev.ingenspark.com/auth/UserMetadata/{user_id}/check_file_hash?file_hash={file_hash}"
+    headers = {"accept": "application/json"}
+    async with httpx.AsyncClient(timeout=10.0) as client:
+        try:
+            r = await client.get(url, headers=headers)
+            r.raise_for_status()
+            return r.json()
+        except httpx.HTTPStatusError as e:
+            return {"success": False, "message": f"HTTP {e.response.status_code}"}
+        except Exception as e:
+            return {"success": False, "message": f"Request failed: {str(e)}"}
+
+
+# --- FIXED: PostgreSQL Metadata Upload with file_hash ---
+async def user_metadata_upload_pg(
+    user_id: str,
+    user_metadata: str,
+    path: str,
+    url: str,
+    filename: str,
+    file_type: str,
+    file_size_bytes: int,
+    file_hash: str,           # Now accepted
+    timeout: float = 10.0
+):
+    payload = {
+        "user_id": user_id,
+        "user_metadata": user_metadata,
+        "path": path,
+        "url": url,
+        "filename": filename,
+        "file_type": file_type,
+        "file_size_bytes": file_size_bytes,
+        "file_hash": file_hash       # This goes into DB now!
+    }
+    print("payload[file_hash]", payload["file_hash"])
+    async with httpx.AsyncClient() as client:
+        
+        try:
+            
+            r = await client.post(
+                "https://mr-mvp-api-dev.dev.ingenspark.com/auth/UserMetadataCreate",
+                json=payload,
+                timeout=timeout
+            )
+            r.raise_for_status()
+            result = r.json()
+            result["file_hash"] = file_hash
+            print("result[file_hash]", result["file_hash"])
+            return {"success": True, "data": result}
+        except httpx.HTTPStatusError as e:
+            return {
+                "success": False,
+                "error": "HTTP error",
+                "status_code": e.response.status_code,
+                "detail": e.response.text,
+                "file_hash": file_hash
+            }
+        except Exception as e:
+            return {
+                "success": False,
+                "error": "Request failed",
+                "detail": str(e),
+                "file_hash": file_hash
+            }
+
+
+# --- MAIN ENDPOINT: FULLY FIXED ---
+@s3_bucket_router1.post("/upload_datasets_v3/")
+async def upload_file(
+    file: UploadFile = File(...),
+    user_id: str = Query(..., description="User ID"),
+    path: str = Query("", description="Optional subpath")
+):
+    html_tmp_dir = None
+    bokeh_tmp_dir = None
+    file_content = None
+
+    try:
+        # 1. Validate extension
+        file_ext = os.path.splitext(file.filename)[1].lower()
+        if file_ext not in ALLOWED_EXTENSIONS:
+            raise HTTPException(status_code=400, detail=f"Unsupported file type: {file_ext}")
+
+        # 2. Read file
+        file_content = await file.read()
+        if not file_content:
+            raise HTTPException(status_code=400, detail="Empty file")
+        if len(file_content) > MAX_FILE_SIZE_BYTES:
+            raise HTTPException(status_code=413, detail="File exceeds 100 MiB limit")
+
+        # 3. Generate hash
+        file_hash = hashlib.sha256(file_content).hexdigest()
+        print(f"Generated hash: {file_hash}")
+
+        # 4. Check hash via API
+        hash_result = await check_file_hash_exists(user_id, file_hash)
+        if hash_result.get("exists") is True:
+            return JSONResponse(
+                status_code=409,
+                content={
+                    "message": "File already uploaded.",
+                    "reason": "Duplicate file detected via SHA-256 hash.",
+                    "file_hash": file_hash,
+                    "user_id": user_id,
+                    "action": "skipped"
+                }
+            )
+        print("Hash check passed. New file.")
+
+        # 5. Load DataFrame
+        try:
+            if file_ext == ".csv":
+                df = pd.read_csv(io.BytesIO(file_content))
+            elif file_ext in {".xlsx", ".xls"}:
+                engine = 'openpyxl' if file_ext == ".xlsx" else 'xlrd'
+                df = pd.read_excel(io.BytesIO(file_content), engine=engine)
+            elif file_ext == ".ods":
+                df = pd.read_excel(io.BytesIO(file_content), engine='odf')
+        except Exception as e:
+            raise HTTPException(status_code=400, detail=f"Failed to parse file: {str(e)}")
+
+        if len(df) > MAX_ROWS_ALLOWED:
+            raise HTTPException(status_code=413, detail=f"Too many rows: {len(df):,} > {MAX_ROWS_ALLOWED:,}")
+
+        # 6. Convert to Parquet
+        parquet_buffer = convert_df_to_parquet(df)
+        parquet_size = parquet_buffer.getbuffer().nbytes
+
+        # 7. Upload Parquet to S3
+        base_filename = os.path.splitext(file.filename)[0]
+        parquet_filename = f"{base_filename}.parquet"
+        file_key = f"{user_id}/files/datasets/{parquet_filename}"
+        file_url = f"{ENDPOINT_URL}/{BUCKET_NAME}/{file_key}"
+
+        s3.upload_fileobj(parquet_buffer, BUCKET_NAME, file_key,
+                          ExtraArgs={'ContentType': 'application/octet-stream'})
+        print(f"Uploaded Parquet: {file_url}")
+
+        # 8. Generate metadata
+        metadata = create_file_metadata_from_df(df, parquet_filename, file_key)
+        metadata.update({
+            "user_id": user_id,
+            "s3_path": file_key,
+            "s3_url": file_url,
+            "source_file": file.filename,
+            "source_file_type": file_ext[1:],
+            "file_type": "parquet",
+            "original_file_size_bytes": len(file_content),
+            "parquet_file_size_bytes": parquet_size,
+            "compression_ratio": f"{(1 - parquet_size/len(file_content))*100:.1f}%",
+            "file_hash": file_hash
+        })
+
+        safe_metadata = sanitize_for_json(metadata)
+
+        # 9. Vector DB
+        check_vdb(user_id)
+        vdb_res = await add_metadata_only("sri_1_files_&_files_metadata", metadata)
+        vdb_success = vdb_res.get("status") == "success"
+        print(f"vdb_success: {vdb_success}")
+
+        # 10. PostgreSQL Metadata + file_hash (FIXED!)
+        pg_result = await user_metadata_upload_pg(
+            user_id=user_id,
+            user_metadata=json.dumps(safe_metadata),
+            path=file_key,
+            url=file_url,
+            filename=parquet_filename,
+            file_type="parquet",
+            file_size_bytes=parquet_size,
+            file_hash=file_hash          # This was missing before!
+        )
+        print(f"pg_result: {pg_result}")
+        pg_success = pg_result.get("success", False)
+
+        # 11. Return success
+        return {
+            "message": "Upload successful.",
+            "filename": parquet_filename,
+            "original_filename": file.filename,
+            "user_id": user_id,
+            "file_path": file_key,
+            "file_url": file_url,
+            "file_hash": file_hash,
+            "source_file_type": file_ext[1:],
+            "file_type": "parquet",
+            "original_file_size_bytes": len(file_content),
+            "parquet_file_size_bytes": parquet_size,
+            "compression_ratio": f"{(1 - parquet_size/len(file_content))*100:.1f}%",
+            "rows": len(df),
+            "columns": len(df.columns),
+            "metadata": safe_metadata,
+            "upload_dataset_vdb": vdb_success,
+            "upload_dataset_pg": pg_success,
+            "pg_details": pg_result
+        }
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        print(f"Unexpected error: {e}")
+        import traceback
+        traceback.print_exc()
+        raise HTTPException(status_code=500, detail=str(e))
+    finally:
+        # Clean up temp directories even on error
+        for d in (html_tmp_dir, bokeh_tmp_dir):
+            if d and os.path.exists(d):
                 shutil.rmtree(d, ignore_errors=True)

s3/r5.py

@@ -0,0 +1,457 @@
+from fastapi import FastAPI, UploadFile, File, HTTPException, Query, APIRouter
+from fastapi.responses import JSONResponse
+import pandas as pd
+from autoviz.AutoViz_Class import AutoViz_Class
+import io, os, boto3, tempfile, glob, matplotlib, json, hashlib, shutil
+matplotlib.use('Agg')
+import matplotlib.pyplot as plt
+import sys
+from pathlib import Path
+from typing import List
+import httpx
+
+# --- Project Root Setup ---
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from retrieve_secret import *
+from s3.meta_data_creation_from_s3 import create_file_metadata_from_df
+from s3.create_dataset_graphs import create_data_set_graphs_dict
+
+# --- File Validation ---
+MAX_FILE_SIZE_BYTES = 100 * 1024 * 1024  # 100 MiB
+MAX_ROWS_ALLOWED = 1_000_000
+ALLOWED_EXTENSIONS = {".csv", ".xlsx", ".xls", ".ods"}
+
+# --- AWS S3 Config ---
+print("AWS S3 config:", AWS_S3_CREDS_KEY_ID, AWS_S3_CREDS_SECRET_KEY, BUCKET_NAME)
+ACCESS_KEY = AWS_S3_CREDS_KEY_ID
+SECRET_KEY = AWS_S3_CREDS_SECRET_KEY
+BUCKET_NAME = BUCKET_NAME
+REGION_NAME = "us-east-1"
+
+s3 = boto3.client(
+    "s3",
+    aws_access_key_id=ACCESS_KEY,
+    aws_secret_access_key=SECRET_KEY,
+    region_name=REGION_NAME
+)
+
+ENDPOINT_URL = f"https://s3.{REGION_NAME}.amazonaws.com"
+
+# --- FastAPI Router ---
+s3_bucket_router1 = APIRouter(prefix="/s3/v3", tags=["s3_v3"])
+
+
+# --- Helper: S3 Key ---
+def make_key(path: str, filename: str) -> str:
+    return f"{path.strip('/')}/{filename}" if path else filename
+
+
+# --- Sanitize for JSON ---
+def sanitize_for_json(obj):
+    if isinstance(obj, dict):
+        return {k: sanitize_for_json(v) for k, v in obj.items()}
+    elif isinstance(obj, list):
+        return [sanitize_for_json(item) for item in obj]
+    elif isinstance(obj, (pd.Timestamp, pd.DatetimeTZDtype)):
+        return str(obj)
+    elif pd.isna(obj):
+        return None
+    elif isinstance(obj, (int, float, str, bool, type(None))):
+        return obj
+    else:
+        return str(obj)
+
+
+# --- Vector DB Placeholders ---
+def check_vdb(user_id: str):
+    print(f"Checking VDB for user: {user_id}")
+
+async def add_metadata_only(collection_name: str, metadata: dict):
+    print(f"Adding metadata to collection: {collection_name}")
+    return {"status": "success", "collection": collection_name}
+
+
+# --- AutoViz HTML ---
+def run_autoviz_html(
+    dfte, depVar="", sep=",", header=0, verbose=0, lowess=False,
+    max_rows_analyzed=150000, max_cols_analyzed=30
+):
+    tmp_dir = tempfile.mkdtemp()
+    AV = AutoViz_Class()
+    AV.AutoViz(
+        filename="", sep=sep, depVar=depVar, dfte=dfte, header=header,
+        verbose=verbose, lowess=lowess, chart_format="html",
+        max_rows_analyzed=max_rows_analyzed, max_cols_analyzed=max_cols_analyzed,
+        save_plot_dir=tmp_dir
+    )
+    print(f"HTML plots saved to: {tmp_dir}")
+    return tmp_dir
+
+
+# --- AutoViz Bokeh ---
+def run_autoviz_bokeh(
+    dfte, depVar="", sep=",", header=0, verbose=0, lowess=False,
+    max_rows_analyzed=150000, max_cols_analyzed=30
+):
+    tmp_dir = tempfile.mkdtemp()
+    print(f"Bokeh temp directory: {tmp_dir}")
+    try:
+        matplotlib.use('Agg')
+        plt.ioff()
+        AV = AutoViz_Class()
+        AV.AutoViz(
+            filename="", sep=sep, depVar=depVar, dfte=dfte, header=header,
+            verbose=verbose, lowess=lowess, chart_format="bokeh",
+            max_rows_analyzed=max_rows_analyzed, max_cols_analyzed=max_cols_analyzed,
+            save_plot_dir=tmp_dir
+        )
+        plt.close('all')
+        print(f"Bokeh plots saved to: {tmp_dir}")
+        return tmp_dir
+    except Exception as e:
+        print(f"Error in Bokeh: {e}")
+        import traceback
+        traceback.print_exc()
+        raise
+
+
+# --- Fallback Matplotlib ---
+def generate_matplotlib_plots(df, user_id, base_filename):
+    print("\nGenerating matplotlib fallback plots...")
+    tmp_dir = tempfile.mkdtemp()
+    plot_metadata = []
+    try:
+        matplotlib.use('Agg')
+        numeric_cols = df.select_dtypes(include=['number']).columns.tolist()
+        if len(numeric_cols) == 0:
+            print("No numeric columns for plotting")
+            return []
+
+        # Correlation heatmap
+        if len(numeric_cols) > 1:
+            plt.figure(figsize=(10, 8))
+            import seaborn as sns
+            corr = df[numeric_cols].corr()
+            sns.heatmap(corr, annot=True, cmap='coolwarm', center=0)
+            plt.title('Correlation Heatmap')
+            plt.tight_layout()
+            png_path = os.path.join(tmp_dir, 'correlation_heatmap.png')
+            plt.savefig(png_path, dpi=100, bbox_inches='tight')
+            plt.close()
+
+            s3_key = f"{user_id}/files/datasets/{base_filename}/pngs/correlation_heatmap.png"
+            with open(png_path, 'rb') as f:
+                s3.put_object(Bucket=BUCKET_NAME, Key=s3_key, Body=f.read(), ContentType='image/png')
+            plot_metadata.append({
+                "file_name": "correlation_heatmap.png",
+                "s3_path": s3_key,
+                "url": f"{ENDPOINT_URL}/{BUCKET_NAME}/{s3_key}",
+                "type": "png",
+                "plot_type": "heatmap"
+            })
+
+        # Histograms
+        for col in numeric_cols[:5]:
+            plt.figure(figsize=(10, 6))
+            df[col].hist(bins=30, edgecolor='black')
+            plt.title(f'Distribution of {col}')
+            plt.xlabel(col)
+            plt.ylabel('Frequency')
+            plt.tight_layout()
+            png_path = os.path.join(tmp_dir, f'distribution_{col}.png')
+            plt.savefig(png_path, dpi=100, bbox_inches='tight')
+            plt.close()
+
+            s3_key = f"{user_id}/files/datasets/{base_filename}/pngs/distribution_{col}.png"
+            with open(png_path, 'rb') as f:
+                s3.put_object(Bucket=BUCKET_NAME, Key=s3_key, Body=f.read(), ContentType='image/png')
+            plot_metadata.append({
+                "file_name": f"distribution_{col}.png",
+                "s3_path": s3_key,
+                "url": f"{ENDPOINT_URL}/{BUCKET_NAME}/{s3_key}",
+                "type": "png",
+                "plot_type": "histogram"
+            })
+        return plot_metadata
+    except Exception as e:
+        print(f"Matplotlib failed: {e}")
+        return []
+    finally:
+        shutil.rmtree(tmp_dir, ignore_errors=True)
+
+
+# --- Upload Viz Files ---
+def upload_viz_files_to_s3(tmp_dir, file_pattern, user_id, base_filename, viz_type):
+    patterns = [file_pattern, f"*.{viz_type}", f"**/*.{viz_type}"]
+    files = []
+    for p in patterns:
+        files.extend(glob.glob(os.path.join(tmp_dir, p), recursive=True))
+    files = list(set(files))
+    if not files:
+        print(f"No {viz_type} files found")
+        return []
+
+    folder = "htmls" if viz_type == "html" else "svgs" if viz_type == "svg" else "pngs"
+    metadata = []
+    content_type_map = {
+        'html': 'text/html', 'svg': 'image/svg+xml', 'png': 'image/png',
+        'jpg': 'image/jpeg', 'jpeg': 'image/jpeg'
+    }
+
+    for file_path in files:
+        name = os.path.basename(file_path)
+        ext = os.path.splitext(name)[1][1:]
+        s3_key = f"{user_id}/files/datasets/{base_filename}/{folder}/{name}"
+        with open(file_path, "rb") as f:
+            body = f.read()
+        s3.put_object(
+            Bucket=BUCKET_NAME, Key=s3_key, Body=body,
+            ContentType=content_type_map.get(ext, 'application/octet-stream')
+        )
+        metadata.append({
+            "file_name": name, "s3_path": s3_key,
+            "url": f"{ENDPOINT_URL}/{BUCKET_NAME}/{s3_key}",
+            "type": viz_type, "size": len(body)
+        })
+        print(f"{viz_type.upper()} uploaded: {s3_key}")
+    return metadata
+
+
+# --- Convert to Parquet ---
+def convert_df_to_parquet(df: pd.DataFrame) -> io.BytesIO:
+    buffer = io.BytesIO()
+    df.to_parquet(buffer, engine='pyarrow', compression='snappy', index=False)
+    buffer.seek(0)
+    return buffer
+
+
+# --- Check File Hash Exists ---
+async def check_file_hash_exists(user_id: str, file_hash: str) -> dict:
+    url = f"https://mr-mvp-api-dev.dev.ingenspark.com/auth/UserMetadata/{user_id}/check_file_hash?file_hash={file_hash}"
+    headers = {"accept": "application/json"}
+    async with httpx.AsyncClient(timeout=10.0) as client:
+        try:
+            r = await client.get(url, headers=headers)
+            r.raise_for_status()
+            return r.json()
+        except httpx.HTTPStatusError as e:
+            return {"success": False, "message": f"HTTP {e.response.status_code}"}
+        except Exception as e:
+            return {"success": False, "message": f"Request failed: {str(e)}"}
+
+
+# --- PostgreSQL Metadata Upload with file_hash ---
+async def user_metadata_upload_pg(
+    user_id: str,
+    user_metadata: str,
+    path: str,
+    url: str,
+    filename: str,
+    file_type: str,
+    file_size_bytes: int,
+    file_hash: str,
+    timeout: float = 10.0
+):
+    payload = {
+        "user_id": user_id,
+        "user_metadata": user_metadata,
+        "path": path,
+        "url": url,
+        "filename": filename,
+        "file_type": file_type,
+        "file_size_bytes": file_size_bytes,
+        "file_hash": file_hash
+    }
+    print("payload[file_hash]", payload["file_hash"])
+    async with httpx.AsyncClient() as client:
+        try:
+            r = await client.post(
+                "https://mr-mvp-api-dev.dev.ingenspark.com/auth/UserMetadataCreate",
+                json=payload,
+                timeout=timeout
+            )
+            r.raise_for_status()
+            result = r.json()
+            result["file_hash"] = file_hash
+            print("result[file_hash]", result["file_hash"])
+            return {"success": True, "data": result}
+        except httpx.HTTPStatusError as e:
+            return {
+                "success": False,
+                "error": "HTTP error",
+                "status_code": e.response.status_code,
+                "detail": e.response.text,
+                "file_hash": file_hash
+            }
+        except Exception as e:
+            return {
+                "success": False,
+                "error": "Request failed",
+                "detail": str(e),
+                "file_hash": file_hash
+            }
+
+
+# --- MAIN ENDPOINT WITH DATASET GRAPHS ---
+@s3_bucket_router1.post("/upload_datasets_v3/")
+async def upload_file(
+    file: UploadFile = File(...),
+    user_id: str = Query(..., description="User ID"),
+    path: str = Query("", description="Optional subpath")
+):
+    html_tmp_dir = None
+    bokeh_tmp_dir = None
+    file_content = None
+
+    try:
+        # 1. Validate extension
+        file_ext = os.path.splitext(file.filename)[1].lower()
+        if file_ext not in ALLOWED_EXTENSIONS:
+            raise HTTPException(status_code=400, detail=f"Unsupported file type: {file_ext}")
+
+        # 2. Read file
+        file_content = await file.read()
+        if not file_content:
+            raise HTTPException(status_code=400, detail="Empty file")
+        if len(file_content) > MAX_FILE_SIZE_BYTES:
+            raise HTTPException(status_code=413, detail="File exceeds 100 MiB limit")
+
+        # 3. Generate hash
+        file_hash = hashlib.sha256(file_content).hexdigest()
+        print(f"Generated hash: {file_hash}")
+
+        # 4. Check hash via API
+        hash_result = await check_file_hash_exists(user_id, file_hash)
+        if hash_result.get("exists") is True:
+            return JSONResponse(
+                status_code=409,
+                content={
+                    "message": "File already uploaded.",
+                    "reason": "Duplicate file detected via SHA-256 hash.",
+                    "file_hash": file_hash,
+                    "user_id": user_id,
+                    "action": "skipped"
+                }
+            )
+        print("Hash check passed. New file.")
+
+        # 5. Load DataFrame
+        try:
+            if file_ext == ".csv":
+                df = pd.read_csv(io.BytesIO(file_content))
+            elif file_ext in {".xlsx", ".xls"}:
+                engine = 'openpyxl' if file_ext == ".xlsx" else 'xlrd'
+                df = pd.read_excel(io.BytesIO(file_content), engine=engine)
+            elif file_ext == ".ods":
+                df = pd.read_excel(io.BytesIO(file_content), engine='odf')
+        except Exception as e:
+            raise HTTPException(status_code=400, detail=f"Failed to parse file: {str(e)}")
+
+        if len(df) > MAX_ROWS_ALLOWED:
+            raise HTTPException(status_code=413, detail=f"Too many rows: {len(df):,} > {MAX_ROWS_ALLOWED:,}")
+
+        # 6. Convert to Parquet
+        parquet_buffer = convert_df_to_parquet(df)
+        parquet_size = parquet_buffer.getbuffer().nbytes
+
+        # 7. Upload Parquet to S3
+        base_filename = os.path.splitext(file.filename)[0]
+        parquet_filename = f"{base_filename}.parquet"
+        file_key = f"{user_id}/files/datasets/{parquet_filename}"
+        file_url = f"{ENDPOINT_URL}/{BUCKET_NAME}/{file_key}"
+
+        s3.upload_fileobj(parquet_buffer, BUCKET_NAME, file_key,
+                          ExtraArgs={'ContentType': 'application/octet-stream'})
+        print(f"Uploaded Parquet: {file_url}")
+
+        # 8. Generate metadata
+        metadata = create_file_metadata_from_df(df, parquet_filename, file_key)
+        metadata.update({
+            "user_id": user_id,
+            "s3_path": file_key,
+            "s3_url": file_url,
+            "source_file": file.filename,
+            "source_file_type": file_ext[1:],
+            "file_type": "parquet",
+            "original_file_size_bytes": len(file_content),
+            "parquet_file_size_bytes": parquet_size,
+            "compression_ratio": f"{(1 - parquet_size/len(file_content))*100:.1f}%",
+            "file_hash": file_hash
+        })
+
+        # 🆕 8.5 Generate dataset preview graphs
+        print("Generating dataset preview graphs...")
+        try:
+            dataset_graphs = create_data_set_graphs_dict(df, max_rows=200)
+            metadata["data_sets_preview_graph"] = dataset_graphs
+            print(f"✅ Generated graphs for {len(dataset_graphs.get('columnSummaries', []))} columns")
+        except Exception as e:
+            print(f"⚠️ Failed to generate dataset graphs: {e}")
+            import traceback
+            traceback.print_exc()
+            metadata["data_sets_preview_graph"] = {
+                "error": str(e),
+                "columnSummaries": []
+            }
+
+        safe_metadata = sanitize_for_json(metadata)
+
+        # 9. Vector DB
+        check_vdb(user_id)
+        vdb_res = await add_metadata_only("sri_1_files_&_files_metadata", metadata)
+        vdb_success = vdb_res.get("status") == "success"
+        print(f"vdb_success: {vdb_success}")
+
+        # 10. PostgreSQL Metadata + file_hash
+        pg_result = await user_metadata_upload_pg(
+            user_id=user_id,
+            user_metadata=json.dumps(safe_metadata),
+            path=file_key,
+            url=file_url,
+            filename=parquet_filename,
+            file_type="parquet",
+            file_size_bytes=parquet_size,
+            file_hash=file_hash
+        )
+        print(f"pg_result: {pg_result}")
+        pg_success = pg_result.get("success", False)
+        print("graphs_generated", len(safe_metadata.get("data_sets_preview_graph", {}).get("columnSummaries", [])))
+
+        # 11. Return success
+        return {
+            "message": "Upload successful.",
+            "filename": parquet_filename,
+            "original_filename": file.filename,
+            "user_id": user_id,
+            "file_path": file_key,
+            "file_url": file_url,
+            "file_hash": file_hash,
+            "source_file_type": file_ext[1:],
+            "file_type": "parquet",
+            "original_file_size_bytes": len(file_content),
+            "parquet_file_size_bytes": parquet_size,
+            "compression_ratio": f"{(1 - parquet_size/len(file_content))*100:.1f}%",
+            "rows": len(df),
+            "columns": len(df.columns),
+            "metadata": safe_metadata,
+            "upload_dataset_vdb": vdb_success,
+            "upload_dataset_pg": pg_success,
+            "pg_details": pg_result,
+            "graphs_generated": len(safe_metadata.get("data_sets_preview_graph", {}).get("columnSummaries", []))
+        }
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        print(f"Unexpected error: {e}")
+        import traceback
+        traceback.print_exc()
+        raise HTTPException(status_code=500, detail=str(e))
+    finally:
+        # Clean up temp directories
+        for d in (html_tmp_dir, bokeh_tmp_dir):
+            if d and os.path.exists(d):
+                shutil.rmtree(d, ignore_errors=True)

s3/r6.py

@@ -0,0 +1,1065 @@
+# from fastapi import FastAPI, UploadFile, File, HTTPException, Query, APIRouter
+# from fastapi.responses import JSONResponse
+# import pandas as pd
+# from autoviz.AutoViz_Class import AutoViz_Class
+# import io, os, boto3, tempfile, glob, matplotlib, json, hashlib, shutil
+# matplotlib.use('Agg')
+# import matplotlib.pyplot as plt
+# import sys
+# from pathlib import Path
+# from typing import List
+# import httpx
+
+# # --- Project Root Setup ---
+# PROJECT_ROOT = Path(__file__).resolve().parents[1]
+# if str(PROJECT_ROOT) not in sys.path:
+#     sys.path.insert(0, str(PROJECT_ROOT))
+
+# from retrieve_secret import *
+# from s3.meta_data_creation_from_s3 import create_file_metadata_from_df
+# from s3.create_dataset_graphs import create_data_set_graphs_dict
+
+# # --- File Validation ---
+# MAX_FILE_SIZE_BYTES = 100 * 1024 * 1024  # 100 MiB
+# MAX_ROWS_ALLOWED = 1_000_000
+# ALLOWED_EXTENSIONS = {".csv", ".xlsx", ".xls", ".ods"}
+
+# # --- AWS S3 Config ---
+# print("AWS S3 config:", AWS_S3_CREDS_KEY_ID, AWS_S3_CREDS_SECRET_KEY, BUCKET_NAME)
+# ACCESS_KEY = AWS_S3_CREDS_KEY_ID
+# SECRET_KEY = AWS_S3_CREDS_SECRET_KEY
+# BUCKET_NAME = BUCKET_NAME
+# REGION_NAME = "us-east-1"
+
+# s3 = boto3.client(
+#     "s3",
+#     aws_access_key_id=ACCESS_KEY,
+#     aws_secret_access_key=SECRET_KEY,
+#     region_name=REGION_NAME
+# )
+
+# ENDPOINT_URL = f"https://s3.{REGION_NAME}.amazonaws.com"
+
+# # --- FastAPI Router ---
+# s3_bucket_router1 = APIRouter(prefix="/s3/v3", tags=["s3_v3"])
+
+
+# # --- Helper: S3 Key ---
+# def make_key(path: str, filename: str) -> str:
+#     return f"{path.strip('/')}/{filename}" if path else filename
+
+
+# # --- Sanitize for JSON ---
+# def sanitize_for_json(obj):
+#     if isinstance(obj, dict):
+#         return {k: sanitize_for_json(v) for k, v in obj.items()}
+#     elif isinstance(obj, list):
+#         return [sanitize_for_json(item) for item in obj]
+#     elif isinstance(obj, (pd.Timestamp, pd.DatetimeTZDtype)):
+#         return str(obj)
+#     elif pd.isna(obj):
+#         return None
+#     elif isinstance(obj, (int, float, str, bool, type(None))):
+#         return obj
+#     else:
+#         return str(obj)
+
+
+# # --- Vector DB Placeholders ---
+# def check_vdb(user_id: str):
+#     print(f"Checking VDB for user: {user_id}")
+
+# async def add_metadata_only(collection_name: str, metadata: dict):
+#     print(f"Adding metadata to collection: {collection_name}")
+#     return {"status": "success", "collection": collection_name}
+
+
+# # --- AutoViz HTML ---
+# def run_autoviz_html(
+#     dfte, depVar="", sep=",", header=0, verbose=0, lowess=False,
+#     max_rows_analyzed=150000, max_cols_analyzed=30
+# ):
+#     tmp_dir = tempfile.mkdtemp()
+#     AV = AutoViz_Class()
+#     AV.AutoViz(
+#         filename="", sep=sep, depVar=depVar, dfte=dfte, header=header,
+#         verbose=verbose, lowess=lowess, chart_format="html",
+#         max_rows_analyzed=max_rows_analyzed, max_cols_analyzed=max_cols_analyzed,
+#         save_plot_dir=tmp_dir
+#     )
+#     print(f"HTML plots saved to: {tmp_dir}")
+#     return tmp_dir
+
+
+# # --- AutoViz Bokeh ---
+# def run_autoviz_bokeh(
+#     dfte, depVar="", sep=",", header=0, verbose=0, lowess=False,
+#     max_rows_analyzed=150000, max_cols_analyzed=30
+# ):
+#     tmp_dir = tempfile.mkdtemp()
+#     print(f"Bokeh temp directory: {tmp_dir}")
+#     try:
+#         matplotlib.use('Agg')
+#         plt.ioff()
+#         AV = AutoViz_Class()
+#         AV.AutoViz(
+#             filename="", sep=sep, depVar=depVar, dfte=dfte, header=header,
+#             verbose=verbose, lowess=lowess, chart_format="bokeh",
+#             max_rows_analyzed=max_rows_analyzed, max_cols_analyzed=max_cols_analyzed,
+#             save_plot_dir=tmp_dir
+#         )
+#         plt.close('all')
+#         print(f"Bokeh plots saved to: {tmp_dir}")
+#         return tmp_dir
+#     except Exception as e:
+#         print(f"Error in Bokeh: {e}")
+#         import traceback
+#         traceback.print_exc()
+#         raise
+
+
+# # --- Fallback Matplotlib ---
+# def generate_matplotlib_plots(df, user_id, base_filename):
+#     print("\nGenerating matplotlib fallback plots...")
+#     tmp_dir = tempfile.mkdtemp()
+#     plot_metadata = []
+#     try:
+#         matplotlib.use('Agg')
+#         numeric_cols = df.select_dtypes(include=['number']).columns.tolist()
+#         if len(numeric_cols) == 0:
+#             print("No numeric columns for plotting")
+#             return []
+
+#         # Correlation heatmap
+#         if len(numeric_cols) > 1:
+#             plt.figure(figsize=(10, 8))
+#             import seaborn as sns
+#             corr = df[numeric_cols].corr()
+#             sns.heatmap(corr, annot=True, cmap='coolwarm', center=0)
+#             plt.title('Correlation Heatmap')
+#             plt.tight_layout()
+#             png_path = os.path.join(tmp_dir, 'correlation_heatmap.png')
+#             plt.savefig(png_path, dpi=100, bbox_inches='tight')
+#             plt.close()
+
+#             s3_key = f"{user_id}/files/datasets/{base_filename}/pngs/correlation_heatmap.png"
+#             with open(png_path, 'rb') as f:
+#                 s3.put_object(Bucket=BUCKET_NAME, Key=s3_key, Body=f.read(), ContentType='image/png')
+#             plot_metadata.append({
+#                 "file_name": "correlation_heatmap.png",
+#                 "s3_path": s3_key,
+#                 "url": f"{ENDPOINT_URL}/{BUCKET_NAME}/{s3_key}",
+#                 "type": "png",
+#                 "plot_type": "heatmap"
+#             })
+
+#         # Histograms
+#         for col in numeric_cols[:5]:
+#             plt.figure(figsize=(10, 6))
+#             df[col].hist(bins=30, edgecolor='black')
+#             plt.title(f'Distribution of {col}')
+#             plt.xlabel(col)
+#             plt.ylabel('Frequency')
+#             plt.tight_layout()
+#             png_path = os.path.join(tmp_dir, f'distribution_{col}.png')
+#             plt.savefig(png_path, dpi=100, bbox_inches='tight')
+#             plt.close()
+
+#             s3_key = f"{user_id}/files/datasets/{base_filename}/pngs/distribution_{col}.png"
+#             with open(png_path, 'rb') as f:
+#                 s3.put_object(Bucket=BUCKET_NAME, Key=s3_key, Body=f.read(), ContentType='image/png')
+#             plot_metadata.append({
+#                 "file_name": f"distribution_{col}.png",
+#                 "s3_path": s3_key,
+#                 "url": f"{ENDPOINT_URL}/{BUCKET_NAME}/{s3_key}",
+#                 "type": "png",
+#                 "plot_type": "histogram"
+#             })
+#         return plot_metadata
+#     except Exception as e:
+#         print(f"Matplotlib failed: {e}")
+#         return []
+#     finally:
+#         shutil.rmtree(tmp_dir, ignore_errors=True)
+
+
+# # --- Upload Viz Files ---
+# def upload_viz_files_to_s3(tmp_dir, file_pattern, user_id, base_filename, viz_type):
+#     patterns = [file_pattern, f"*.{viz_type}", f"**/*.{viz_type}"]
+#     files = []
+#     for p in patterns:
+#         files.extend(glob.glob(os.path.join(tmp_dir, p), recursive=True))
+#     files = list(set(files))
+#     if not files:
+#         print(f"No {viz_type} files found")
+#         return []
+
+#     folder = "htmls" if viz_type == "html" else "svgs" if viz_type == "svg" else "pngs"
+#     metadata = []
+#     content_type_map = {
+#         'html': 'text/html', 'svg': 'image/svg+xml', 'png': 'image/png',
+#         'jpg': 'image/jpeg', 'jpeg': 'image/jpeg'
+#     }
+
+#     for file_path in files:
+#         name = os.path.basename(file_path)
+#         ext = os.path.splitext(name)[1][1:]
+#         s3_key = f"{user_id}/files/datasets/{base_filename}/{folder}/{name}"
+#         with open(file_path, "rb") as f:
+#             body = f.read()
+#         s3.put_object(
+#             Bucket=BUCKET_NAME, Key=s3_key, Body=body,
+#             ContentType=content_type_map.get(ext, 'application/octet-stream')
+#         )
+#         metadata.append({
+#             "file_name": name, "s3_path": s3_key,
+#             "url": f"{ENDPOINT_URL}/{BUCKET_NAME}/{s3_key}",
+#             "type": viz_type, "size": len(body)
+#         })
+#         print(f"{viz_type.upper()} uploaded: {s3_key}")
+#     return metadata
+
+
+# # --- Convert to Parquet ---
+# def convert_df_to_parquet(df: pd.DataFrame) -> io.BytesIO:
+#     buffer = io.BytesIO()
+#     df.to_parquet(buffer, engine='pyarrow', compression='snappy', index=False)
+#     buffer.seek(0)
+#     return buffer
+
+
+# # --- Check File Hash Exists (FIXED) ---
+# async def check_file_hash_exists(user_id: str, file_hash: str) -> dict:
+#     """
+#     Check if a file hash already exists for a user.
+#     Returns a dict with 'success', 'exists', and optional 'data' keys.
+#     """
+#     url = f"https://mr-mvp-api-dev.dev.ingenspark.com/auth/UserMetadata/{user_id}/check_file_hash?file_hash={file_hash}"
+#     headers = {"accept": "application/json"}
+#     async with httpx.AsyncClient(timeout=10.0) as client:
+#         try:
+#             r = await client.get(url, headers=headers)
+#             r.raise_for_status()
+#             result = r.json()
+            
+#             # Log the actual response for debugging
+#             print(f"Hash check API response: {result}")
+            
+#             # Check for 'exists' field first, then parse message if not present
+#             exists = result.get("exists", None)
+            
+#             # If 'exists' field is not present, parse the message
+#             if exists is None and "message" in result:
+#                 message_lower = result["message"].lower()
+#                 # Check if message indicates file already exists
+#                 exists = "already existed" in message_lower or "already exists" in message_lower or "duplicate" in message_lower
+            
+#             # Default to False if still None
+#             if exists is None:
+#                 exists = False
+            
+#             return {
+#                 "success": True,
+#                 "exists": exists,
+#                 "data": result
+#             }
+#         except httpx.HTTPStatusError as e:
+#             print(f"Hash check HTTP error: {e.response.status_code} - {e.response.text}")
+#             return {
+#                 "success": False, 
+#                 "exists": False,  # Assume not exists on error to be safe
+#                 "message": f"HTTP {e.response.status_code}",
+#                 "error_detail": e.response.text
+#             }
+#         except Exception as e:
+#             print(f"Hash check exception: {str(e)}")
+#             return {
+#                 "success": False, 
+#                 "exists": False,  # Assume not exists on error
+#                 "message": f"Request failed: {str(e)}"
+#             }
+
+
+# # --- PostgreSQL Metadata Upload with file_hash ---
+# async def user_metadata_upload_pg(
+#     user_id: str,
+#     user_metadata: str,
+#     path: str,
+#     url: str,
+#     filename: str,
+#     file_type: str,
+#     file_size_bytes: int,
+#     file_hash: str,
+#     timeout: float = 10.0
+# ):
+#     payload = {
+#         "user_id": user_id,
+#         "user_metadata": user_metadata,
+#         "path": path,
+#         "url": url,
+#         "filename": filename,
+#         "file_type": file_type,
+#         "file_size_bytes": file_size_bytes,
+#         "file_hash": file_hash
+#     }
+#     print(f"PostgreSQL payload file_hash: {payload['file_hash']}")
+#     async with httpx.AsyncClient() as client:
+#         try:
+#             r = await client.post(
+#                 "https://mr-mvp-api-dev.dev.ingenspark.com/auth/UserMetadataCreate",
+#                 json=payload,
+#                 timeout=timeout
+#             )
+#             r.raise_for_status()
+#             result = r.json()
+#             result["file_hash"] = file_hash
+#             print(f"PostgreSQL result file_hash: {result['file_hash']}")
+#             return {"success": True, "data": result}
+#         except httpx.HTTPStatusError as e:
+#             return {
+#                 "success": False,
+#                 "error": "HTTP error",
+#                 "status_code": e.response.status_code,
+#                 "detail": e.response.text,
+#                 "file_hash": file_hash
+#             }
+#         except Exception as e:
+#             return {
+#                 "success": False,
+#                 "error": "Request failed",
+#                 "detail": str(e),
+#                 "file_hash": file_hash
+#             }
+
+
+# # --- DEBUG ENDPOINT (Optional - for troubleshooting) ---
+# @s3_bucket_router1.get("/debug/check_hash/{user_id}/{file_hash}")
+# async def debug_check_hash(user_id: str, file_hash: str):
+#     """Debug endpoint to test hash checking"""
+#     result = await check_file_hash_exists(user_id, file_hash)
+#     return {
+#         "raw_result": result,
+#         "exists_value": result.get("exists"),
+#         "exists_type": type(result.get("exists")).__name__,
+#         "success_value": result.get("success"),
+#         "interpretation": "File exists" if result.get("exists") is True else "File does not exist or check failed"
+#     }
+
+
+# # --- MAIN ENDPOINT WITH FIXED HASH CHECK AND DATASET GRAPHS ---
+# @s3_bucket_router1.post("/upload_datasets_v3/")
+# async def upload_file(
+#     file: UploadFile = File(...),
+#     user_id: str = Query(..., description="User ID"),
+#     path: str = Query("", description="Optional subpath")
+# ):
+#     html_tmp_dir = None
+#     bokeh_tmp_dir = None
+#     file_content = None
+
+#     try:
+#         # 1. Validate extension
+#         file_ext = os.path.splitext(file.filename)[1].lower()
+#         if file_ext not in ALLOWED_EXTENSIONS:
+#             raise HTTPException(status_code=400, detail=f"Unsupported file type: {file_ext}")
+
+#         # 2. Read file
+#         file_content = await file.read()
+#         if not file_content:
+#             raise HTTPException(status_code=400, detail="Empty file")
+#         if len(file_content) > MAX_FILE_SIZE_BYTES:
+#             raise HTTPException(status_code=413, detail="File exceeds 100 MiB limit")
+
+#         # 3. Generate hash
+#         file_hash = hashlib.sha256(file_content).hexdigest()
+#         print(f"Generated file hash: {file_hash}")
+
+#         # 4. Check hash via API (FIXED LOGIC)
+#         hash_result = await check_file_hash_exists(user_id, file_hash)
+        
+#         # Enhanced logging
+#         print(f"Hash check result: {hash_result}")
+        
+#         # Check if the API call was successful first
+#         if not hash_result.get("success", False):
+#             print(f"⚠️ Warning: Hash check API failed: {hash_result.get('message')}")
+#             # You can decide to fail here if hash check is critical
+#             # raise HTTPException(status_code=503, detail="Hash check service unavailable")
+        
+#         # Check if file exists
+#         if hash_result.get("exists") is True:
+#             print(f"🚫 Duplicate file detected: {file_hash}")
+#             return JSONResponse(
+#                 status_code=409,
+#                 content={
+#                     "message": "File already uploaded.",
+#                     "reason": "Duplicate file detected via SHA-256 hash.",
+#                     "file_hash": file_hash,
+#                     "user_id": user_id,
+#                     "filename": file.filename,
+#                     "action": "skipped",
+#                     "existing_file_info": hash_result.get("data")
+#                 }
+#             )
+        
+#         print("✅ Hash check passed. New file - proceeding with upload.")
+
+#         # 5. Load DataFrame
+#         try:
+#             if file_ext == ".csv":
+#                 df = pd.read_csv(io.BytesIO(file_content))
+#             elif file_ext in {".xlsx", ".xls"}:
+#                 engine = 'openpyxl' if file_ext == ".xlsx" else 'xlrd'
+#                 df = pd.read_excel(io.BytesIO(file_content), engine=engine)
+#             elif file_ext == ".ods":
+#                 df = pd.read_excel(io.BytesIO(file_content), engine='odf')
+#         except Exception as e:
+#             raise HTTPException(status_code=400, detail=f"Failed to parse file: {str(e)}")
+
+#         if len(df) > MAX_ROWS_ALLOWED:
+#             raise HTTPException(status_code=413, detail=f"Too many rows: {len(df):,} > {MAX_ROWS_ALLOWED:,}")
+
+#         # 6. Convert to Parquet
+#         parquet_buffer = convert_df_to_parquet(df)
+#         parquet_size = parquet_buffer.getbuffer().nbytes
+
+#         # 7. Upload Parquet to S3
+#         base_filename = os.path.splitext(file.filename)[0]
+#         parquet_filename = f"{base_filename}.parquet"
+#         file_key = f"{user_id}/files/datasets/{parquet_filename}"
+#         file_url = f"{ENDPOINT_URL}/{BUCKET_NAME}/{file_key}"
+
+#         s3.upload_fileobj(parquet_buffer, BUCKET_NAME, file_key,
+#                           ExtraArgs={'ContentType': 'application/octet-stream'})
+#         print(f"Uploaded Parquet: {file_url}")
+
+#         # 8. Generate metadata
+#         metadata = create_file_metadata_from_df(df, parquet_filename, file_key)
+#         metadata.update({
+#             "user_id": user_id,
+#             "s3_path": file_key,
+#             "s3_url": file_url,
+#             "source_file": file.filename,
+#             "source_file_type": file_ext[1:],
+#             "file_type": "parquet",
+#             "original_file_size_bytes": len(file_content),
+#             "parquet_file_size_bytes": parquet_size,
+#             "compression_ratio": f"{(1 - parquet_size/len(file_content))*100:.1f}%",
+#             "file_hash": file_hash
+#         })
+
+#         # 8.5 Generate dataset preview graphs
+#         print("Generating dataset preview graphs...")
+#         try:
+#             dataset_graphs = create_data_set_graphs_dict(df, max_rows=200)
+#             metadata["data_sets_preview_graph"] = dataset_graphs
+#             print(f"✅ Generated graphs for {len(dataset_graphs.get('columnSummaries', []))} columns")
+#         except Exception as e:
+#             print(f"⚠️ Failed to generate dataset graphs: {e}")
+#             import traceback
+#             traceback.print_exc()
+#             metadata["data_sets_preview_graph"] = {
+#                 "error": str(e),
+#                 "columnSummaries": []
+#             }
+
+#         safe_metadata = sanitize_for_json(metadata)
+
+#         # 9. Vector DB
+#         check_vdb(user_id)
+#         vdb_res = await add_metadata_only("sri_1_files_&_files_metadata", metadata)
+#         vdb_success = vdb_res.get("status") == "success"
+#         print(f"VDB upload success: {vdb_success}")
+
+#         # 10. PostgreSQL Metadata + file_hash
+#         pg_result = await user_metadata_upload_pg(
+#             user_id=user_id,
+#             user_metadata=json.dumps(safe_metadata),
+#             path=file_key,
+#             url=file_url,
+#             filename=parquet_filename,
+#             file_type="parquet",
+#             file_size_bytes=parquet_size,
+#             file_hash=file_hash
+#         )
+#         print(f"PostgreSQL upload result: {pg_result}")
+#         pg_success = pg_result.get("success", False)
+        
+#         graphs_count = len(safe_metadata.get("data_sets_preview_graph", {}).get("columnSummaries", []))
+#         print(f"Graphs generated: {graphs_count}")
+
+#         # 11. Return success
+#         return {
+#             "message": "Upload successful.",
+#             "filename": parquet_filename,
+#             "original_filename": file.filename,
+#             "user_id": user_id,
+#             "file_path": file_key,
+#             "file_url": file_url,
+#             "file_hash": file_hash,
+#             "source_file_type": file_ext[1:],
+#             "file_type": "parquet",
+#             "original_file_size_bytes": len(file_content),
+#             "parquet_file_size_bytes": parquet_size,
+#             "compression_ratio": f"{(1 - parquet_size/len(file_content))*100:.1f}%",
+#             "rows": len(df),
+#             "columns": len(df.columns),
+#             "metadata": safe_metadata,
+#             "upload_dataset_vdb": vdb_success,
+#             "upload_dataset_pg": pg_success,
+#             "pg_details": pg_result,
+#             "graphs_generated": graphs_count
+#         }
+
+#     except HTTPException:
+#         raise
+#     except Exception as e:
+#         print(f"Unexpected error: {e}")
+#         import traceback
+#         traceback.print_exc()
+#         raise HTTPException(status_code=500, detail=str(e))
+#     finally:
+#         # Clean up temp directories
+#         for d in (html_tmp_dir, bokeh_tmp_dir):
+#             if d and os.path.exists(d):
+#                 shutil.rmtree(d, ignore_errors=True)
+
+
+
+
+
+from fastapi import FastAPI, UploadFile, File, HTTPException, Query, APIRouter
+from fastapi.responses import JSONResponse
+import pandas as pd
+from autoviz.AutoViz_Class import AutoViz_Class
+import io, os, boto3, tempfile, glob, matplotlib, json, hashlib, shutil
+matplotlib.use('Agg')
+import matplotlib.pyplot as plt
+import sys
+from pathlib import Path
+from typing import List
+import httpx
+
+# --- Project Root Setup ---
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from retrieve_secret import *
+from s3.meta_data_creation_from_s3 import create_file_metadata_from_df
+from s3.create_dataset_graphs import create_data_set_graphs_dict
+
+# --- File Validation ---
+MAX_FILE_SIZE_BYTES = 100 * 1024 * 1024  # 100 MiB
+MAX_ROWS_ALLOWED = 1_000_000
+ALLOWED_EXTENSIONS = {".csv", ".xlsx", ".xls", ".ods"}
+
+# --- AWS S3 Config ---
+print("AWS S3 config:", AWS_S3_CREDS_KEY_ID, AWS_S3_CREDS_SECRET_KEY, BUCKET_NAME)
+ACCESS_KEY = AWS_S3_CREDS_KEY_ID
+SECRET_KEY = AWS_S3_CREDS_SECRET_KEY
+BUCKET_NAME = BUCKET_NAME
+REGION_NAME = "us-east-1"
+
+s3 = boto3.client(
+    "s3",
+    aws_access_key_id=ACCESS_KEY,
+    aws_secret_access_key=SECRET_KEY,
+    region_name=REGION_NAME
+)
+
+ENDPOINT_URL = f"https://s3.{REGION_NAME}.amazonaws.com"
+
+# --- FastAPI Router ---
+s3_bucket_router1 = APIRouter(prefix="/s3/v3", tags=["s3_v3"])
+
+
+# --- Helper: S3 Key ---
+def make_key(path: str, filename: str) -> str:
+    return f"{path.strip('/')}/{filename}" if path else filename
+
+
+# --- Sanitize for JSON ---
+def sanitize_for_json(obj):
+    if isinstance(obj, dict):
+        return {k: sanitize_for_json(v) for k, v in obj.items()}
+    elif isinstance(obj, list):
+        return [sanitize_for_json(item) for item in obj]
+    elif isinstance(obj, (pd.Timestamp, pd.DatetimeTZDtype)):
+        return str(obj)
+    elif pd.isna(obj):
+        return None
+    elif isinstance(obj, (int, float, str, bool, type(None))):
+        return obj
+    else:
+        return str(obj)
+
+
+# --- Vector DB Placeholders ---
+def check_vdb(user_id: str):
+    print(f"Checking VDB for user: {user_id}")
+
+async def add_metadata_only(collection_name: str, metadata: dict):
+    print(f"Adding metadata to collection: {collection_name}")
+    return {"status": "success", "collection": collection_name}
+
+
+# --- AutoViz HTML ---
+def run_autoviz_html(
+    dfte, depVar="", sep=",", header=0, verbose=0, lowess=False,
+    max_rows_analyzed=150000, max_cols_analyzed=30
+):
+    tmp_dir = tempfile.mkdtemp()
+    AV = AutoViz_Class()
+    AV.AutoViz(
+        filename="", sep=sep, depVar=depVar, dfte=dfte, header=header,
+        verbose=verbose, lowess=lowess, chart_format="html",
+        max_rows_analyzed=max_rows_analyzed, max_cols_analyzed=max_cols_analyzed,
+        save_plot_dir=tmp_dir
+    )
+    print(f"HTML plots saved to: {tmp_dir}")
+    return tmp_dir
+
+
+# --- AutoViz Bokeh ---
+def run_autoviz_bokeh(
+    dfte, depVar="", sep=",", header=0, verbose=0, lowess=False,
+    max_rows_analyzed=150000, max_cols_analyzed=30
+):
+    tmp_dir = tempfile.mkdtemp()
+    print(f"Bokeh temp directory: {tmp_dir}")
+    try:
+        matplotlib.use('Agg')
+        plt.ioff()
+        AV = AutoViz_Class()
+        AV.AutoViz(
+            filename="", sep=sep, depVar=depVar, dfte=dfte, header=header,
+            verbose=verbose, lowess=lowess, chart_format="bokeh",
+            max_rows_analyzed=max_rows_analyzed, max_cols_analyzed=max_cols_analyzed,
+            save_plot_dir=tmp_dir
+        )
+        plt.close('all')
+        print(f"Bokeh plots saved to: {tmp_dir}")
+        return tmp_dir
+    except Exception as e:
+        print(f"Error in Bokeh: {e}")
+        import traceback
+        traceback.print_exc()
+        raise
+
+
+# --- Fallback Matplotlib ---
+def generate_matplotlib_plots(df, user_id, base_filename):
+    print("\nGenerating matplotlib fallback plots...")
+    tmp_dir = tempfile.mkdtemp()
+    plot_metadata = []
+    try:
+        matplotlib.use('Agg')
+        numeric_cols = df.select_dtypes(include=['number']).columns.tolist()
+        if len(numeric_cols) == 0:
+            print("No numeric columns for plotting")
+            return []
+
+        # Correlation heatmap
+        if len(numeric_cols) > 1:
+            plt.figure(figsize=(10, 8))
+            import seaborn as sns
+            corr = df[numeric_cols].corr()
+            sns.heatmap(corr, annot=True, cmap='coolwarm', center=0)
+            plt.title('Correlation Heatmap')
+            plt.tight_layout()
+            png_path = os.path.join(tmp_dir, 'correlation_heatmap.png')
+            plt.savefig(png_path, dpi=100, bbox_inches='tight')
+            plt.close()
+
+            s3_key = f"{user_id}/files/datasets/{base_filename}/pngs/correlation_heatmap.png"
+            with open(png_path, 'rb') as f:
+                s3.put_object(Bucket=BUCKET_NAME, Key=s3_key, Body=f.read(), ContentType='image/png')
+            plot_metadata.append({
+                "file_name": "correlation_heatmap.png",
+                "s3_path": s3_key,
+                "url": f"{ENDPOINT_URL}/{BUCKET_NAME}/{s3_key}",
+                "type": "png",
+                "plot_type": "heatmap"
+            })
+
+        # Histograms
+        for col in numeric_cols[:5]:
+            plt.figure(figsize=(10, 6))
+            df[col].hist(bins=30, edgecolor='black')
+            plt.title(f'Distribution of {col}')
+            plt.xlabel(col)
+            plt.ylabel('Frequency')
+            plt.tight_layout()
+            png_path = os.path.join(tmp_dir, f'distribution_{col}.png')
+            plt.savefig(png_path, dpi=100, bbox_inches='tight')
+            plt.close()
+
+            s3_key = f"{user_id}/files/datasets/{base_filename}/pngs/distribution_{col}.png"
+            with open(png_path, 'rb') as f:
+                s3.put_object(Bucket=BUCKET_NAME, Key=s3_key, Body=f.read(), ContentType='image/png')
+            plot_metadata.append({
+                "file_name": f"distribution_{col}.png",
+                "s3_path": s3_key,
+                "url": f"{ENDPOINT_URL}/{BUCKET_NAME}/{s3_key}",
+                "type": "png",
+                "plot_type": "histogram"
+            })
+        return plot_metadata
+    except Exception as e:
+        print(f"Matplotlib failed: {e}")
+        return []
+    finally:
+        shutil.rmtree(tmp_dir, ignore_errors=True)
+
+
+# --- Upload Viz Files ---
+def upload_viz_files_to_s3(tmp_dir, file_pattern, user_id, base_filename, viz_type):
+    patterns = [file_pattern, f"*.{viz_type}", f"**/*.{viz_type}"]
+    files = []
+    for p in patterns:
+        files.extend(glob.glob(os.path.join(tmp_dir, p), recursive=True))
+    files = list(set(files))
+    if not files:
+        print(f"No {viz_type} files found")
+        return []
+
+    folder = "htmls" if viz_type == "html" else "svgs" if viz_type == "svg" else "pngs"
+    metadata = []
+    content_type_map = {
+        'html': 'text/html', 'svg': 'image/svg+xml', 'png': 'image/png',
+        'jpg': 'image/jpeg', 'jpeg': 'image/jpeg'
+    }
+
+    for file_path in files:
+        name = os.path.basename(file_path)
+        ext = os.path.splitext(name)[1][1:]
+        s3_key = f"{user_id}/files/datasets/{base_filename}/{folder}/{name}"
+        with open(file_path, "rb") as f:
+            body = f.read()
+        s3.put_object(
+            Bucket=BUCKET_NAME, Key=s3_key, Body=body,
+            ContentType=content_type_map.get(ext, 'application/octet-stream')
+        )
+        metadata.append({
+            "file_name": name, "s3_path": s3_key,
+            "url": f"{ENDPOINT_URL}/{BUCKET_NAME}/{s3_key}",
+            "type": viz_type, "size": len(body)
+        })
+        print(f"{viz_type.upper()} uploaded: {s3_key}")
+    return metadata
+
+
+# --- Convert to Parquet ---
+def convert_df_to_parquet(df: pd.DataFrame) -> io.BytesIO:
+    buffer = io.BytesIO()
+    df.to_parquet(buffer, engine='pyarrow', compression='snappy', index=False)
+    buffer.seek(0)
+    return buffer
+
+
+# --- Check File Hash Exists (FIXED) ---
+async def check_file_hash_exists(user_id: str, file_hash: str) -> dict:
+    """
+    Check if a file hash already exists for a user.
+    Returns a dict with 'success', 'exists', and optional 'data' keys.
+    """
+    url = f"https://mr-mvp-api-dev.dev.ingenspark.com/auth/UserMetadata/{user_id}/check_file_hash?file_hash={file_hash}"
+    headers = {"accept": "application/json"}
+    async with httpx.AsyncClient(timeout=10.0) as client:
+        try:
+            r = await client.get(url, headers=headers)
+            r.raise_for_status()
+            result = r.json()
+            
+            # Log the actual response for debugging
+            print(f"Hash check API response: {result}")
+            
+            # Check for 'exists' field first, then parse message if not present
+            exists = result.get("exists", None)
+            
+            # If 'exists' field is not present, parse the message
+            if exists is None and "message" in result:
+                message_lower = result["message"].lower()
+                # Check if message indicates file already exists
+                exists = "already existed" in message_lower or "already exists" in message_lower or "duplicate" in message_lower
+            
+            # Default to False if still None
+            if exists is None:
+                exists = False
+            
+            return {
+                "success": True,
+                "exists": exists,
+                "data": result
+            }
+        except httpx.HTTPStatusError as e:
+            print(f"Hash check HTTP error: {e.response.status_code} - {e.response.text}")
+            return {
+                "success": False, 
+                "exists": False,  # Assume not exists on error to be safe
+                "message": f"HTTP {e.response.status_code}",
+                "error_detail": e.response.text
+            }
+        except Exception as e:
+            print(f"Hash check exception: {str(e)}")
+            return {
+                "success": False, 
+                "exists": False,  # Assume not exists on error
+                "message": f"Request failed: {str(e)}"
+            }
+
+
+# --- PostgreSQL Metadata Upload with file_hash ---
+async def user_metadata_upload_pg(
+    user_id: str,
+    user_metadata: str,
+    path: str,
+    url: str,
+    filename: str,
+    file_type: str,
+    file_size_bytes: int,
+    file_hash: str,
+    timeout: float = 10.0,
+    data_sets_preview_graph: str = None
+    
+):
+    payload = {
+        "user_id": user_id,
+        "user_metadata": user_metadata,
+        "path": path,
+        "url": url,
+        "filename": filename,
+        "file_type": file_type,
+        "file_size_bytes": file_size_bytes,
+        "file_hash": file_hash,
+        "data_sets_preview_graph": data_sets_preview_graph
+    }
+    print(f"PostgreSQL payload file_hash: {payload['file_hash']}")
+    print("data_sets_preview_graph", data_sets_preview_graph)
+    async with httpx.AsyncClient() as client:
+        try:
+            r = await client.post(
+                "https://mr-mvp-api-dev.dev.ingenspark.com/auth/UserMetadataCreate",
+                json=payload,
+                timeout=timeout
+            )
+            r.raise_for_status()
+            result = r.json()
+            result["file_hash"] = file_hash
+            print(f"PostgreSQL result file_hash: {result['file_hash']}")
+            return {"success": True, "data": result}
+        except httpx.HTTPStatusError as e:
+            return {
+                "success": False,
+                "error": "HTTP error",
+                "status_code": e.response.status_code,
+                "detail": e.response.text,
+                "file_hash": file_hash
+            }
+        except Exception as e:
+            return {
+                "success": False,
+                "error": "Request failed",
+                "detail": str(e),
+                "file_hash": file_hash
+            }
+
+
+# --- DEBUG ENDPOINT (Optional - for troubleshooting) ---
+@s3_bucket_router1.get("/debug/check_hash/{user_id}/{file_hash}")
+async def debug_check_hash(user_id: str, file_hash: str):
+    """Debug endpoint to test hash checking"""
+    result = await check_file_hash_exists(user_id, file_hash)
+    return {
+        "raw_result": result,
+        "exists_value": result.get("exists"),
+        "exists_type": type(result.get("exists")).__name__,
+        "success_value": result.get("success"),
+        "interpretation": "File exists" if result.get("exists") is True else "File does not exist or check failed"
+    }
+
+
+# --- MAIN ENDPOINT WITH FIXED HASH CHECK AND DATASET GRAPHS ---
+@s3_bucket_router1.post("/upload_datasets_v3/")
+async def upload_file(
+    file: UploadFile = File(...),
+    user_id: str = Query(..., description="User ID"),
+    path: str = Query("", description="Optional subpath")
+):
+    html_tmp_dir = None
+    bokeh_tmp_dir = None
+    file_content = None
+
+    try:
+        # 1. Validate extension
+        file_ext = os.path.splitext(file.filename)[1].lower()
+        if file_ext not in ALLOWED_EXTENSIONS:
+            raise HTTPException(status_code=400, detail=f"Unsupported file type: {file_ext}")
+
+        # 2. Read file
+        file_content = await file.read()
+        if not file_content:
+            raise HTTPException(status_code=400, detail="Empty file")
+        if len(file_content) > MAX_FILE_SIZE_BYTES:
+            raise HTTPException(status_code=413, detail="File exceeds 100 MiB limit")
+
+        # 3. Generate hash
+        file_hash = hashlib.sha256(file_content).hexdigest()
+        print(f"Generated file hash: {file_hash}")
+
+        # 4. Check hash via API (FIXED LOGIC)
+        hash_result = await check_file_hash_exists(user_id, file_hash)
+        
+        # Enhanced logging
+        print(f"Hash check result: {hash_result}")
+        
+        # Check if the API call was successful first
+        if not hash_result.get("success", False):
+            print(f"⚠️ Warning: Hash check API failed: {hash_result.get('message')}")
+            # You can decide to fail here if hash check is critical
+            # raise HTTPException(status_code=503, detail="Hash check service unavailable")
+        
+        # Check if file exists
+        if hash_result.get("exists") is True:
+            print(f"🚫 Duplicate file detected: {file_hash}")
+            return JSONResponse(
+                status_code=409,
+                content={
+                    "message": "File already uploaded.",
+                    "reason": "Duplicate file detected via SHA-256 hash.",
+                    "file_hash": file_hash,
+                    "user_id": user_id,
+                    "filename": file.filename,
+                    "action": "skipped",
+                    "existing_file_info": hash_result.get("data")
+                }
+            )
+        
+        print("✅ Hash check passed. New file - proceeding with upload.")
+
+        # 5. Load DataFrame
+        try:
+            if file_ext == ".csv":
+                df = pd.read_csv(io.BytesIO(file_content))
+            elif file_ext in {".xlsx", ".xls"}:
+                engine = 'openpyxl' if file_ext == ".xlsx" else 'xlrd'
+                df = pd.read_excel(io.BytesIO(file_content), engine=engine)
+            elif file_ext == ".ods":
+                df = pd.read_excel(io.BytesIO(file_content), engine='odf')
+        except Exception as e:
+            raise HTTPException(status_code=400, detail=f"Failed to parse file: {str(e)}")
+
+        if len(df) > MAX_ROWS_ALLOWED:
+            raise HTTPException(status_code=413, detail=f"Too many rows: {len(df):,} > {MAX_ROWS_ALLOWED:,}")
+
+        # 6. Convert to Parquet
+        parquet_buffer = convert_df_to_parquet(df)
+        parquet_size = parquet_buffer.getbuffer().nbytes
+
+        # 7. Upload Parquet to S3
+        base_filename = os.path.splitext(file.filename)[0]
+        parquet_filename = f"{base_filename}.parquet"
+        file_key = f"{user_id}/files/datasets/{parquet_filename}"
+        file_url = f"{ENDPOINT_URL}/{BUCKET_NAME}/{file_key}"
+
+        s3.upload_fileobj(parquet_buffer, BUCKET_NAME, file_key,
+                          ExtraArgs={'ContentType': 'application/octet-stream'})
+        print(f"Uploaded Parquet: {file_url}")
+
+        # 8. Generate metadata
+        metadata = create_file_metadata_from_df(df, parquet_filename, file_key)
+        metadata.update({
+            "user_id": user_id,
+            "s3_path": file_key,
+            "s3_url": file_url,
+            "source_file": file.filename,
+            "source_file_type": file_ext[1:],
+            "file_type": "parquet",
+            "original_file_size_bytes": len(file_content),
+            "parquet_file_size_bytes": parquet_size,
+            "compression_ratio": f"{(1 - parquet_size/len(file_content))*100:.1f}%",
+            "file_hash": file_hash
+        })
+
+        # 8.5 Generate dataset preview graphs (separate from metadata)
+        print("Generating dataset preview graphs...")
+        dataset_graphs = None
+        try:
+            dataset_graphs = create_data_set_graphs_dict(df, max_rows=200)
+            print(f"✅ Generated graphs for {len(dataset_graphs.get('columnSummaries', []))} columns")
+        except Exception as e:
+            print(f"⚠️ Failed to generate dataset graphs: {e}")
+            import traceback
+            traceback.print_exc()
+            dataset_graphs = {
+                "error": str(e),
+                "columnSummaries": []
+            }
+
+        safe_metadata = sanitize_for_json(metadata)
+        safe_dataset_graphs = sanitize_for_json(dataset_graphs) if dataset_graphs else {"columnSummaries": []}
+      
+        print(f"safe_dataset_graphs: {safe_dataset_graphs}")
+
+        # 9. Vector DB
+        check_vdb(user_id)
+        vdb_res = await add_metadata_only("sri_1_files_&_files_metadata", metadata)
+        vdb_success = vdb_res.get("status") == "success"
+        print(f"VDB upload success: {vdb_success}")
+
+        # 10. PostgreSQL Metadata + file_hash
+        pg_result = await user_metadata_upload_pg(
+            user_id=user_id,
+            user_metadata=json.dumps(safe_metadata),
+            path=file_key,
+            url=file_url,
+            filename=parquet_filename,
+            file_type="parquet",
+            file_size_bytes=parquet_size,
+            file_hash=file_hash,
+            data_sets_preview_graph=safe_dataset_graphs
+
+        )
+        print(f"PostgreSQL upload result: {pg_result}")
+        pg_success = pg_result.get("success", False)
+        
+        graphs_count = len(safe_dataset_graphs.get("columnSummaries", []))
+        print(f"Graphs generated: {graphs_count}")
+
+        # 11. Return success
+        return {
+            "message": "Upload successful.",
+            "filename": parquet_filename,
+            "original_filename": file.filename,
+            "user_id": user_id,
+            "file_path": file_key,
+            "file_url": file_url,
+            "file_hash": file_hash,
+            "source_file_type": file_ext[1:],
+            "file_type": "parquet",
+            "original_file_size_bytes": len(file_content),
+            "parquet_file_size_bytes": parquet_size,
+            "compression_ratio": f"{(1 - parquet_size/len(file_content))*100:.1f}%",
+            "rows": len(df),
+            "columns": len(df.columns),
+            "metadata": safe_metadata,
+            "data_sets_preview_graph": safe_dataset_graphs,  # Separate key at root level
+            "upload_dataset_vdb": vdb_success,
+            "upload_dataset_pg": pg_success,
+            "pg_details": pg_result,
+            "graphs_generated": graphs_count
+        }
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        print(f"Unexpected error: {e}")
+        import traceback
+        traceback.print_exc()
+        raise HTTPException(status_code=500, detail=str(e))
+    finally:
+        # Clean up temp directories
+        for d in (html_tmp_dir, bokeh_tmp_dir):
+            if d and os.path.exists(d):
+                shutil.rmtree(d, ignore_errors=True)

s3_viewer_backend/get_data_graphs.py

@@ -0,0 +1,82 @@
+import psycopg2
+import sys
+from pathlib import Path
+from typing import List
+import httpx
+
+# --- Project Root Setup ---
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+from retrieve_secret import *
+def print_table_headers():
+    # Connect to PostgreSQL using your decrypted credentials
+    conn = psycopg2.connect(
+        host=CONNECTIONS_HOST,
+        database=CONNECTIONS_DB,
+        user=CONNECTIONS_USER,
+        password=CONNECTIONS_PASS
+    )
+
+    cursor = conn.cursor()
+
+    # Query to fetch column names from the table
+    cursor.execute("""
+        SELECT column_name
+        FROM information_schema.columns
+        WHERE table_schema = 'public'
+        AND table_name   = 'user_datasets_metadata'
+        ORDER BY ordinal_position;
+    """)
+
+    columns = cursor.fetchall()
+
+    print("Table Headers:")
+    for col in columns:
+        print(col[0])
+
+    cursor.close()
+    conn.close()
+
+
+print_table_headers()
+
+
+
+def data_sets_preview_graph_retrieve(user_id, file_path):
+    try:
+        conn = psycopg2.connect(
+            host=CONNECTIONS_HOST,
+            database=CONNECTIONS_DB,
+            user=CONNECTIONS_USER,
+            password=CONNECTIONS_PASS
+        )
+        cursor = conn.cursor()
+
+        query = """
+            SELECT data_sets_preview_graph
+            FROM public.user_datasets_metadata
+            WHERE user_id = %s
+            AND path = %s
+            LIMIT 1;
+        """
+
+        cursor.execute(query, (user_id, file_path))
+        result = cursor.fetchone()
+
+        cursor.close()
+        conn.close()
+
+        if result:
+            return result[0]   # return the data_sets_preview_graph field
+        else:
+            return None
+
+    except Exception as e:
+        print(f"Error: {e}")
+        return None
+
+# user_id  = "18171ae4-134c-404a-a3d8-42fa8603dc3f"
+# file_path = "18171ae4-134c-404a-a3d8-42fa8603dc3f/files/datasets/marketing_data_daily.parquet"
+# data_sets_preview_graph = data_sets_preview_graph_retrieve(user_id, file_path)
+# print(data_sets_preview_graph)

s3_viewer_backend/s3_viewer.py

@@ -262,6 +262,8 @@ from botocore.exceptions import ClientError
 import urllib.parse
 from fastapi import APIRouter
 from retrieve_secret import *
+from pydantic import BaseModel
+
 # === CONFIG ===
 # ENDPOINT_URL = "https://s3.us-west-1.idrivee2.com"
 # ACCESS_KEY = "rNuPBAQetemqpEeBospZ"
@@ -295,6 +297,10 @@ s3 = boto3.client(
     region_name=REGION_NAME
 )
 
+class FileRequest(BaseModel):
+    file: str
+    user_id: str | None = None
+
 ENDPOINT_URL = f"https://s3.{REGION_NAME}.amazonaws.com"  # For file URL generation
 
 s3_viewer_router = APIRouter(prefix="/s3_viewer", tags=["s3_viewer"])
@@ -464,13 +470,35 @@ def get_files():
 #         )
 #         return result
 
+from s3_viewer_backend.get_data_graphs import data_sets_preview_graph_retrieve
+
 @s3_viewer_router.get("/api/file")
-def get_file(file: str, request: Request):
-    """Return preview content for CSV, text, PDF, images, Parquet, Office, etc."""
+def get_file(file: str, user_id: str | None = None, request: Request = None):
+    """Return preview content for CSV, text, PDF, images, Parquet, Office files — AND dataset preview graph."""
+
     ext = file.lower().split('.')[-1] if '.' in file else ''
-    result = {"fileType": None, "fileContent": None, "filePreview": None}
 
-    # === Special handling for Parquet ===
+    # Base Response
+    result = {
+        "user_id": user_id,              # <-- returned in output
+        "fileType": None,
+        "fileContent": None,
+        "filePreview": None,
+        "data_sets_preview": None        # <-- preview graph from DB
+    }
+
+    # ---------------------------------------------------------
+    # Get Graph Preview from DB
+    # ---------------------------------------------------------
+    try:
+        if user_id:
+            result["data_sets_preview"] = data_sets_preview_graph_retrieve(user_id, file)
+    except Exception as e:
+        result["data_sets_preview"] = f"Error retrieving graph: {str(e)}"
+
+    # ---------------------------------------------------------
+    # PARQUET
+    # ---------------------------------------------------------
     if ext == "parquet":
         try:
             df = read_parquet_from_s3(file)
@@ -486,32 +514,41 @@ def get_file(file: str, request: Request):
             )
             result["fileContent"] = f"Rows: {len(df)}, Columns: {len(df.columns)}"
             return result
+
         except Exception as e:
             raise HTTPException(status_code=500, detail=f"Error reading Parquet: {str(e)}")
 
-    # === For all other file types: use raw bytes ===
-    content = read_file_from_s3(file)  # ← This returns bytes or None
+    # ---------------------------------------------------------
+    # Read raw file from S3
+    # ---------------------------------------------------------
+    content = read_file_from_s3(file)
     if not content:
         raise HTTPException(status_code=404, detail="File not found")
 
-    # === Office Files (PPTX, DOCX, XLSX) ===
+    # ---------------------------------------------------------
+    # Office Files
+    # ---------------------------------------------------------
     office_extensions = ["pptx", "docx", "xlsx"]
     if ext in office_extensions:
         download_url = get_download_url(file, request)
-        embed_url = f"https://view.officeapps.live.com/op/embed.aspx?src={urllib.parse.quote(download_url)}"
+        embed_url = (
+            "https://view.officeapps.live.com/op/embed.aspx?src="
+            + urllib.parse.quote(download_url)
+        )
 
         result["fileType"] = ext
         result["filePreview"] = (
-            f'<iframe src="{embed_url}" '
-            'width="100%" height="600px" frameborder="0" '
-            'style="border: none; max-width: 100%;"></iframe>'
+            f'<iframe src="{embed_url}" width="100%" height="600px" '
+            'frameborder="0" style="border: none; max-width: 100%;"></iframe>'
             f'<p style="margin-top: 10px;"><small>'
             f'If preview fails, <a href="{download_url}" target="_blank">download {file}</a>.'
             f'</small></p>'
         )
         return result
 
-    # === CSV ===
+    # ---------------------------------------------------------
+    # CSV
+    # ---------------------------------------------------------
     if ext == "csv":
         try:
             df = pd.read_csv(StringIO(content.decode("utf-8")))
@@ -520,54 +557,70 @@ def get_file(file: str, request: Request):
                 classes="table table-striped table-sm",
                 index=False,
                 border=0,
-                justify="left"
+                justify="left",
             )
         except Exception as e:
             result["fileType"] = "csv-error"
             result["fileContent"] = f"Error reading CSV: {str(e)}"
         return result
 
-    # === Text files ===
-    elif ext in ["txt", "log", "json", "md", "py", "js", "css", "html", "xml"]:
+    # ---------------------------------------------------------
+    # Text Files
+    # ---------------------------------------------------------
+    if ext in ["txt", "log", "json", "md", "py", "js", "css", "html", "xml"]:
         result["fileType"] = "text"
         try:
             text_content = content.decode("utf-8", errors="replace")
             result["fileContent"] = text_content
+
             if len(text_content) < 10000:
-                result["filePreview"] = f"<pre style='max-height: 500px; overflow: auto; background: #f4f4f4; padding: 10px; border-radius: 5px;'>{text_content}</pre>"
+                result["filePreview"] = (
+                    "<pre style='max-height: 500px; overflow: auto;"
+                    " background: #f4f4f4; padding: 10px; border-radius: 5px;'>"
+                    f"{text_content}</pre>"
+                )
         except:
             result["fileContent"] = "Unable to decode text file."
         return result
 
-    # === Images ===
-    elif ext in ["png", "jpg", "jpeg", "gif", "webp", "bmp", "ico"]:
+    # ---------------------------------------------------------
+    # Images
+    # ---------------------------------------------------------
+    if ext in ["png", "jpg", "jpeg", "gif", "webp", "bmp", "ico"]:
         b64_data = base64.b64encode(content).decode("utf-8")
         mime_type = "jpeg" if ext == "jpg" else ext
+
         result["fileType"] = "image"
         result["filePreview"] = (
             f'<img src="data:image/{mime_type};base64,{b64_data}" '
-            'style="max-width:100%; height:auto; border-radius: 8px; box-shadow: 0 2px 8px rgba(0,0,0,0.1);"/>'
+            'style="max-width:100%; height:auto; border-radius: 8px; '
+            'box-shadow: 0 2px 8px rgba(0,0,0,0.1);" />'
         )
         return result
 
-    # === SVG ===
-    elif ext == "svg":
+    # ---------------------------------------------------------
+    # SVG
+    # ---------------------------------------------------------
+    if ext == "svg":
         try:
             svg_text = content.decode("utf-8")
             if "<script" in svg_text.lower():
-                result["fileType"] = "svg"
-                result["filePreview"] = "<p>SVG contains scripts. Preview blocked for security.</p>"
+                result["filePreview"] = "<p>SVG contains scripts. Preview blocked.</p>"
             else:
-                result["fileType"] = "svg"
                 result["filePreview"] = svg_text
-        except:
+
             result["fileType"] = "svg"
+        except:
             result["filePreview"] = "<p>Invalid SVG content.</p>"
+            result["fileType"] = "svg"
         return result
 
-    # === PDF ===
-    elif ext == "pdf":
+    # ---------------------------------------------------------
+    # PDF
+    # ---------------------------------------------------------
+    if ext == "pdf":
         b64_data = base64.b64encode(content).decode("utf-8")
+
         result["fileType"] = "pdf"
         result["filePreview"] = (
             f'<iframe src="data:application/pdf;base64,{b64_data}" '
@@ -575,30 +628,34 @@ def get_file(file: str, request: Request):
         )
         return result
 
-    # === HTML (open in new tab) ===
-    elif ext in ["html", "htm"]:
+    # ---------------------------------------------------------
+    # HTML Links
+    # ---------------------------------------------------------
+    if ext in ["html", "htm"]:
         result["fileType"] = "html"
         result["filePreview"] = (
-            f'<p><a href="/s3_viewer/api/html?file={urllib.parse.quote(file)}" target="_blank" '
-            f'style="color: #007bff; text-decoration: underline;">'
+            f'<p><a href="/s3_viewer/api/html?file={urllib.parse.quote(file)}" '
+            'target="_blank" style="color: #007bff; text-decoration: underline;">'
             f'Open {file} in new tab</a></p>'
         )
         return result
 
-    # === Fallback: Other files ===
-    else:
-        download_url = get_download_url(file, request)
-        result["fileType"] = "other"
-        result["fileContent"] = f"File type: .{ext}\nSize: {len(content):,} bytes"
-        result["filePreview"] = (
-            f'<div style="padding: 20px; text-align: center; background: #f8f9fa; border-radius: 8px;">'
-            f'<p><strong>Preview not available</strong> for <code>.{ext}</code> files.</p>'
-            f'<a href="{download_url}" target="_blank" download '
-            f'style="display: inline-block; padding: 10px 20px; background: #007bff; color: white; '
-            f'text-decoration: none; border-radius: 5px; margin-top: 10px;">'
-            f'Download {file}</a></div>'
-        )
-        return result
+    # ---------------------------------------------------------
+    # Fallback for unsupported types
+    # ---------------------------------------------------------
+    download_url = get_download_url(file, request)
+    result["fileType"] = "other"
+    result["fileContent"] = f"File type: .{ext}\nSize: {len(content):,} bytes"
+    result["filePreview"] = (
+        '<div style="padding: 20px; text-align: center; background: #f8f9fa; '
+        'border-radius: 8px;">'
+        f'<p><strong>Preview not available</strong> for <code>.{ext}</code> files.</p>'
+        f'<a href="{download_url}" target="_blank" download '
+        'style="display: inline-block; padding: 10px 20px; background: #007bff; '
+        'color: white; text-decoration: none; border-radius: 5px; margin-top: 10px;">'
+        f'Download {file}</a></div>'
+    )
+    return result
 
 # @s3_viewer_router.get("/api/html")
 # def view_html(file: str):