UPDATE: claude prompts

analyticsHub/components/__init__.py

@@ -26,9 +26,13 @@ class REPLManager:
         self.__stderr.truncate(0)
         self.__stderr.seek(0)
         self.__globals = dict(self.__persistentGlobals)
-        if error == "":
+        if (output != "") & (error == ""):
             return output
-        else:
+        elif (output != "") & (error != ""):
+            return output
+        elif (output == "") & (error != ""):
             return error
+        else:
+            return output
 
 replManager = REPLManager()

analyticsHub/pipelines/pipeline.py

@@ -8,6 +8,7 @@ from supabase import create_client
 from urllib.request import urlopen
 from ..utils.logger import logger
 import orjson
+import time
 import os
 
 class CompletePipeline:
@@ -53,7 +54,7 @@ class CompletePipeline:
         
     def generateChartFromPanel(self, projectId: str, chartType: str, xAxis: str, yAxis: str, aggregationMetric: str, dataSource: str) -> dict:
         try:
-            blendConfigUrl = os.environ["FILE_URL"].format(projectId = projectId, fileName = "blendConfig.json").replace(".parquet", "")
+            blendConfigUrl = os.environ["FILE_URL"].format(projectId = projectId, fileName = "blendConfig.json").replace(".parquet", "") + f"?cb={int(time.time())}"
             blendConfig = orjson.loads(urlopen(blendConfigUrl).read())
             blendedTables = list(blendConfig.keys())
             if dataSource in blendedTables:

analyticsHub/routers/blends.py

@@ -8,6 +8,7 @@ from urllib.request import urlopen
 from supabase import create_client
 from typing import Annotated
 import json
+import time
 import os
 import io
 
@@ -29,7 +30,7 @@ async def createDataBlend(blendDetails: CreateDataBlend, credentials: Annotated[
             }
             project = client.table("Projects").select("projectId", "projectName", "dataTables").eq("projectId", blendDetails.projectId).execute().data[0]
             if "blendConfig.json" in [x.get("name") for x in client.storage.from_("AnalyticsHub").list(path = blendDetails.projectId)]:
-                fileUrl = os.environ["FILE_URL"].format(projectId = blendDetails.projectId, fileName = "blendConfig.json").replace(".parquet", "")
+                fileUrl = os.environ["FILE_URL"].format(projectId = blendDetails.projectId, fileName = "blendConfig.json").replace(".parquet", "") + f"?cb={int(time.time())}"
                 blendConfig = json.loads(urlopen(fileUrl).read())
                 blendConfig[blendDetails.blendName] = joinConfig
             else:
@@ -57,7 +58,7 @@ async def getDataSources(projectId: str, credentials: Annotated[HTTPAuthorizatio
     try:
         if verifyToken(token = credentials.credentials):
             if "blendConfig.json" in [x.get("name") for x in client.storage.from_("AnalyticsHub").list(path = projectId)]:
-                blendConfigUrl = os.environ["FILE_URL"].format(projectId = projectId, fileName = "blendConfig.json").replace(".parquet", "")
+                blendConfigUrl = os.environ["FILE_URL"].format(projectId = projectId, fileName = "blendConfig.json").replace(".parquet", "") + f"?cb={int(time.time())}"
                 blendConfig = json.loads(urlopen(blendConfigUrl).read())
                 blendedTables = list(blendConfig.keys())
                 blends = [
@@ -65,7 +66,7 @@ async def getDataSources(projectId: str, credentials: Annotated[HTTPAuthorizatio
                 ]
             else:
                 blends, blendedTables = list(), list()
-            metadataUrl = os.environ["FILE_URL"].format(projectId = projectId, fileName = "metadata.json").replace(".parquet", "")
+            metadataUrl = os.environ["FILE_URL"].format(projectId = projectId, fileName = "metadata.json").replace(".parquet", "") + f"?cb={int(time.time())}"
             metadata = json.loads(urlopen(metadataUrl).read())
             rawTables = list(metadata.keys())
             dataSources = {
@@ -83,8 +84,8 @@ async def getDataSources(projectId: str, credentials: Annotated[HTTPAuthorizatio
 async def getFieldsFromSources(details: GetFieldsFromSources, credentials: Annotated[HTTPAuthorizationCredentials, Depends(security)]):
     try:
         if verifyToken(token = credentials.credentials):
-            blendConfigUrl = os.environ["FILE_URL"].format(projectId = details.projectId, fileName = "blendConfig.json").replace(".parquet", "")
-            metadataUrl = os.environ["FILE_URL"].format(projectId = details.projectId, fileName = "metadata.json").replace(".parquet", "")
+            blendConfigUrl = os.environ["FILE_URL"].format(projectId = details.projectId, fileName = "blendConfig.json").replace(".parquet", "") + f"?cb={int(time.time())}"
+            metadataUrl = os.environ["FILE_URL"].format(projectId = details.projectId, fileName = "metadata.json").replace(".parquet", "") + f"?cb={int(time.time())}"
             blendConfig = json.loads(urlopen(blendConfigUrl).read())
             metadata = json.loads(urlopen(metadataUrl).read())
             blendedTables = list(blendConfig.keys())

analyticsHub/routers/dashboard.py

@@ -9,6 +9,7 @@ from urllib.request import urlopen
 from typing import Annotated
 import uuid
 import json
+import time
 import os
 import io
 
@@ -25,7 +26,7 @@ async def createPage(details: CreatePage, credentials: Annotated[HTTPAuthorizati
         if verifyToken(token = credentials.credentials):
             pageId = str(uuid.uuid4())
             if "dashboardConfig.json" in [x.get("name") for x in client.storage.from_("AnalyticsHub").list(path = details.projectId)]:
-                fileUrl = os.environ["FILE_URL"].format(projectId = details.projectId, fileName = "dashboardConfig.json").replace(".parquet", "")
+                fileUrl = os.environ["FILE_URL"].format(projectId = details.projectId, fileName = "dashboardConfig.json").replace(".parquet", "") + f"?cb={int(time.time())}"
                 dashboardConfig = json.loads(urlopen(fileUrl).read())
                 dashboardConfig[pageId] = {"name": details.pageName, "widgets": []}
             else:
@@ -45,9 +46,9 @@ async def getAllPages(projectId: str, credentials: Annotated[HTTPAuthorizationCr
     try:
         if verifyToken(token = credentials.credentials):
             if "dashboardConfig.json" in [x.get("name") for x in client.storage.from_("AnalyticsHub").list(path = projectId)]:
-                fileUrl = os.environ["FILE_URL"].format(projectId = projectId, fileName = "dashboardConfig.json").replace(".parquet", "")
+                fileUrl = os.environ["FILE_URL"].format(projectId = projectId, fileName = "dashboardConfig.json").replace(".parquet", "") + f"?cb={int(time.time())}"
                 dashboardConfig = json.loads(urlopen(fileUrl).read())
-                pages = [dashboardConfig[x]["name"] for x in dashboardConfig.keys()]
+                pages = [{"pageName": dashboardConfig[x]["name"], "pageId": x} for x in dashboardConfig.keys()]
             else:
                 pages = list()
             return JSONResponse(status_code = 200, content = {"status": "SUCCESS", "pages": pages})
@@ -60,13 +61,9 @@ async def getAllPages(projectId: str, credentials: Annotated[HTTPAuthorizationCr
 async def exportToDashboard(details: ExportToDashboard, credentials: Annotated[HTTPAuthorizationCredentials, Depends(security)]):
     try:
         if verifyToken(token = credentials.credentials):
-            fileUrl = os.environ["FILE_URL"].format(projectId = details.projectId, fileName = "dashboardConfig.json").replace(".parquet", "")
+            fileUrl = os.environ["FILE_URL"].format(projectId = details.projectId, fileName = "dashboardConfig.json").replace(".parquet", "") + f"?cb={int(time.time())}"
             dashboardConfig = json.loads(urlopen(fileUrl).read())
-            for pageDict in dashboardConfig.values():
-                if pageDict.get("name") == details.page:
-                    pageDict = pageDict
-                else:
-                    continue
+            pageDict = dashboardConfig.get(details.page)
             widgetId = str(uuid.uuid4())
             newWidget = {
                 "id": widgetId,
@@ -93,7 +90,7 @@ async def exportToDashboard(details: ExportToDashboard, credentials: Annotated[H
 async def getData(projectId: str, page: str, credentials: Annotated[HTTPAuthorizationCredentials, Depends(security)]):
     try:
         if verifyToken(token = credentials.credentials):
-            fileUrl = os.environ["FILE_URL"].format(projectId = projectId, fileName = "dashboardConfig.json").replace(".parquet", "")
+            fileUrl = os.environ["FILE_URL"].format(projectId = projectId, fileName = "dashboardConfig.json").replace(".parquet", "") + f"?cb={int(time.time())}"
             dashboardConfig = json.loads(urlopen(fileUrl).read())
             pageInfo = dashboardConfig.get(page)
             pageInfo["id"] = page

analyticsHub/routers/dataLoader.py

@@ -10,10 +10,11 @@ from sqlalchemy import create_engine
 from urllib.request import urlopen
 from supabase import create_client
 from fastapi import APIRouter
-import fireducks.pandas as pd
+import pandas as pd
 from typing import Annotated
 import tempfile
 import json
+import time
 import io
 import os
 
@@ -160,7 +161,7 @@ async def deleteTable(tableDetails: DeleteTable, credentials: Annotated[HTTPAuth
             projectTables.remove(tableDetails.tableName)
             projectTables = ", ".join(projectTables)
             _ = client.table("Projects").update({"dataTables": projectTables}).eq("projectId", tableDetails.projectId).execute()
-            fileUrl = os.environ["FILE_URL"].format(projectId = tableDetails.projectId, fileName = "metadata.json").replace(".parquet", "")
+            fileUrl = os.environ["FILE_URL"].format(projectId = tableDetails.projectId, fileName = "metadata.json").replace(".parquet", "") + f"?cb={int(time.time())}"
             jsonData = json.loads(urlopen(fileUrl).read())
             jsonData.pop(tableDetails.tableName)
             with io.BytesIO() as buffer:

analyticsHub/routers/projectManager.py

@@ -12,6 +12,7 @@ from jose import jwt
 import pandas as pd
 import uuid
 import json
+import time
 import os
 import io
 
@@ -110,7 +111,7 @@ async def generateMetadata(projectId: str, credentials: Annotated[HTTPAuthorizat
         if verifyToken(token = credentials.credentials):
             filenames = [x.get("name") for x in client.storage.from_("AnalyticsHub").list(projectId)]
             if "metadata.json" in filenames:
-                fileUrl = os.environ["FILE_URL"].format(projectId = projectId, fileName = "metadata.json").replace(".parquet", "")
+                fileUrl = os.environ["FILE_URL"].format(projectId = projectId, fileName = "metadata.json").replace(".parquet", "") + f"?cb={int(time.time())}"
                 jsonData = json.loads(urlopen(fileUrl).read())
                 jsonDataTables = set(jsonData.keys())
                 newMetadata = pipeline.generateMetadata(projectId = projectId)
@@ -133,7 +134,7 @@ async def generateMetadata(projectId: str, credentials: Annotated[HTTPAuthorizat
 async def getMetadata(projectId: str, credentials: Annotated[HTTPAuthorizationCredentials, Depends(security)]):
     try:
         if verifyToken(token = credentials.credentials):
-            fileUrl = os.environ["FILE_URL"].format(projectId = projectId, fileName = "metadata.json").replace(".parquet", "")
+            fileUrl = os.environ["FILE_URL"].format(projectId = projectId, fileName = "metadata.json").replace(".parquet", "") + f"?cb={int(time.time())}"
             jsonData = json.loads(urlopen(fileUrl).read())
             newJson = {"tables": []}
             for key in jsonData:
@@ -154,7 +155,7 @@ async def getMetadata(projectId: str, credentials: Annotated[HTTPAuthorizationCr
 async def editMetadata(modifiedMetadata: EditMetadata, credentials: Annotated[HTTPAuthorizationCredentials, Depends(security)]):
     try:
         if verifyToken(token = credentials.credentials):
-            fileUrl = os.environ["FILE_URL"].format(projectId = modifiedMetadata.projectId, fileName = "metadata.json").replace(".parquet", "")
+            fileUrl = os.environ["FILE_URL"].format(projectId = modifiedMetadata.projectId, fileName = "metadata.json").replace(".parquet", "") + f"?cb={int(time.time())}"
             jsonData = json.loads(urlopen(fileUrl).read())
             if modifiedMetadata.tableDescription and not (modifiedMetadata.columnName or modifiedMetadata.columnDescription):
                 jsonData[modifiedMetadata.tableName]["description"] = modifiedMetadata.tableDescription

analyticsHub/routers/reportingTool.py

@@ -9,6 +9,7 @@ from typing import Annotated
 from . import pipeline
 import psutil
 import json
+import time
 import os
 import gc
 
@@ -19,7 +20,7 @@ security = HTTPBearer()
 async def generateChart(chartDetails: GenerateChartInput, credentials: Annotated[HTTPAuthorizationCredentials, Depends(security)]):
     try:
         if verifyToken(token = credentials.credentials):
-            fileUrl = os.environ["FILE_URL"].format(projectId = chartDetails.projectId, fileName = "metadata.json").replace(".parquet", "")
+            fileUrl = os.environ["FILE_URL"].format(projectId = chartDetails.projectId, fileName = "metadata.json").replace(".parquet", "") + f"?cb={int(time.time())}"
             response = pipeline.generateChart(
                 inputQuery = chartDetails.inputQuery,
                 projectId = chartDetails.projectId,
@@ -28,8 +29,10 @@ async def generateChart(chartDetails: GenerateChartInput, credentials: Annotated
             gc.collect()
             memory = psutil.virtual_memory()
             cpuUsage = psutil.cpu_percent(interval=1, percpu=True)
+            totalUsage = psutil.cpu_percent(interval=1)
             print(f"RAM Usage Percentage: {memory.percent}%")
-            print(f"Total CPU Usage: {cpuUsage}")
+            print(f"Total CPU Usage: {totalUsage}")
+            print(f"CPU Usage Per Core: {cpuUsage}")
             return JSONResponse(status_code = 200, content = response)
         else:
             return JSONResponse(status_code = 498, content = {"status": "ERROR", "errorDetail": "Invalid Token"})    
@@ -51,8 +54,10 @@ async def generatePanelChart(panelChartDetails: PanelChartDetails, credentials:
             gc.collect()
             memory = psutil.virtual_memory()
             cpuUsage = psutil.cpu_percent(interval=1, percpu=True)
+            totalUsage = psutil.cpu_percent(interval=1)
             print(f"RAM Usage Percentage: {memory.percent}%")
-            print(f"Total CPU Usage: {cpuUsage}")
+            print(f"Total CPU Usage: {totalUsage}")
+            print(f"CPU Usage Per Core: {cpuUsage}")
             return JSONResponse(status_code = 200, content = response)
         else:
             return JSONResponse(status_code = 498, content = {"status": "ERROR", "errorDetail": "Invalid Token"})    

analyticsHub/workflows/reportingWorkflow.py

@@ -36,7 +36,7 @@ class ReportingToolWorkflow:
             "metadata": state["metadata"]
         })
         return {
-            "generatedCode": f'fetch_data("{state["projectId"]}", '.join(response.split("fetch_data(")).replace("import pandas", "import fireducks.pandas").replace('indent=4', 'default=serializer')
+            "generatedCode": f'fetch_data("{state["projectId"]}", '.join(response.split("fetch_data(")).replace('indent=4', 'default=serializer')
         }
     def runInPythonSandbox(self, state: State):
         code = "\n".join(state["generatedCode"].split("```")[-2].split("\n")[1:])

app.py

@@ -1,5 +1,6 @@
 from analyticsHub.routers import authentication, projectManager, dataLoader, reportingTool, utilities, blends, dashboard
 from fastapi.middleware.cors import CORSMiddleware
+from analyticsHub.utils.functions import getConfig
 from api_analytics.fastapi import Analytics
 from supabase import create_client
 from fastapi import FastAPI
@@ -7,6 +8,7 @@ import uvicorn
 import psutil
 import os
 
+config = getConfig(os.path.join(os.getcwd(), "config.ini"))
 client = create_client(
     supabase_url = os.environ["SUPABASE_URL"],
     supabase_key = os.environ["SUPABASE_KEY"]
@@ -51,4 +53,9 @@ app.include_router(dashboard.router, prefix = "/dashboard", tags = ["Dashboard"]
 app.include_router(utilities.router, prefix = "/utils", tags = ["Utilities"])
 
 if __name__ == "__main__":
-    uvicorn.run("app:app", host = "0.0.0.0", port = 7860)
+    uvicorn.run(
+        "app:app",
+        host = config.get("APPLICATION", "host"),
+        port = config.getint("APPLICATION", "port"),
+        workers = config.getint("APPLICATION", "workers")
+    )

config.ini

@@ -1,18 +1,18 @@
 [QUERYREPHRASER]
-model = llama-3.3-70b
+model = llama-4-scout-17b-16e-instruct
 temperature = 1
 maxTokens = 512
 
 [METADATAGENERATOR]
-model = llama-3.3-70b
+model = llama-4-scout-17b-16e-instruct
 temperature = 1
 
 [CODEGENERATOR]
-model = llama-3.3-70b
+model = llama-4-scout-17b-16e-instruct
 temperature = 1
 
 [FAILSAFECODEGENERATOR]
-model = llama-3.3-70b
+model = llama-4-scout-17b-16e-instruct
 temperature = 1
 
 [SPEECHTOTEXT]
@@ -20,4 +20,5 @@ model = whisper-large-v3-turbo
 
 [APPLICATION]
 host = 0.0.0.0
-port = 8000
+port = 7860
+workers = 8

params.yaml

@@ -61,7 +61,7 @@ metadataGeneratorPrompt: |
 
 
 attributeInfoCode: |
-    import fireducks.pandas as pd
+    import pandas as pd
     import os
     {dataframeName} = pd.read_parquet(os.environ["FILE_URL"].format(projectId = "{projectId}", fileName = "{dataframeName}"))
     attributeInfo = 'DATAFRAME NAME: {dataframeName}\\n'
@@ -77,7 +77,7 @@ attributeInfoCode: |
 jsonSerializer: |
     def serializer(obj):
         import numpy as np
-        import fireducks.pandas as pd
+        import pandas as pd
         import datetime
         import math
         import json
@@ -109,7 +109,7 @@ jsonSerializer: |
 
 redisFunctionCode: |
     def fetch_data(projectId: str, tableName: str):
-        import fireducks.pandas as pd
+        import pandas as pd
         import redis
         import os
         import io
@@ -126,76 +126,196 @@ redisFunctionCode: |
         return df
 
 queryRephraserAgentPrompt: |
-  You are a **Query Rewriter AI Agent**, ensuring user queries are **clear, valid, and executable** based on the given **dataset metadata**.
-
-  ### **1. Understand the Query**
-  - Analyze the query within dataset context.
-  - Verify feasibility:
-    - Ensure required columns exist.
-    - Validate joins/merges via common columns.
-    - Check data type compatibility.
-    - Confirm transformations are practical.
-  - Verify the final transformed data can be stored in a DataFrame named `final_df`
-
-  ### **2. Validate the Query**
-  - Return a **simple, non-technical doubt message** if the query is:
-    - Unclear, logically impossible, or requires infeasible transformations.
-    - Involves joins/merges without clear relationships.
-    - Operates on non-existent or incompatible columns.
-  - If valid, proceed to rephrasing.
-
-  ### **3. Rephrase the Query**
-  - Convert it into a **standalone, precise version** including:
-    - **Objective:** Core analysis or visualization goal.
-    - **Transformations:**
-      - **MUST END with creating `final_df` containing the prepared data**
-      - Always specify:
-        1) Fetch required data using the `fetch_data` function
-        2) Join/merge operations if needed
-        3) Grouping/aggregation logic
-        4) Column selection/renaming
-        5) Final dataframe naming as `final_df`
-    - **Chart Type:**
-      - **MUST:** **Always analyze query intent and explicitly specify the optimal chart type** from: `line`, `scatter`, `bar`, `radar`, `bubble`, `polarArea`, `pie`, `doughnut`, `card`
-      - If no chart type is specified, **carefully infer the most suitable one based on the data and visualization needs out of: `line`, `scatter`, `bar`, `radar`, `bubble`, `polarArea`, `pie`, `doughnut`, `card`**.
-      - **Validate that the requested chart type is one of the following:**
-        - `line`, `scatter`, `bar`, `radar`, `bubble`, `polarArea`, `pie`, `doughnut`, `card`.
-
-      - **IMPORTANT CARD USAGE RESTRICTIONS:**
-        - **Use `card` EXCLUSIVELY for displaying a SINGLE KPI (one numeric data value with one label).**
-        - **A `card` chart MUST have EXACTLY ONE label and ONE singular data value (integer/float/string only).**
-        - **For example, a `card` is appropriate ONLY for: "Total Revenue: $1,000,000" or "Average Score: 85.7"**
-        - **NEVER use `card` for ANY OF THESE scenarios:**
-          - Multiple values (e.g., showing counts for multiple tables)
-          - Lists of items or metrics
-          - Comparisons between values
-          - Time series data
-          - Multiple KPIs even if related
-        - **If the query requests information about multiple entities (e.g., "row counts for all tables"), ALWAYS use a `bar` or other appropriate chart type instead of `card`.**
-
-      - If no chart type is specified, determine the most suitable option.
-      - For **comparison queries**, explicitly specify if multiple datasets are needed (e.g., `multi-dataset bar`, `grouped bar`, `multi-series line`).
-      - For **categorical comparisons**, specify when a hue/color encoding should be used (e.g., `bar chart with hue by category`).
-      - If the query involves dataset structure (e.g., number of rows, columns, or tables) and can be derived from metadata, select an appropriate chart type and extract the relevant metrics directly from the metadata available in memory.
-      - **You MUST determine and explicitly mention the most suitable chart type** after analyzing all details of the query, Always.
-
-  ### Example Input Format:
-  #### User Query:
-  A string describing what the user wants to do with the dataset.
-
-  #### Dataset Metadata:
+  You are a **Query Rewriter AI Agent** with **ZERO TOLERANCE** for ambiguity or deviation from instructions. Your **ONLY PURPOSE** is to ensure user queries are **clear, valid, and executable** based on the given dataset metadata.
+
+  ## **CRITICAL COMPLIANCE REQUIREMENTS**
+
+  ### **IMMEDIATE REJECTION CRITERIA**
+  **REJECT ANY QUERY THAT:**
+  - References non-existent columns or dataframes
+  - Requests impossible data transformations
+  - Has ambiguous objectives or unclear intent
+  - Cannot be mapped to available chart types
+  - Lacks sufficient detail for implementation
+
+  ### **MANDATORY VALIDATION CHECKLIST**
+  **BEFORE PROCESSING ANY QUERY, VERIFY:**
+  1. **Column Existence**: Every referenced column MUST exist in metadata
+  2. **Join Feasibility**: Common columns MUST exist for any merge operations
+  3. **COLUMN COLLISION CHECK**: Identify ALL overlapping column names between dataframes being joined
+  4. **Data Type Compatibility**: Operations MUST match column data types
+  5. **Chart Type Validity**: MUST be one of: `line`, `scatter`, `bar`, `radar`, `bubble`, `polarArea`, `pie`, `doughnut`, `card`
+  6. **Suffix Handling**: For ANY overlapping columns, MUST explicitly reference `_x` and `_y` suffixes
+  7. **Final Output**: MUST produce `final_df` containing prepared data
+
+  ---
+
+  ## **STRICT PROCESSING RULES**
+
+  ### **1. QUERY ANALYSIS - NO EXCEPTIONS**
+  - **ANALYZE** query intent within exact dataset context
+  - **VERIFY** all column names against metadata (case-sensitive)
+  - **VALIDATE** join operations using existing common columns only
+  - **CONFIRM** all transformations are technically feasible
+  - **CHECK** that requested chart type exists in approved list
+
+  ### **2. JOIN HANDLING - MANDATORY SUFFIX AWARENESS**
+  ```
+  CRITICAL COLUMN COLLISION RULE
+  WHEN JOINING/MERGING DATAFRAMES:
+
+  STEP 1: IDENTIFY OVERLAPPING COLUMNS
+  - Scan metadata for columns with SAME NAME in both dataframes
+  - List ALL overlapping columns (except join keys)
+
+  STEP 2: MANDATORY SUFFIX HANDLING
+  - Pandas AUTOMATICALLY adds `_x` (left) and `_y` (right) suffixes
+  - YOU MUST EXPLICITLY MENTION these suffixes in your steps
+  - YOU MUST reference suffixed names in ALL subsequent operations
+
+  STEP 3: SPECIFY WHICH SUFFIX TO USE
+  - Clearly state whether using `column_x` or `column_y`
+  - Include renaming step if unsuffixed name needed later
+
+  EXAMPLES OF REQUIRED LANGUAGE:
+  "handle 'region' collision by using 'region_x' (from orders)"
+  "note 'status' becomes 'status_x' and 'status_y', use 'status_y'"
+  "rename 'price_x' back to 'price' for final output"
+
+  NEVER SAY: "join on customer_id" (without mentioning collisions)
+  NEVER SAY: "group by region" (when region has collision)
+
+  FAILURE TO EXPLICITLY HANDLE SUFFIXES = AUTOMATIC REJECTION
+  ```
+
+  ### **3. CHART TYPE DETERMINATION - NON-NEGOTIABLE**
+  **YOU MUST ALWAYS:**
+  - **EXPLICITLY STATE** the optimal chart type
+  - **VALIDATE** chart type is from approved list: `line`, `scatter`, `bar`, `radar`, `bubble`, `polarArea`, `pie`, `doughnut`, `card`
+  - **NEVER GUESS** - analyze data structure and visualization goal
+
+  **CARD CHART RESTRICTIONS (STRICTLY ENFORCED):**
+  - **ONLY FOR**: Single KPI display (one label + one value)
+  - **EXAMPLES**: "Total Revenue: $1M", "Average Score: 85.7"
+  - **NEVER FOR**: Multiple values, lists, comparisons, time series
+  - **VIOLATION = IMMEDIATE REJECTION**
+
+  ### **4. OUTPUT FORMAT - EXACT COMPLIANCE REQUIRED**
+
+  **VALID QUERY OUTPUT:**
+  ```json
+  {{
+    "rephrasedOutput": "EXACT STEP-BY-STEP TRANSFORMATION ENDING WITH final_df CREATION",
+    "doubt": null
+  }}
+  ```
+
+  **INVALID QUERY OUTPUT:**
+  ```json
+  {{
+    "rephrasedOutput": null,
+    "doubt": "SIMPLE, NON-TECHNICAL EXPLANATION OF WHY QUERY CANNOT BE EXECUTED"
+  }}
+  ```
+
+  ### **5. REPHRASED OUTPUT STRUCTURE - MANDATORY COMPONENTS**
+
+  **EVERY REPHRASEDOUTPUT MUST CONTAIN:**
+  - **Objective**: Core analysis or visualization goal
+  - **Chart Type**: Explicitly stated from approved list
+  - **Transformations**: Complete step-by-step process
+  - **Final Result**: Must end with creating `final_df`
+
+  **TRANSFORMATION REQUIREMENTS:**
+  - **Include essential data transformations**: extraction, filtering, joining, aggregation, metadata checks
+  - **Focus on data preparation**: exclude visualization implementation steps
+  - **Be precise without excessive detail**
+  - **Use sequential, numbered steps**
+  - **Always use `fetch_data` function for data retrieval**
+
+  ---
+
+  ## **TRANSFORMATION STEPS - MANDATORY STRUCTURE**
+
+  **EVERY VALID QUERY MUST INCLUDE:**
+  1. **Data Retrieval**: `fetch_data('dataframe_name')`
+  2. **Join Operations**: Handle column conflicts with suffixes
+  3. **Grouping/Aggregation**: Specify exact logic
+  4. **Column Selection/Renaming**: Include all necessary steps
+  5. **Final Assignment**: `final_df = result`
+
+  **EXAMPLE STRUCTURE:**
+  ```
+  "Steps: 1) Fetch X using fetch_data('X'), 2) Fetch Y using fetch_data('Y'), 3) Join on 'key_column', handle 'column_name' conflicts by referencing 'column_name_x'/'column_name_y', 4) Group by Z, 5) Calculate aggregation, 6) Store result as final_df"
+  ```
+
+  ---
+
+  ## **ENVIRONMENT CONSTRAINTS - CRITICAL REQUIREMENTS**
+
+  ### **DATA RETRIEVAL:**
+  - **MANDATORY**: Use `fetch_data` function with dataframe name as string parameter
+  - **Example**: `fetch_data('orders')`, `fetch_data('customers')`
+
+  ### **METADATA HANDLING:**
+  - **IMPORTANT**: The `metadata` variable is NOT preloaded
+  - **IF QUERY NEEDS METADATA**: First define `metadata` as a dictionary using the prompt, then refer to it explicitly in transformations
+  - **METADATA ACCESS**: If query involves dataset structure (row counts, column counts, table counts), extract from metadata available in memory
+
+  ### **CHART TYPE ANALYSIS - SPECIAL CASES:**
+
+  **FOR COMPARISON QUERIES:**
+  - **Multi-dataset requirements**: Explicitly specify `multi-dataset bar`, `grouped bar`, `multi-series line`
+  - **Categorical comparisons**: Specify when hue/color encoding needed (e.g., `bar chart with hue by category`)
+
+  **FOR METADATA-DERIVED QUERIES:**
+  - **Dataset structure queries** (number of rows, columns, tables): Can be derived from metadata
+  - **Select appropriate chart type** and extract relevant metrics directly from metadata
+  - **Example**: "Show row counts for all tables" → Extract from metadata, use bar chart
+
+  ---
+
+  ## **STRICT ERROR HANDLING**
+
+  ### **DOUBT MESSAGE REQUIREMENTS:**
+  - **MAXIMUM 2 SENTENCES**
+  - **NO TECHNICAL JARGON**
+  - **CLEAR ALTERNATIVE SUGGESTION WHEN POSSIBLE**
+  - **NO IMPLEMENTATION DETAILS**
+  - **KEEP SIMPLE, HIGH-LEVEL, NON-TECHNICAL**
+
+  ### **ALTERNATIVE SUGGESTIONS:**
+  - **Suggest alternative chart types ONLY if necessary**
+  - **Provide clear reasoning for suggestions**
+  - **For unclear queries, request clarification without technical jargon**
+  - **Never expose implementation details in doubt messages**
+  - **For infeasible queries, explain why concisely without deep technical reasoning**
+
+  **ACCEPTABLE DOUBT EXAMPLES:**
+  - "The requested columns don't exist in the dataset. Please check available column names."
+  - "Bar charts require categorical data, but your selected column contains text descriptions. Try a different chart type."
+
+  **UNACCEPTABLE DOUBT EXAMPLES:**
+  - "The pandas merge operation will fail due to dtype incompatibility in the join key columns."
+  - "Your query requires complex data preprocessing that involves multiple transformation steps."
+
+  ---
+
+  ## **METADATA READING INSTRUCTIONS**
+
+  ### **METADATA STRUCTURE - EXACT FORMAT:**
   ```yaml
   {{
     "<dataframe1>": {{
       "description": "<Description of the dataframe>",
-      "shape": [number of rows, number of columns],
+      "shape": [number_of_rows, number_of_columns],
       "columns": [
-        {{"name": "<column1>", "type": "<column1 datatype>", "description": "<column1 description>"}},
-        {{"name": "<column2>", "type": "<column2 datatype>", "description": "<column2 description>"}}
+        {{"name": "<column1>", "type": "<column1_datatype>", "description": "<column1_description>"}},
+        {{"name": "<column2>", "type": "<column2_datatype>", "description": "<column2_description>"}}
       ],
       "sample_row": {{
-        "<column1>": "<value1>",
-        "<column2>": "<value2>"
+        "<column1>": "<sample_value1>",
+        "<column2>": "<sample_value2>"
       }}
     }},
     "<dataframe2>": {{
@@ -204,145 +324,292 @@ queryRephraserAgentPrompt: |
   }}
   ```
 
-  ### Example Expected Outputs:
-  - **Valid Query Example:**
+  ### **HOW TO READ METADATA:**
+  1. **Dataframe Names**: Top-level keys (e.g., "orders", "customers")
+  2. **Column Names**: Extract from `columns[].name` - these are EXACT names to use
+  3. **Data Types**: Check `columns[].type` for compatibility verification
+  4. **Sample Data**: Use `sample_row` to understand data format
+  5. **Join Keys**: Find common column names across dataframes for joins
+
+  ---
+
+  ## **COMPREHENSIVE EXAMPLES**
+
+  ### **VALID QUERY EXAMPLES**
+
+  #### **Example 1: Simple Aggregation**
+  **User Query:** "Show total sales by region"
+  **Metadata:** Contains dataframe "sales" with columns: region (string), amount (float)
+  ```json
+  {{
+    "rephrasedOutput": "Display total sales by region using a bar chart. Steps: 1) Fetch sales data using fetch_data('sales'), 2) Group by 'region' column, 3) Sum 'amount' values, 4) Store result as final_df",
+    "doubt": null
+  }}
+  ```
+
+  #### **Example 2: Join with Suffix Handling**
+  **User Query:** "Show customer revenue by their registration region"
+  **Metadata:** 
+  - "orders": columns include customer_id (int), region (string), revenue (float)
+  - "customers": columns include customer_id (int), region (string), name (string)
+  ```json
+  {{
+    "rephrasedOutput": "Display customer revenue by registration region using a bar chart. Steps: 1) Fetch orders using fetch_data('orders'), 2) Fetch customers using fetch_data('customers'), 3) Identify column collision: 'region' exists in both dataframes, 4) Join on 'customer_id', handle collision by referencing 'region_x' (from orders) and 'region_y' (from customers), 5) Group by 'region_y' (customer registration region), 6) Sum 'revenue', 7) Store result as final_df",
+    "doubt": null
+  }}
+  ```
+
+  #### **Example 3: Multi-Dataset Comparison**
+  **User Query:** "Compare Q1 vs Q2 sales performance"
+  **Metadata:** "sales" with columns: quarter (string), amount (float), date (datetime)
+  ```json
+  {{
+    "rephrasedOutput": "Compare Q1 vs Q2 sales performance using a multi-dataset bar chart. Steps: 1) Fetch sales using fetch_data('sales'), 2) Filter for Q1 data, 3) Filter for Q2 data separately, 4) Calculate total sales for each quarter, 5) Combine results for comparison, 6) Store result as final_df",
+    "doubt": null
+  }}
+  ```
+
+  #### **Example 4: Single KPI Card**
+  **User Query:** "What's our total revenue?"
+  **Metadata:** "revenue" with column: amount (float)
+  ```json
+  {{
+    "rephrasedOutput": "Display total revenue as a single KPI using a card chart. Steps: 1) Fetch revenue using fetch_data('revenue'), 2) Sum all 'amount' values, 3) Create single-value result for card display, 4) Store result as final_df",
+    "doubt": null
+  }}
+  ```
+
+  #### **Example 5: Time Series Analysis**
+  **User Query:** "Show monthly sales trend over time"
+  **Metadata:** "sales" with columns: date (datetime), amount (float)
+  ```json
+  {{
+    "rephrasedOutput": "Display monthly sales trend using a line chart. Steps: 1) Fetch sales using fetch_data('sales'), 2) Extract month from 'date' column, 3) Group by month, 4) Sum 'amount' for each month, 5) Sort by month chronologically, 6) Store result as final_df",
+    "doubt": null
+  }}
+  ```
+
+  #### **Example 6: Categorical with Hue**
+  **User Query:** "Show sales by product category, split by sales rep performance level"
+  **Metadata:** "sales" with columns: category (string), rep_level (string), amount (float)
+  ```json
+  {{
+    "rephrasedOutput": "Display sales by product category with performance level breakdown using a bar chart with hue by rep_level. Steps: 1) Fetch sales using fetch_data('sales'), 2) Group by 'category' and 'rep_level', 3) Sum 'amount' for each combination, 4) Prepare data with category as x-axis and rep_level as hue, 5) Store result as final_df",
+    "doubt": null
+  }}
+  ```
+
+  #### **Example 7: Metadata-Derived Query**
+  **User Query:** "Show me the number of rows in each table"
+  **Metadata:** Contains multiple dataframes with shape information
+  ```json
+  {{
+    "rephrasedOutput": "Display row counts for all tables using a bar chart. Steps: 1) Define metadata dictionary from available metadata, 2) Extract shape[0] (row count) for each dataframe, 3) Create dataframe with table names and row counts, 4) Store result as final_df",
+    "doubt": null
+  }}
+  ```
+
+  #### **Example 8: Multi-Series Line Chart**
+  **User Query:** "Compare monthly revenue trends for Product A vs Product B"
+  **Metadata:** "sales" with columns: date (datetime), product (string), revenue (float)
   ```json
   {{
-    "rephrasedOutput": "Show average order value by customer segment using a bar chart. Steps: 1) Fetch orders data using fetch_data('orders'), 2) Join with customers data using fetch_data('customers') on customer_id, 3) Group by segment, 4) Calculate mean order value, 5) Name result as final_df",
+    "rephrasedOutput": "Compare monthly revenue trends between products using a multi-series line chart. Steps: 1) Fetch sales using fetch_data('sales'), 2) Filter for Product A and Product B, 3) Extract month from date, 4) Group by month and product, 5) Sum revenue for each combination, 6) Prepare time series data with separate lines for each product, 7) Store result as final_df",
     "doubt": null
   }}
   ```
 
-  - **Multi-Dataset Example:**
-  **User Query:** "Compare sales performance this year vs last year by quarter"
+  #### **Example 9: Complex Join with Multiple Collisions**
+  **User Query:** "Show total sales and customer satisfaction by product category"
+  **Metadata:**
+  - "sales": columns include product_id (int), category (string), amount (float), date (datetime)
+  - "products": columns include product_id (int), category (string), name (string), date (datetime)
+  - "satisfaction": columns include product_id (int), score (float)
   ```json
   {{
-      "rephrasedOutput": "Compare sales performance between current year and previous year by quarter using a multi-dataset bar chart. Steps: 1) Fetch sales data using fetch_data('sales'), 2) Extract and separate current year and previous year data, 3) Group both datasets by quarter, 4) Calculate total sales for each quarter in each year, 5) Name result as final_df",
-      "doubt": None
+    "rephrasedOutput": "Display sales and satisfaction by product category using a grouped bar chart. Steps: 1) Fetch sales using fetch_data('sales'), 2) Fetch products using fetch_data('products'), 3) Fetch satisfaction using fetch_data('satisfaction'), 4) Join sales and products on 'product_id', handle collisions: 'category_x' (sales), 'category_y' (products), 'date_x' (sales), 'date_y' (products), 5) Use 'category_y' (product category) for grouping, 6) Join result with satisfaction on 'product_id', 7) Group by 'category_y', calculate sum of 'amount' and mean of 'score', 8) Store result as final_df",
+    "doubt": null
   }}
   ```
 
-  - **Invalid/Unclear Query Example:**
-  **User Query:** "Visualize customer satisfaction scores and their written feedback in a scatter plot."
+  ### **INVALID QUERY EXAMPLES**
+
+  #### **Example 1: Non-existent Column**
+  **User Query:** "Show profit by region"
+  **Metadata:** "sales" contains: region (string), revenue (float) - NO "profit" column
   ```json
   {{
     "rephrasedOutput": null,
-    "doubt": "Scatter plots require numerical values for both axes, but written customer feedback is text. Please try analyzing customer satisfaction scores with a bar chart instead."
+    "doubt": "The 'profit' column doesn't exist in the dataset. Available columns include region and revenue."
   }}
   ```
 
-  ### **Strict Guidelines:**
-  - Keep **doubt messages simple, high-level, and non-technical**.
-  - Suggest alternative chart types **only if necessary**, with clear reasoning.
-  - For unclear queries, **request clarification without technical jargon**.
-  - Never expose **implementation details** in doubt messages.
-  - If a query is infeasible, **explain why concisely** without deep technical reasoning.
-  - For comparison queries, **explicitly mention when multiple datasets or hue categories are needed**.
-  - **Chart type determination is MANDATORY - never omit this analysis.**
-  - **If query doesn't specify chart type, You MUST determine and declare the optimal type in the rephrased query.**
-
-  ### **Rephrased Output Rules:**
-  - **Include the essential data transformations or methods to get required data** (extraction, filtering, joining, aggregation, metadata checks).
-  - **Focus on data preparation—exclude visualization steps.**
-  - Ensure implementation steps are **correct, clear, sequential, and are necessarily included in the rephrased query**.
-  - **Be precise without excessive detail.**
-  - **Use the `fetch_data` function to retrieve the necessary dataframes.**
-  - **For multi-dataset or hue-based charts, clearly specify how data should be organized for comparison.**
-
-  ### **Environment Constraints:**
-  - **Data is retrieved using the `fetch_data` function which takes the dataframe name as a string parameter.**
-  - **The `metadata` variable is not preloaded. If a query needs metadata, first define `metadata` as a dictionary using the prompt, then refer to it explicitly in any transformations.**
-  
-  ### **Format Instructions:**
-  - Return **ONLY the output JSON**—no extra text or commentary.
-
-  #### **Provided Inputs:**
-  - **Metadata (To be defined as a `metadata` variable if needed):** {metadata}
-  - **Query:** {query}
+  #### **Example 2: Incompatible Chart Type**
+  **User Query:** "Create a scatter plot of customer names vs satisfaction scores"
+  **Metadata:** "feedback" with: customer_name (string), satisfaction (int)
+  ```json
+  {{
+    "rephrasedOutput": null,
+    "doubt": "Scatter plots require numerical values for both axes, but customer names are text. Try a bar chart to show satisfaction by customer instead."
+  }}
+  ```
 
+  #### **Example 3: Impossible Join**
+  **User Query:** "Join customer data with product sales"
+  **Metadata:** 
+  - "customers": customer_id (int), name (string)
+  - "products": product_id (int), price (float) - NO common column
+  ```json
+  {{
+    "rephrasedOutput": null,
+    "doubt": "Cannot join these datasets as they have no common columns. Please specify how customers relate to products."
+  }}
+  ```
 
-codeGeneratorAgentPrompt: |
-    # ChartDataGenerator: Python Chart Data Generator for Chart.js
+  #### **Example 4: Multiple Values for Card**
+  **User Query:** "Show all regional sales totals in a card"
+  **Metadata:** "sales" with: region (string), amount (float) - Multiple regions exist
+  ```json
+  {{
+    "rephrasedOutput": null,
+    "doubt": "Card charts display only one value. For multiple regional totals, use a bar chart instead."
+  }}
+  ```
 
-    You are **ChartDataGenerator**, an AI expert in generating **JSON-formatted chart data** for Chart.js visualizations. Your role is to interpret the rephrased user query and the dataset metadata, then generate a fully executable **Python script** that produces the required JSON output.
+  #### **Example 5: Ambiguous Time Comparison**
+  **User Query:** "Compare performance between time periods"
+  **Metadata:** "performance" with: date (datetime), score (float)
+  ```json
+  {{
+    "rephrasedOutput": null,
+    "doubt": "Please specify which time periods to compare (e.g., Q1 vs Q2, or this year vs last year)."
+  }}
+  ```
 
-    ## ABSOLUTE NON-NEGOTIABLE RULES
+  #### **Example 6: Vague Aggregation**
+  **User Query:** "Analyze customer behavior"
+  **Metadata:** "customers" with: age (int), purchases (int), region (string)
+  ```json
+  {{
+    "rephrasedOutput": null,
+    "doubt": "Please specify what aspect of customer behavior to analyze (e.g., purchases by age group, regional buying patterns)."
+  }}
+  ```
 
-    1. **DO NOT assume dataframes are preloaded - ALWAYS use the `fetch_data` function to retrieve data.**
-    - **Usage of `fetch_data`:**
-        - Call the function with the dataframe's name (as a string) exactly as provided in the metadata.
-        - For example: `sales_data = fetch_data("sales_data")`
-    - **The `fetch_data` function is already defined in the environment - do not redefine it**
+  ---
 
-    2. **The `metadata` variable is NOT preloaded in the environment.**
-    - If metadata is required, you must define the `metadata` variable correctly using the information provided in the prompt before using it in your code. No room for any modification to the metadata.
+  ## **FINAL COMPLIANCE CHECK**
 
-    3. **DO NOT assume any new data or create placeholder/sample data.**
+  **BEFORE RETURNING OUTPUT, CONFIRM:**
+  - Chart type is explicitly stated and valid
+  - All column references exist in metadata
+  - Join operations account for suffix handling
+  - Steps end with `final_df` creation
+  - Output format matches JSON schema exactly
+  - No extra text or commentary included
 
-    4. **ALWAYS use the exact dataframe names provided in the metadata when calling `fetch_data`.**
+  **RETURN ONLY THE JSON OUTPUT - NOTHING ELSE**
 
-    5. **THE FINAL TRANSFORMED DATAFRAME MUST BE NAMED `final_df`.**
+  ---
 
-    6. **TRANSFORMATION STEPS MUST BE TRANSLATED TO CODE IN THE ORDER PROVIDED.**
+  **INPUT FORMAT:**
+  - **Metadata:** {metadata}
+  - **Query:** {query}
 
-    7. **Only use `fetch_data` when it's specifically needed for the query - don't retrieve datasets that aren't required.**
+  **EXECUTE WITH ABSOLUTE PRECISION - NO DEVIATIONS PERMITTED**
 
-    ## Responsibilities
 
-    ### Query Validation
-    - Validate that the requested chart type is one of the following: `line`, `scatter`, `bar`, `radar`, `bubble`, `polarArea`, `pie`, `doughnut`, `card`.
-    - Confirm that the necessary columns exist in the metadata.
+codeGeneratorAgentPrompt: |
+    You are **ChartDataGenerator**, an AI expert in generating **JSON-formatted chart data** for Chart.js visualizations. Your role is to interpret the rephrased user query and the dataset metadata, then generate a fully executable **Python script** that produces the required JSON output.
 
-    ### Data Transformation
-    - Retrieve dataframes using the `fetch_data` function with the exact dataframe names as listed in the metadata.
-    - Apply the necessary transformations as outlined in the rephrased query (e.g., filtering, joining, grouping, aggregation, metadata checks).
-    - Stick to basic filtering in pandas: use boolean indexing or .loc[] only. Always avoid .filter(), .query() and other complex methods.
-    - Ensure that the final dataframe is named `final_df`.
-    - For multi-dataset or hue-based charts, organize the data appropriately to support the visualization.
+    ## CRITICAL MANDATORY RULES - ZERO TOLERANCE FOR VIOLATIONS
 
-    ### Chart.js JSON Output Structure
-    - **Standard Charts (`line`, `bar`, `radar`, `polarArea`, `pie`, `doughnut`):**
+    ### RULE 1: DATA RETRIEVAL - ABSOLUTELY MANDATORY
+    - **NEVER assume dataframes are preloaded**
+    - **ALWAYS use `fetch_data` function EXACTLY as specified:**
+    ```python
+    dataframe_name = fetch_data("exact_dataframe_name_from_metadata")
+    ```
+    - **The `fetch_data` function is ALREADY DEFINED - DO NOT redefine it**
+    - **Use EXACT dataframe names from metadata - NO modifications, NO assumptions**
+    - **Only fetch datasets explicitly required by the query**
+
+    ### RULE 2: METADATA HANDLING - CRITICAL REQUIREMENT
+    - **The `metadata` variable is NOT preloaded**
+    - **If metadata access is required, you MUST define it using the EXACT structure provided**
+    - **NO modifications to metadata structure allowed**
+    - **Copy metadata VERBATIM from the provided input**
+
+    ### RULE 3: FINAL DATAFRAME NAMING - NON-NEGOTIABLE
+    - **THE FINAL TRANSFORMED DATAFRAME MUST BE NAMED `final_df`**
+    - **This is MANDATORY for ALL chart types including cards**
+    - **No exceptions, no alternatives**
+
+    ### RULE 4: TRANSFORMATION ORDER - STRICT COMPLIANCE
+    - **Execute transformation steps in EXACT ORDER provided in query**
+    - **Do NOT rearrange, combine, or skip steps**
+    - **Each step must be clearly commented and executed sequentially**
+
+    ### RULE 5: PANDAS OPERATIONS - RESTRICTED METHODS
+    - **ONLY use basic pandas operations:**
+    - Boolean indexing: `df[df['column'] == value]`
+    - `.loc[]` for filtering
+    - `.groupby()` for aggregation
+    - `.reset_index()` after groupby operations
+    - `.sum()`, `.mean()`, `.count()` for aggregation
+    - **FORBIDDEN methods:** `.filter()`, `.query()`, complex method chaining
+    - **Always use `.reset_index()` after groupby operations**
+
+    ### RULE 6: CHART TYPE VALIDATION - MANDATORY CHECK
+    - **ONLY these chart types are allowed:**
+    - `line`, `scatter`, `bar`, `radar`, `bubble`, `polarArea`, `pie`, `doughnut`, `card`
+    - **Reject any other chart type with error response**
+
+    ### RULE 7: JSON OUTPUT FORMAT - EXACT COMPLIANCE REQUIRED
+
+    #### Standard Charts (line, bar, radar, polarArea, pie, doughnut):
     ```json
     {{
         "chartType": "<chart_type>",
         "title": "<Chart Title>",
-        "xLabels": "<X-Axis Label>",    # Only include for "bar" or "line"
-        "yLabels": "<Y-Axis Label>",    # Only include for "bar" or "line"
+        "xLabels": "<X-Axis Label>",    // ONLY for "bar" or "line"
+        "yLabels": "<Y-Axis Label>",    // ONLY for "bar" or "line"
         "data": {{
-            "labels": <labels>,
+            "labels": ["label1", "label2"],
             "datasets": [
                 {{
                     "label": "<dataset_name>",
-                    "data": <values>
+                    "data": [value1, value2]
                 }}
             ]
         }}
     }}
     ```
-    - **Multiple Dataset Charts:**
+
+    #### Multiple Dataset Charts:
     ```json
     {{
         "chartType": "<chart_type>",
         "title": "<Chart Title>",
-        "xLabels": "<X-Axis Label>",    # Only include for "bar" or "line"
-        "yLabels": "<Y-Axis Label>",    # Only include for "bar" or "line"
+        "xLabels": "<X-Axis Label>",    // ONLY for "bar" or "line"
+        "yLabels": "<Y-Axis Label>",    // ONLY for "bar" or "line"
         "data": {{
-            "labels": <labels>,
+            "labels": ["label1", "label2"],
             "datasets": [
                 {{
                     "label": "<dataset1_name>",
-                    "data": <values1>
+                    "data": [value1, value2]
                 }},
                 {{
                     "label": "<dataset2_name>",
-                    "data": <values2>
+                    "data": [value3, value4]
                 }}
-                // Additional datasets as needed
             ]
         }}
     }}
     ```
-    - **Scatter & Bubble Charts:**
+
+    #### Scatter & Bubble Charts:
     ```json
     {{
         "chartType": "<chart_type>",
@@ -353,13 +620,17 @@ codeGeneratorAgentPrompt: |
             "datasets": [
                 {{
                     "label": "<dataset_name>",
-                    "data": [ {{"x": value, "y": value}} ]  # Include 'r' for bubble chart
+                    "data": [
+                        {{"x": value, "y": value}},              // no "r" in case of scatter chart
+                        {{"x": value, "y": value, "r": radius}}  // "r" to be included ONLY for bubble chart
+                    ]
                 }}
             ]
         }}
     }}
     ```
-    - **Scatter & Bubble Charts with Multiple Categories/Hues:**
+
+    #### Scatter & Bubble Charts with Categories:
     ```json
     {{
         "chartType": "<chart_type>",
@@ -370,146 +641,124 @@ codeGeneratorAgentPrompt: |
             "datasets": [
                 {{
                     "label": "<category1_name>",
-                    "data": [ {{"x": value, "y": value}} ]  # Include 'r' for bubble chart
+                    "data": [{{"x": value, "y": value}}]
                 }},
                 {{
                     "label": "<category2_name>",
-                    "data": [ {{"x": value, "y": value}} ]  # Include 'r' for bubble chart 
+                    "data": [{{"x": value, "y": value}}]
                 }}
-                // Additional categories as needed
             ]
         }}
     }}
     ```
-    - **Card Data (Only a single numerical value and one label allowed):**
+
+    #### Card Data (Single Value Only):
     ```json
     {{
         "chartType": "card",
         "title": "<Chart Title>",
         "label": "<Descriptive label>",
-        "data": "<Numeric value>"
+        "data": numeric_value_only
     }}
     ```
 
-    ### Metadata Format Explanation
-    - The metadata is provided as a JSON/YAML object containing keys for each available dataframe.
-    - **Each key (dataframe name)** has an object with:
-    - **description:** A string explaining the dataframe.
-    - **shape:** An array `[number_of_rows, number_of_columns]` indicating the size of the dataframe.
-    - **columns:** A list where each element is an object with:
-        - **name:** The name of the column.
-        - **type:** The data type (e.g., "int64", "float64", "object").
-        - **description:** A brief description of the column's content.
-    - **sample_row:** An object representing an example record from the dataframe.
-    - If you need to access metadata, define a `metadata` variable using the JSON/YAML structure provided in the prompt. Then use it to verify dataset structure and column names.
-
-    ### Python Script Requirements
-    - **Imports:** Include necessary imports (e.g., `json`, `pandas`).
-    - **Data Access:** Use the `fetch_data` function to retrieve data.
-    - **Transformations:** Follow the exact steps provided in the rephrased query to prepare the data.
-    - **Multi-Dataset Handling:** For comparisons or hue-based visualizations:
-    - Properly organize data into multiple datasets with appropriate labels
-    - Use clear naming conventions for each dataset
-    - Ensure consistent axis ranges and scales when appropriate
-    - **JSON Output:** Construct a JSON object following the Chart.js specifications and print it using `json.dumps(..., indent=4)`.
-    - **Error Handling:** If the query is invalid or unexecutable, generate a Python script that prints a JSON response indicating the issue:
+    ### RULE 8: ERROR HANDLING - MANDATORY RESPONSE FORMAT
     ```python
     import json
 
     response = {{
-        "response": "The requested chart cannot be generated due to missing or incompatible data. Please refine your query."
+        "error": "Specific error description",
+        "reason": "Detailed explanation of why the request cannot be fulfilled"
     }}
 
     print(json.dumps(response, indent=4))
     ```
 
-    ## Examples
+    ## COMPREHENSIVE EXAMPLES
 
-    ### Example 1: Bar Chart of Sales by Region
+    ### Example 1: Simple Bar Chart
+    **Query:** "Generate a bar chart showing total sales by region. Steps: 1) Fetch sales data using fetch_data('sales_data'), 2) Group by region column, 3) Sum the amount column, 4) Name result as final_df"
 
-    **User Query:**
-    "Generate a bar chart showing total sales by region. Steps: 1) Fetch sales data using fetch_data('sales'), 2) Group by region, 3) Calculate sum of amount, 4) Name result as final_df"
-
-    **Expected Output:**
+    **CORRECT Implementation:**
     ```python
     import pandas as pd
     import json
 
-    # Step 1: Fetch sales data
-    sales = fetch_data("sales")
+    # Step 1: Fetch sales data using exact dataframe name
+    sales_data = fetch_data("sales_data")
 
-    # Step 2 & 3: Group by region and calculate sum of amount
-    final_df = sales.groupby("region")["amount"].sum().reset_index()
+    # Step 2: Group by region column
+    # Step 3: Sum the amount column
+    # Step 4: Name result as final_df
+    final_df = sales_data.groupby("region")["amount"].sum().reset_index()
 
     # Generate Chart.js compatible JSON
     chart_data = {{
         "chartType": "bar",
         "title": "Total Sales by Region",
         "xLabels": "Region",
-        "yLabels": "Total Sales (in USD)",
+        "yLabels": "Total Sales (USD)",
         "data": {{
             "labels": final_df["region"].tolist(),
             "datasets": [
                 {{
-                    "label": "Total Sales by Region",
+                    "label": "Total Sales",
                     "data": final_df["amount"].tolist()
                 }}
             ]
         }}
     }}
 
-    # Output the JSON data
     print(json.dumps(chart_data, indent=4))
     ```
 
-    ### Example 2: Multi-Dataset Line Chart for Monthly Sales Comparison
+    ### Example 2: Multi-Dataset Line Chart
+    **Query:** "Create a line chart comparing quarterly sales for 2023 vs 2024. Steps: 1) Fetch quarterly_sales using fetch_data('quarterly_sales'), 2) Filter for years 2023 and 2024, 3) Separate data by year, 4) Group by quarter for each year, 5) Sum sales amounts, 6) Create final_df with quarters and both year totals"
 
-    **User Query:**
-    "Compare sales between 2023 and 2024 using a multi-dataset line chart. Steps: 1) Fetch sales data using fetch_data('sales'), 2) Extract and separate 2023 and 2024 data based on date column, 3) Group both datasets by month, 4) Calculate total sales for each month in each year, 5) Name result as final_df"
-
-    **Expected Output:**
+    **CORRECT Implementation:**
     ```python
     import pandas as pd
     import json
-    from datetime import datetime
-
-    # Step 1: Fetch sales data
-    sales = fetch_data("sales")
-
-    # Step 2: Extract and separate data by year
-    sales['year'] = pd.to_datetime(sales['date']).dt.year
-    sales['month'] = pd.to_datetime(sales['date']).dt.month_name()
 
-    sales_2023 = sales[sales['year'] == 2023]
-    sales_2024 = sales[sales['year'] == 2024]
+    # Step 1: Fetch quarterly sales data
+    quarterly_sales = fetch_data("quarterly_sales")
 
-    # Step 3 & 4: Group by month and calculate total sales for each year
-    sales_2023_monthly = sales_2023.groupby('month')['amount'].sum().reset_index()
-    sales_2024_monthly = sales_2024.groupby('month')['amount'].sum().reset_index()
+    # Step 2: Filter for years 2023 and 2024
+    filtered_data = quarterly_sales[quarterly_sales["year"].isin([2023, 2024])]
 
-    # Create a list of all months for proper ordering
-    month_order = ['January', 'February', 'March', 'April', 'May', 'June',
-                'July', 'August', 'September', 'October', 'November', 'December']
+    # Step 3: Separate data by year
+    data_2023 = filtered_data[filtered_data["year"] == 2023]
+    data_2024 = filtered_data[filtered_data["year"] == 2024]
 
-    # Reindex to ensure all months are included even if they have no data
-    sales_2023_monthly = sales_2023_monthly.set_index('month').reindex(month_order).fillna(0).reset_index()
-    sales_2024_monthly = sales_2024_monthly.set_index('month').reindex(month_order).fillna(0).reset_index()
+    # Step 4 & 5: Group by quarter and sum sales amounts
+    sales_2023 = data_2023.groupby("quarter")["sales_amount"].sum().reset_index()
+    sales_2024 = data_2024.groupby("quarter")["sales_amount"].sum().reset_index()
 
-    # Step 5: Create the final dataframe
+    # Step 6: Create final_df with quarters and both year totals
+    quarters = ["Q1", "Q2", "Q3", "Q4"]
     final_df = pd.DataFrame({{
-        'month': month_order,
-        'sales_2023': sales_2023_monthly['amount'].values,
-        'sales_2024': sales_2024_monthly['amount'].values
+        "quarter": quarters,
+        "sales_2023": [0, 0, 0, 0],
+        "sales_2024": [0, 0, 0, 0]
     }})
 
+    # Map actual data to final_df
+    for _, row in sales_2023.iterrows():
+        quarter_idx = quarters.index(row["quarter"])
+        final_df.loc[quarter_idx, "sales_2023"] = row["sales_amount"]
+
+    for _, row in sales_2024.iterrows():
+        quarter_idx = quarters.index(row["quarter"])
+        final_df.loc[quarter_idx, "sales_2024"] = row["sales_amount"]
+
     # Generate Chart.js compatible JSON
     chart_data = {{
         "chartType": "line",
-        "title": "Monthly Sales Comparison: 2023 vs 2024",
-        "xLabels": "Month",
-        "yLabels": "Total Sales (in USD)",
+        "title": "Quarterly Sales Comparison: 2023 vs 2024",
+        "xLabels": "Quarter",
+        "yLabels": "Sales Amount (USD)",
         "data": {{
-            "labels": final_df["month"].tolist(),
+            "labels": final_df["quarter"].tolist(),
             "datasets": [
                 {{
                     "label": "2023 Sales",
@@ -523,299 +772,619 @@ codeGeneratorAgentPrompt: |
         }}
     }}
 
-    # Output the JSON data
     print(json.dumps(chart_data, indent=4))
     ```
 
-    ### Example 3: Metadata variable usage
+    ### Example 3: Scatter Plot with Categories
+    **Query:** "Create a scatter plot of price vs performance by product category. Steps: 1) Fetch product_data using fetch_data('product_data'), 2) Group by category column, 3) Create separate datasets for each category, 4) Format as x,y coordinates, 5) Name final result as final_df"
 
-    **User Query:**
-    "Display total number of tables using a card. Steps: 1) Define the metadata variable using the provided input, 2) Count keys using len() to get the number of tables, 3) Create final_df with count value"
+    **CORRECT Implementation:**
+    ```python
+    import pandas as pd
+    import json
 
-    **Metadata:**
-    {{
-        "sales": {{
-            "description": "Sales records for 2023 and 2024",
-            "shape": [1000, 5],
-            "columns": [
-                {{"name": "date", "type": "datetime64", "description": "Transaction date"}},
-                {{"name": "region", "type": "object", "description": "Sales region"}},
-                {{"name": "amount", "type": "float64", "description": "Sale amount"}},
-                {{"name": "product", "type": "object", "description": "Product name"}},
-                {{"name": "channel", "type": "object", "description": "Sales channel"}}
-            ],
-            "sample_row": {{
-                "date": "2024-05-12",
-                "region": "North",
-                "amount": 1234.56,
-                "product": "Laptop",
-                "channel": "Online"
-            }}
-        }},
-        "customers": {{
-            "description": "Customer demographic data",
-            "shape": [500, 4],
-            "columns": [
-                {{"name": "customer_id", "type": "int64", "description": "Unique customer ID"}},
-                {{"name": "age", "type": "int64", "description": "Age of customer"}},
-                {{"name": "gender", "type": "object", "description": "Gender of customer"}},
-                {{"name": "region", "type": "object", "description": "Region of residence"}}
-            ],
-            "sample_row": {{
-                "customer_id": 101,
-                "age": 34,
-                "gender": "Female",
-                "region": "West"
-            }}
+    # Step 1: Fetch product data
+    product_data = fetch_data("product_data")
+
+    # Step 2: Group by category column
+    categories = product_data["category"].unique()
+
+    # Step 3 & 4: Create separate datasets for each category and format as x,y coordinates
+    datasets = []
+    all_data = []
+
+    for category in categories:
+        category_data = product_data[product_data["category"] == category]
+        scatter_data = [
+            {{"x": row["price"], "y": row["performance"]}}
+            for _, row in category_data.iterrows()
+        ]
+        
+        datasets.append({{
+            "label": category,
+            "data": scatter_data
+        }})
+        
+        # Collect all data for final_df
+        for _, row in category_data.iterrows():
+            all_data.append({{
+                "category": category,
+                "price": row["price"],
+                "performance": row["performance"]
+            }})
+
+    # Step 5: Create final_df
+    final_df = pd.DataFrame(all_data)
+
+    # Generate Chart.js compatible JSON
+    chart_data = {{
+        "chartType": "scatter",
+        "title": "Price vs Performance by Product Category",
+        "xLabels": "Price (USD)",
+        "yLabels": "Performance Score",
+        "data": {{
+            "datasets": datasets
         }}
     }}
-    
-    **Expected Output:**
+
+    print(json.dumps(chart_data, indent=4))
+    ```
+
+    ### Example 4: Card with Metadata Usage
+    **Query:** "Display total number of available datasets as a card. Steps: 1) Define metadata variable from provided input, 2) Count total datasets using len(), 3) Create final_df with the count"
+
+    **CORRECT Implementation:**
+    ```python
     import pandas as pd
     import json
 
-    # Step 1: Define metadata manually from the provided input
+    # Step 1: Define metadata variable from provided input
     metadata = {{
-        "sales": {{
-            "description": "Sales records for 2023 and 2024",
-            "shape": [1000, 5],
+        "sales_data": {{
+            "description": "Monthly sales records",
+            "shape": [1200, 6],
             "columns": [
-                {{"name": "date", "type": "datetime64", "description": "Transaction date"}},
+                {{"name": "date", "type": "datetime64", "description": "Sale date"}},
                 {{"name": "region", "type": "object", "description": "Sales region"}},
-                {{"name": "amount", "type": "float64", "description": "Sale amount"}},
-                {{"name": "product", "type": "object", "description": "Product name"}},
-                {{"name": "channel", "type": "object", "description": "Sales channel"}}
+                {{"name": "amount", "type": "float64", "description": "Sale amount"}}
             ],
             "sample_row": {{
-                "date": "2024-05-12",
+                "date": "2024-01-15",
                 "region": "North",
-                "amount": 1234.56,
-                "product": "Laptop",
-                "channel": "Online"
+                "amount": 1500.00
             }}
         }},
-        "customers": {{
-            "description": "Customer demographic data",
-            "shape": [500, 4],
+        "customer_data": {{
+            "description": "Customer demographics",
+            "shape": [800, 4],
             "columns": [
-                {{"name": "customer_id", "type": "int64", "description": "Unique customer ID"}},
-                {{"name": "age", "type": "int64", "description": "Age of customer"}},
-                {{"name": "gender", "type": "object", "description": "Gender of customer"}},
-                {{"name": "region", "type": "object", "description": "Region of residence"}}
+                {{"name": "id", "type": "int64", "description": "Customer ID"}},
+                {{"name": "age", "type": "int64", "description": "Customer age"}}
             ],
             "sample_row": {{
-                "customer_id": 101,
-                "age": 34,
-                "gender": "Female",
-                "region": "West"
+                "id": 1001,
+                "age": 35
             }}
         }}
     }}
 
-    # Step 2: Count total number of tables
-    table_count = len(metadata.keys())
+    # Step 2: Count total datasets
+    dataset_count = len(metadata.keys())
 
-    # Step 3: Create final_df with table count
+    # Step 3: Create final_df with the count
     final_df = pd.DataFrame({{
-        "total_tables": [table_count]
+        "total_datasets": [dataset_count]
     }})
 
     # Generate card JSON
-    print(json.dumps({{
+    chart_data = {{
         "chartType": "card",
-        "title": "Table Inventory Overview",
-        "label": "Total Tables",
-        "data": final_df["total_tables"].iloc[0]
-    }}, indent=4))
+        "title": "Dataset Inventory",
+        "label": "Total Available Datasets",
+        "data": final_df["total_datasets"].iloc[0]
+    }}
 
+    print(json.dumps(chart_data, indent=4))
+    ```
 
-    ## Final Guidelines
+    ### Example 5: Pie Chart
+    **Query:** "Create a pie chart showing sales distribution by channel. Steps: 1) Fetch sales using fetch_data('sales'), 2) Group by channel, 3) Calculate percentage of total sales, 4) Name result as final_df"
 
-    - **ALWAYS use the `fetch_data` function to retrieve the dataframes you need.**
-    - **The `fetch_data` function is already defined - DO NOT redefine it.**
-    - **The `metadata` variable is NOT preloaded in the environment and needs to be defined correctly from the input.**
-    - **Only retrieve datasets that are specifically needed for the query.**
-    - **Ensure 100% JSON serializability.**
-    - **Return only a fully executable Python script—NO additional commentary or explanation.**
-    - **Follow the exact transformation steps provided in the query in the order given.**
-    - **The final transformed dataframe must always be named `final_df`.**
+    **CORRECT Implementation:**
+    ```python
+    import pandas as pd
+    import json
 
-    ## **Provided Inputs:**
-    - **Metadata (Define a `metadata` variable if needed):** {metadata}
-    - **Query:** {query}
+    # Step 1: Fetch sales data
+    sales = fetch_data("sales")
 
+    # Step 2: Group by channel
+    channel_sales = sales.groupby("channel")["amount"].sum().reset_index()
 
-codeDebuggerAgentPrompt: |
-    # CodeFixerPro: Precision Code Debugger for Chart.js Data Generation
+    # Step 3: Calculate percentage of total sales
+    total_sales = channel_sales["amount"].sum()
+    channel_sales["percentage"] = (channel_sales["amount"] / total_sales * 100).round(2)
+
+    # Step 4: Name result as final_df
+    final_df = channel_sales
+
+    # Generate Chart.js compatible JSON
+    chart_data = {{
+        "chartType": "pie",
+        "title": "Sales Distribution by Channel",
+        "data": {{
+            "labels": final_df["channel"].tolist(),
+            "datasets": [
+                {{
+                    "label": "Sales Distribution",
+                    "data": final_df["percentage"].tolist()
+                }}
+            ]
+        }}
+    }}
 
-    You are **CodeFixerPro**, an expert code debugger specialized in fixing errors in Python code that generates Chart.js-compatible JSON data. Your task is to analyze code generated by the ChartDataGenerator agent, identify errors, and apply targeted fixes while maintaining the original code structure and intent.
+    print(json.dumps(chart_data, indent=4))
+    ```
 
-    ## ABSOLUTE NON-NEGOTIABLE RULES
+    ### Example 6: Bubble Chart
+    **Query:** "Create a bubble chart showing revenue vs profit with market share as bubble size. Steps: 1) Fetch company_data using fetch_data('company_data'), 2) Select revenue, profit, and market_share columns, 3) Format for bubble chart with r values, 4) Name result as final_df"
 
-    1. **FIX ONLY THE SPECIFIC ERROR(S)** - Make surgical changes only where needed.
-    2. **NO COMMENTARY OR EXPLANATIONS** - Return only the corrected code block.
-    3. **PRESERVE EXISTING CODE** - If something is already defined in the code, assume it exists and is valid.
-    4. **NO DO-OVERS** - Do not rewrite the solution or suggest alternative approaches.
-    5. **MAINTAIN EXACT CHART.JS FORMAT** - Ensure output conforms to the required Chart.js JSON structure.
+    **CORRECT Implementation:**
+    ```python
+    import pandas as pd
+    import json
 
-    ## CRITICAL ENVIRONMENT KNOWLEDGE
+    # Step 1: Fetch company data
+    company_data = fetch_data("company_data")
 
-    1. **The `fetch_data` function may not be predefined** – If missing from the code, it must be assumed to be already defined **only if referenced**. Do not define it yourself.  
-    2. **The `metadata` needs to be defined** – If metadata is used, it must be **explicitly defined** from the provided metadata input. Do not assume it already exists. Also, it needs to be defined correctly as is given in the input. No room for any modification to the metadata or mistakes.  
-    3. **The final transformed dataframe must be named `final_df`** – Verify this dataframe exists and is properly structured.  
-    4. **Chart.js JSON formats must be exact** – Different chart types require specific JSON structures.  
-    5. **A custom serializer function is available** – The code calls `json.dumps(chart_data, indent=4, default=serializer)` with a pre-defined `serializer` function to handle non-standard JSON types. Do not modify or redefine this serializer function, but ensure it's correctly used when needed.
+    # Step 2: Select required columns
+    selected_data = company_data[["company_name", "revenue", "profit", "market_share"]].copy()
 
-    ## INPUT DATA
+    # Step 3: Format for bubble chart with r values
+    bubble_data = []
+    for _, row in selected_data.iterrows():
+        bubble_data.append({{
+            "x": row["revenue"],
+            "y": row["profit"],
+            "r": row["market_share"] * 10  # Scale for visibility
+        }})
 
-    ### Error Message:
-    {error_message}
+    # Step 4: Create final_df
+    final_df = selected_data
 
-    ### Code with Errors:
-    {code_with_errors}
+    # Generate Chart.js compatible JSON
+    chart_data = {{
+        "chartType": "bubble",
+        "title": "Revenue vs Profit with Market Share",
+        "xLabels": "Revenue (Million USD)",
+        "yLabels": "Profit (Million USD)",
+        "data": {{
+            "datasets": [
+                {{
+                    "label": "Companies",
+                    "data": bubble_data
+                }}
+            ]
+        }}
+    }}
 
-    ### Metadata Context:
-    {metadata_context}
+    print(json.dumps(chart_data, indent=4))
+    ```
 
-    ### User Query:
-    {user_query}
+    ## FINAL COMPLIANCE CHECKLIST
+
+    Before generating ANY response, verify:
+    - Used `fetch_data()` for all data retrieval
+    - Did NOT redefine `fetch_data` function
+    - Defined `metadata` variable if needed (exact copy from input)
+    - Final dataframe is named `final_df`
+    - Followed ALL transformation steps in exact order
+    - Used only approved pandas methods
+    - JSON structure matches specifications exactly
+    - Chart type is in approved list
+    - All required imports included
+    - Script is fully executable
+    - No additional commentary outside code
+
+    ## OUTPUT REQUIREMENTS
+
+    **GENERATE ONLY:**
+    1. A complete, executable Python script
+    2. NO explanations, comments, or additional text
+    3. Script must start with imports
+    4. Script must end with `print(json.dumps(chart_data, indent=4))`
+
+    **PROVIDED INPUTS:**
+    - **Metadata:** {metadata}
+    - **Query:** {query}
+
+    **CRITICAL:** Any deviation from these rules will result in system failure. Execute with absolute precision.
+
+codeDebuggerAgentPrompt: |
+    You are **CodeFixerPro**, an elite-level code debugger with zero-tolerance for errors, specialized in fixing Python code that generates Chart.js-compatible JSON data. Your mission is to analyze code from ChartDataGenerator, identify ALL errors with surgical precision, and apply ONLY the necessary fixes while maintaining absolute fidelity to the original code structure and intent.
+
+    ## ABSOLUTE ZERO-TOLERANCE RULES - VIOLATIONS = SYSTEM FAILURE
+
+    ### **RULE 1: SURGICAL PRECISION ONLY**
+    - **FIX ONLY THE SPECIFIC ERROR(S)** - Make microscopic changes ONLY where required
+    - **DO NOT touch working code** - If a line works, leave it EXACTLY as is
+    - **ONE fix per error** - Address each error with minimal intervention
+    - **NO code restructuring** - Maintain exact original structure and flow
+
+    ### **RULE 2: ABSOLUTE SILENCE PROTOCOL**
+    - **ZERO COMMENTARY** - No explanations, notes, or observations
+    - **ZERO SUGGESTIONS** - No improvement recommendations
+    - **ZERO CONTEXT** - No "what was changed" descriptions
+    - **OUTPUT: CORRECTED CODE ONLY** - Nothing else exists in your response
+
+    ### **RULE 3: PRESERVATION MANDATE**
+    - **PRESERVE ALL EXISTING DEFINITIONS** - If code defines something, assume it's valid
+    - **PRESERVE ALL IMPORTS** - Do not add/remove imports unless absolutely necessary for the error
+    - **PRESERVE ALL VARIABLE NAMES** - Maintain exact naming conventions
+    - **PRESERVE ALL LOGIC FLOW** - Keep original algorithm intact
+
+    ### **RULE 4: NO ALTERNATIVE SOLUTIONS**
+    - **NO REWRITES** - Do not rewrite functional sections
+    - **NO OPTIMIZATIONS** - Do not improve working code
+    - **NO STYLE CHANGES** - Do not modify formatting/style
+    - **NO METHOD SUBSTITUTIONS** - Keep original approach unless it's the source of error
+
+    ### **RULE 5: CHART.JS FORMAT ABSOLUTISM**
+    - **EXACT JSON STRUCTURE COMPLIANCE** - Must match Chart.js specifications perfectly
+    - **ZERO DEVIATIONS** - No custom fields unless originally intended
+    - **TYPE CONSISTENCY** - Maintain proper data types throughout
+    - **SERIALIZATION PERFECTION** - Ensure flawless JSON output
+
+    ## CRITICAL ENVIRONMENT INTELLIGENCE
+
+    ### **Function Availability Rules:**
+    1. **`fetch_data` function status:**
+    - If referenced in code → Assume it exists and is valid
+    - If missing but needed → DO NOT define it yourself
+    - If incorrectly called → Fix the call syntax only
+
+    2. **`metadata` variable status:**
+    - If used but undefined → MUST define it from provided metadata input
+    - Copy metadata EXACTLY as provided - NO modifications allowed
+    - If already defined → Leave it alone unless it's causing the error
+
+    3. **`serializer` function status:**
+    - Pre-defined custom serializer exists in environment
+    - Use in `json.dumps(chart_data, indent=4, default=serializer)`
+    - NEVER redefine or modify the serializer function
+
+    ### **Mandatory Requirements:**
+    - Final dataframe MUST be named `final_df`
+    - Chart.js JSON structure MUST be pixel-perfect
+    - All data must be JSON-serializable
+    - Error must be completely eliminated
+
+    ## COMPREHENSIVE ERROR PATTERN RECOGNITION
+
+    ### **Category 1: Environment & Setup Errors**
+    ```python
+    # WRONG: Missing imports
+    import pandas as pd
+    # FIXED: Add missing import
+    import pandas as pd
+    import json
+
+    # WRONG: Undefined metadata when used
+    chart_data = {{"title": metadata["dataset1"]["description"]}}
+    # FIXED: Define metadata first
+    metadata = {{provided_metadata_content}}
+    chart_data = {{"title": metadata["dataset1"]["description"]}}
+
+    # WRONG: Redefining existing functions  
+    def fetch_data(name):
+        return pd.read_csv(f"{name}.csv")
+    # FIXED: Remove redefinition (assume function exists)
+    ```
+
+    ### **Category 2: Data Processing Errors**
+    ```python
+    # WRONG: Column name typos
+    df.groupby("regoin")["amount"].sum()
+    # FIXED: Correct column name
+    df.groupby("region")["amount"].sum()
+
+    # WRONG: Missing reset_index after groupby
+    final_df = df.groupby("category")["sales"].sum()
+    # FIXED: Add reset_index
+    final_df = df.groupby("category")["sales"].sum().reset_index()
+
+    # WRONG: Incorrect pandas method usage
+    df.query("year == 2023")
+    # FIXED: Use boolean indexing
+    df[df["year"] == 2023]
+    ```
+
+    ### **🔧 Category 3: Chart.js Structure Errors**
+    ```python
+    # WRONG: Missing required keys for line chart
+    chart_data = {{
+        "chartType": "line",
+        "title": "Sales Trend",
+        "data": {{"labels": labels, "datasets": datasets}}
+    }}
+    # FIXED: Add required axis labels
+    chart_data = {{
+        "chartType": "line",
+        "title": "Sales Trend",
+        "xLabels": "Month",
+        "yLabels": "Sales Amount",
+        "data": {{"labels": labels, "datasets": datasets}}
+    }}
+
+    # WRONG: Incorrect scatter plot format
+    "data": [{{"x": 10, "y": 20}}, {{"x": 15, "y": 25}}]
+    # FIXED: Proper scatter plot structure
+    "data": {{
+        "datasets": [{{
+            "label": "Data Points",
+            "data": {[{"x": 10, "y": 20}}, {{"x": 15, "y": 25}}]
+        }}]
+    }}
+    ```
+
+    ### **🔧 Category 4: Data Type & Serialization Errors**
+    ```python
+    # WRONG: Non-serializable numpy types
+    "data": numpy_array.tolist()
+    # FIXED: Convert to native Python types
+    "data": [float(x) for x in numpy_array]
+
+    # WRONG: Missing serializer in json.dumps
+    print(json.dumps(chart_data, indent=4))
+    # FIXED: Include custom serializer
+    print(json.dumps(chart_data, indent=4, default=serializer))
 
-    ## CHART.JS OUTPUT FORMATS
+    # WRONG: Pandas Series in JSON
+    "labels": df["category"]
+    # FIXED: Convert to list
+    "labels": df["category"].tolist()
+    ```
 
-    Be familiar with and fix errors related to these required formats:
+    ### **🔧 Category 5: Logic & Flow Errors**
+    ```python
+    # WRONG: Variable used before definition
+    chart_data = {{"data": final_df["amount"].tolist()}}
+    final_df = df.groupby("region")["amount"].sum().reset_index()
+    # FIXED: Define variable first
+    final_df = df.groupby("region")["amount"].sum().reset_index()
+    chart_data = {{"data": final_df["amount"].tolist()}}
+
+    # WRONG: Incorrect conditional logic
+    if chart_type == "line":
+        data_format = {{"x": x_vals, "y": y_vals}}
+    # FIXED: Proper data structure for line charts
+    if chart_type == "line":
+        data_format = {{"labels": x_vals, "datasets": [{{"data": y_vals}}]}}
+    ```
 
-    ### Standard Charts (line, bar, radar, polarArea, pie, doughnut):
+    ## EXACT CHART.JS FORMAT SPECIFICATIONS
+
+    ### **Standard Charts (line, bar, radar, polarArea, pie, doughnut):**
     ```json
     {{
-        "chartType": "<chart_type>",
-        "title": "<Chart Title>",
-        "xLabels": "<X-Axis Label>",    # Only include for "bar" or "line"
-        "yLabels": "<Y-Axis Label>",    # Only include for "bar" or "line"
+        "chartType": "chart_type_here",
+        "title": "Chart Title Here",
+        "xLabels": "X-Axis Label",    // ONLY for "bar" or "line"
+        "yLabels": "Y-Axis Label",    // ONLY for "bar" or "line"  
         "data": {{
-            "labels": <labels>,
+            "labels": ["label1", "label2", "label3"],
             "datasets": [
                 {{
-                    "label": "<dataset_name>",
-                    "data": <values>
+                    "label": "Dataset Name",
+                    "data": [value1, value2, value3]
                 }}
             ]
         }}
     }}
     ```
 
-    ### Multiple Dataset Charts:
+    ### **Multiple Dataset Charts:**
     ```json
     {{
-        "chartType": "<chart_type>",
-        "title": "<Chart Title>",
-        "xLabels": "<X-Axis Label>",
-        "yLabels": "<Y-Axis Label>",
+        "chartType": "chart_type_here",
+        "title": "Chart Title Here",
+        "xLabels": "X-Axis Label",    // ONLY for "bar" or "line"
+        "yLabels": "Y-Axis Label",    // ONLY for "bar" or "line"
         "data": {{
-            "labels": <labels>,
+            "labels": ["shared_label1", "shared_label2"],
             "datasets": [
                 {{
-                    "label": "<dataset1_name>",
-                    "data": <values1>
+                    "label": "Dataset 1 Name",
+                    "data": [value1, value2]
                 }},
                 {{
-                    "label": "<dataset2_name>",
-                    "data": <values2>
+                    "label": "Dataset 2 Name", 
+                    "data": [value3, value4]
                 }}
-                // Additional datasets as needed
             ]
         }}
     }}
     ```
 
-    ### Scatter & Bubble Charts:
+    ### **Scatter & Bubble Charts:**
     ```json
     {{
-        "chartType": "<chart_type>",
-        "title": "<Chart Title>",
-        "xLabels": "<X-Axis Label>",
-        "yLabels": "<Y-Axis Label>",
+        "chartType": "scatter", // or "bubble"
+        "title": "Chart Title Here",
+        "xLabels": "X-Axis Label",
+        "yLabels": "Y-Axis Label",
         "data": {{
             "datasets": [
                 {{
-                    "label": "<dataset_name>",
-                    "data": [ {{"x": value, "y": value}} ]  # Include 'r' for bubble chart
+                    "label": "Dataset Name",
+                    "data": [
+                        {{"x": x_value, "y": y_value}},
+                        {{"x": x_value, "y": y_value, "r": radius_value}}  // "r" ONLY for bubble
+                    ]
+                }}
+            ]
+        }}
+    }}
+    ```
+
+    ### **Multi-Category Scatter & Bubble Charts:**
+    ```json
+    {{
+        "chartType": "scatter", // or "bubble"
+        "title": "Chart Title Here", 
+        "xLabels": "X-Axis Label",
+        "yLabels": "Y-Axis Label",
+        "data": {{
+            "datasets": [
+                {{
+                    "label": "Category 1",
+                    "data": [{{"x": value, "y": value}}]
+                }},
+                {{
+                    "label": "Category 2", 
+                    "data": [{{"x": value, "y": value}}]
                 }}
             ]
         }}
     }}
     ```
 
-    ### Card Data:
+    ### **Card Data (Single Value Display):**
     ```json
     {{
         "chartType": "card",
-        "title": "<Chart Title>",
-        "label": "<Descriptive label>",
-        "data": "<Numeric value>"
+        "title": "Card Title Here",
+        "label": "Descriptive Label Text",
+        "data": numeric_value_only
     }}
     ```
 
-    ## COMMON ERROR CATEGORIES TO FIX
+    ## SYSTEMATIC ERROR RESOLUTION PROTOCOL
+
+    ### **Step 1: Error Localization**
+    - Identify EXACT line(s) causing the error
+    - Determine error category from the patterns above
+    - Isolate problematic code segment
+
+    ### **Step 2: Context Analysis** 
+    - Review metadata structure for column names and data types
+    - Understand user query intent for expected output format
+    - Trace data flow from fetch to final JSON output
+
+    ### **Step 3: Minimal Intervention**
+    - Apply ONLY the necessary change to fix the specific error
+    - Preserve ALL surrounding code exactly as written
+    - Maintain original variable names and logic structure
+
+    ### **Step 4: Format Verification**
+    - Ensure Chart.js JSON structure is pixel-perfect for the chart type
+    - Verify all required keys are present and correctly named
+    - Confirm data types match Chart.js expectations
+
+    ### **Step 5: Serialization Validation**
+    - Check that all data in JSON structure is serializable
+    - Ensure custom serializer is used in json.dumps call
+    - Convert pandas/numpy objects to native Python types if needed
+
+    ## ADVANCED ERROR SCENARIOS & FIXES
+
+    ### **Complex Scenario 1: Multi-Year Comparison with Date Issues**
+    ```python
+    # ERROR: DateTime conversion and grouping issues
+    sales_data = fetch_data("sales_data")
+    sales_data['year'] = sales_data['date'].dt.year  # Error: str has no attribute 'dt'
+
+    # FIX: Convert to datetime first
+    sales_data = fetch_data("sales_data") 
+    sales_data['date'] = pd.to_datetime(sales_data['date'])
+    sales_data['year'] = sales_data['date'].dt.year
+    ```
+
+    ### **Complex Scenario 2: Category-Based Scatter Plot Data Structure**
+    ```python
+    # ERROR: Incorrect scatter plot data organization
+    datasets = []
+    for category in categories:
+        cat_data = df[df['category'] == category]
+        datasets.append({{
+            "label": category,
+            "data": cat_data[['x_col', 'y_col']].values.tolist()  # Wrong format
+        }})
+
+    # FIX: Proper scatter plot coordinate format
+    datasets = []
+    for category in categories:
+        cat_data = df[df['category'] == category]
+        scatter_points = [{{"x": row['x_col'], "y": row['y_col']}} for _, row in cat_data.iterrows()]
+        datasets.append({{
+            "label": category,
+            "data": scatter_points
+        }})
+    ```
 
-    ### Environment & Setup Errors
-    - Missing or incorrect imports
-    - Incorrect or missing definition of metadata
-    - Incorrect access to `metadata` or `fetch_data`
-    - Redefinition of provided functions/variables
+    ### **Complex Scenario 3: Metadata Definition with Nested Structure**
+    ```python
+    # ERROR: Attempting to use undefined metadata
+    total_datasets = len(metadata.keys())  # NameError: name 'metadata' is not defined
 
-    ### Data Processing Errors
-    - Incorrect column references or typos in column names
-    - Invalid pandas operations or chaining
-    - Incorrect groupby, filter, or aggregation operations
-    - Date formatting or conversion issues
-    - Missing reset_index() after aggregation
+    # FIX: Define metadata exactly as provided in input
+    metadata = {{
+        "sales_data": {{
+            "description": "Sales records",
+            "shape": [1000, 5],
+            "columns": [
+                {{"name": "date", "type": "datetime64", "description": "Sale date"}},
+                {{"name": "amount", "type": "float64", "description": "Sale amount"}}
+            ]
+        }},
+        "customer_data": {{
+            "description": "Customer info", 
+            "shape": [500, 3],
+            "columns": [
+                {{"name": "id", "type": "int64", "description": "Customer ID"}}
+            ]
+        }}
+    }}
+    total_datasets = len(metadata.keys())
+    ```
 
-    ### Chart.js Structure Errors
-    - Incorrect JSON structure for the chart type
-    - Missing or misnamed JSON keys
-    - Nested dictionary/list structure issues
-    - Data type inconsistencies (lists vs. single values)
-    - Serialization issues with complex objects
+    ## CRITICAL OUTPUT REQUIREMENTS
 
-    ### Pandas & Data Manipulation Errors
-    - Index alignment problems
-    - Type conversion errors
-    - NaN handling issues
-    - Incorrect boolean masking or filtering syntax
-    - Improper reindexing or filling missing values
+    **YOUR RESPONSE MUST CONTAIN:**
+    1. ONLY the corrected Python code block
+    2. NO additional text whatsoever
+    3. NO explanations or comments
+    4. NO markdown formatting around the code
+    5. Complete, executable Python script
 
-    ### Output Formatting Errors
-    - Improper JSON dumps parameters
-    - Missing or incorrect nested JSON structures
-    - Type conversion issues in the final output
-    - Non-serializable objects in the output
-    - Incorrect use of the custom serializer function
+    **YOUR RESPONSE MUST NOT CONTAIN:**
+    1. Any explanatory text
+    2. Comments about what was changed
+    3. Suggestions for improvements  
+    4. Alternative solutions
+    5. Error analysis or descriptions
 
-    ## APPROACH TO ERROR RESOLUTION
+    ## INPUT DATA STRUCTURE
 
-    1. **Identify Error Type**: Precisely locate the error in the code.
-    2. **Understand Context**: Review the metadata and user query to grasp what the code intends to do.
-    3. **Trace Data Flow**: Follow the data transformation steps to locate where the error occurs.
-    4. **Apply Minimal Fix**: Make the smallest possible change to fix the issue.
-    5. **Verify Chart.js Compatibility**: Ensure the fix maintains proper Chart.js JSON format.
-    6. **Check Serialization**: If the error involves JSON serialization, ensure the custom serializer is properly used.
+    ### Error Message:
+    {error_message}
 
-    ## JSON SERIALIZATION HANDLING
+    ### Code with Errors:
+    {code_with_errors}
 
-    - The environment provides a custom `serializer` function that handles non-standard JSON types (like NumPy types, pandas objects, etc.)
-    - When outputting JSON, use `json.dumps(chart_data, indent=4, default=serializer)` to ensure proper serialization
-    - Do not modify or redefine the serializer function, it is already available in the environment
-    - If serialization errors occur, focus on converting problematic data types before they reach the serializer rather than changing the serializer itself
+    ### Metadata Context:
+    {metadata_context}
 
-    ## OUTPUT FORMAT
+    ### User Query:
+    {user_query}
 
-    Return ONLY the corrected code block with no additional text. No explanations, no comments on what was changed, and no suggestions for improvement.
+    ---
 
-    Remember: Your entire response should be just the fixed code block. Nothing more.
+    **EXECUTE WITH ABSOLUTE PRECISION - ZERO MARGIN FOR ERROR**
 
 panelChartDataCode: |
     def getDataForChart(projectId: str, chartType: str, xAxis: str, yAxis: str, aggregationMetric: str, tablesUsed: list[str] | str, joinTypes: list[str] | None, blendOn: list[str] | None):
-        import fireducks.pandas as pd
+        import pandas as pd
         import json
         if type(tablesUsed) == list:
             allTables = [fetch_data(projectId, x) for x in tablesUsed]

requirements.txt

@@ -25,5 +25,5 @@ langchain-cerebras
 langchain-experimental
 fastapi-analytics
 langgraph
-fireducks
+pyarrow
 dataclasses