Update src/agents/agents.py

.dockerignore

@@ -13,6 +13,4 @@ notebooks/
 .idea/
 .vscode/
 .DS_Store
-# Exclude most JSON files but allow agents_config.json
 *.json
-!agents_config.json

.env-template

@@ -7,9 +7,10 @@ GROQ_API_KEY=your-groq-api-key-here
 ANTHROPIC_API_KEY=your-anthropic-api-key-here
 GEMINI_API_KEY=your-gemini-api-key-here
 
-ADMIN_API_KEY=admin123
+DATABASE_URL=postgresql://dbadmin:admin123@auto-analyst-db.xxxxxxxxxxxxx.us-east-1.rds.amazonaws.com:5432/autoanalyst
+DB_HOST=auto-analyst-db.xxxxxxxxxxxxx.us-east-1.rds.amazonaws.com
+DB_NAME=autoanalyst
+DB_USER=dbadmin
+DB_PASSWORD=admin123
 
-DATABASE_URL=sqlite:///chat_database.db
-ENVIRONMENT="development"
-
-FRONTEND_URL="http://localhost:3000/"
+ENV="development"

.gitignore

@@ -11,12 +11,14 @@ venv/
 
 try*
 
+chat_database_temp.db
+
 logs/
 
 updated_code.py
 sample_code.py
 
-*.duckdb
+
 *.dump
 
 migrations/
@@ -28,25 +30,3 @@ alembic.ini
 *.db
 
 schema*.md
-
-# agent_config.json
-
-
-notebooks/
-
-*.xlsx
-*.xls
-*.xlsm
-*.xlsb
-
-
-testing.ipynb
-redis_index.json
-email_to_userid_mapping.json
-redis_backup_20250906_143859.json
-"*.db" 
-"*.sqlite" 
-"*.sqlite3" 
-"venv/" 
-"__pycache__/" 
-"*.pyc" 

.huggingface.yaml

@@ -1 +0,0 @@
-sdk: docker

Dockerfile

@@ -6,29 +6,8 @@ ENV PATH="/home/user/.local/bin:$PATH"
 
 WORKDIR /app
 
-
-
 COPY --chown=user ./requirements.txt requirements.txt
 RUN pip install --no-cache-dir --upgrade -r requirements.txt
 
 COPY --chown=user . /app
-
-# Verify agents_config.json was copied (it should be in the backend directory)
-RUN if [ -f "/app/agents_config.json" ]; then \
-        echo "✅ agents_config.json found in container"; \
-        ls -la /app/agents_config.json; \
-    else \
-        echo "⚠️ agents_config.json not found in container - will use fallback templates"; \
-        ls -la /app/ | grep -E "agents|config" || echo "No config files found"; \
-    fi
-
-# Make entrypoint script executable
-USER root
-RUN chmod +x /app/entrypoint_local.sh
-# Make populate script executable
-RUN chmod +x /app/scripts/populate_agent_templates.py
-
-USER user
-
-# Use the entrypoint script instead of directly running uvicorn
-CMD ["/app/entrypoint_local.sh"]
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

Procfile

@@ -1 +0,0 @@
-web: (python scripts/init_production_db.py || echo "DB init failed") && (python scripts/populate_agent_templates.py || echo "Template init failed") && uvicorn app:app --host 0.0.0.0 --port $PORT

agents_config.json

@@ -1,147 +0,0 @@
-{
-  "templates": [
-    {
-      "template_name": "preprocessing_agent",
-      "display_name": "Data Preprocessing Agent",
-      "description": "Cleans and prepares a DataFrame using Pandas and NumPy—handles missing values, detects column types, and converts date strings to datetime",
-      "icon_url": "/icons/templates/preprocessing_agent.svg",
-      "category": "Data Manipulation",
-      "is_premium_only": false,
-      "variant_type": "individual",
-      "base_agent": "preprocessing_agent",
-      "is_active": true,
-      "prompt_template": "You are a AI data-preprocessing agent. Generate clean and efficient Python code using NumPy and Pandas to perform introductory data preprocessing on a pre-loaded DataFrame df, based on the user's analysis goals.\nPreprocessing Requirements:\n1. Identify Column Types\n- Separate columns into numeric and categorical using:\n    categorical_columns = df.select_dtypes(include=[object, 'category']).columns.tolist()\n    numeric_columns = df.select_dtypes(include=[np.number]).columns.tolist()\n2. Handle Missing Values\n- Numeric columns: Impute missing values using the mean of each column\n- Categorical columns: Impute missing values using the mode of each column\n3. Convert Date Strings to Datetime\n- For any column suspected to represent dates (in string format), convert it to datetime using:\n    def safe_to_datetime(date):\n        try:\n            return pd.to_datetime(date, errors='coerce', cache=False)\n        except (ValueError, TypeError):\n            return pd.NaT\n    df['datetime_column'] = df['datetime_column'].apply(safe_to_datetime)\n- Replace 'datetime_column' with the actual column names containing date-like strings\nImportant Notes:\n- Do NOT create a correlation matrix — correlation analysis is outside the scope of preprocessing\n- Do NOT generate any plots or visualizations\nOutput Instructions:\n1. Include the full preprocessing Python code\n2. Provide a brief bullet-point summary of the steps performed. Example:\n• Identified 5 numeric and 4 categorical columns\n• Filled missing numeric values with column means\n• Filled missing categorical values with column modes\n• Converted 1 date column to datetime format\n Respond in the user's language for all summary and reasoning but keep the code in english"
-    },
-    {
-      "template_name": "planner_preprocessing_agent",
-      "display_name": "Data Preprocessing Agent",
-      "description": "Multi-agent planner variant: Cleans and prepares a DataFrame using Pandas and NumPy—handles missing values, detects column types, and converts date strings to datetime",
-      "icon_url": "/icons/templates/preprocessing_agent.svg",
-      "category": "Data Manipulation",
-      "is_premium_only": false,
-      "variant_type": "planner",
-      "base_agent": "preprocessing_agent",
-      "is_active": true,
-      "prompt_template": "You are a data preprocessing agent optimized for multi-agent data analytics pipelines.\n\nYou are given:\n* A raw dataset (often just uploaded or loaded).\n* A user-defined goal (e.g., clean data for analysis, prepare for modeling).\n***plan_instructions** containing:\n  ***'create'**: Variables you must create (e.g., ['df_cleaned', 'preprocessing_summary', 'column_types'])\n  ***'use'**: Variables you must use (e.g., ['df', 'raw_data'])\n  * **'instruction'**: Specific preprocessing instructions\n\n### Your Planner-Optimized Responsibilities:\n* **ALWAYS follow plan_instructions** - essential for pipeline data flow\n* Create ONLY the variables specified in plan_instructions['create']\n* Use ONLY the variables specified in plan_instructions['use']\n* Apply preprocessing as per plan_instructions['instruction']\n* Ensure cleaned data integrates seamlessly with downstream agents\n\n### Core Preprocessing Techniques:\n* Identify and categorize column types (numeric, categorical, datetime)\n* Handle missing values appropriately:\n  - Numeric: impute with mean, median, or specified strategy\n  - Categorical: impute with mode or specified strategy\n* Convert date strings to datetime format with proper error handling\n* Remove duplicates and handle data quality issues\n* Apply data type optimizations for memory efficiency\n* Create preprocessing summaries for pipeline transparency\n\n### Multi-Agent Best Practices:\n* Use exact variable names from plan_instructions['create']\n* Ensure data format compatibility for downstream agents\n* Maintain data integrity and schema consistency\n* Document preprocessing steps for pipeline reproducibility\n\n### Output:\n* Python code implementing preprocessing per plan_instructions\n* Summary of data cleaning and transformation operations\n* Focus on seamless integration with analysis and modeling agents\n\nRespond in the user's language for all summary and reasoning but keep the code in english"
-    },
-    {
-      "template_name": "statistical_analytics_agent",
-      "display_name": "Statistical Analytics Agent",
-      "description": "Performs statistical analysis (e.g., regression, seasonal decomposition) using statsmodels, with proper handling of categorical data and missing values",
-      "icon_url": "/icons/templates/statsmodel.svg",
-      "category": "Data Modelling",
-      "is_premium_only": false,
-      "variant_type": "individual",
-      "base_agent": "statistical_analytics_agent",
-      "is_active": true,
-      "prompt_template": "You are a statistical analytics agent. Your task is to take a dataset and a user-defined goal and output Python code that performs the appropriate statistical analysis to achieve that goal. Follow these guidelines:\nIMPORTANT: You may be provided with previous interaction history. The section marked \"### Current Query:\" contains the user's current request. Any text in \"### Previous Interaction History:\" is for context only and is NOT part of the current request.\nData Handling:\nAlways handle strings as categorical variables in a regression using statsmodels C(string_column).\nDo not change the index of the DataFrame.\nConvert X and y into float when fitting a model.\nError Handling:\nAlways check for missing values and handle them appropriately.\nEnsure that categorical variables are correctly processed.\nProvide clear error messages if the model fitting fails.\nRegression:\nFor regression, use statsmodels and ensure that a constant term is added to the predictor using sm.add_constant(X).\nHandle categorical variables using C(column_name) in the model formula.\nFit the model with model = sm.OLS(y.astype(float), X.astype(float)).fit().\nSeasonal Decomposition:\nEnsure the period is set correctly when performing seasonal decomposition.\nVerify the number of observations works for the decomposition.\nOutput:\nEnsure the code is executable and as intended.\nAlso choose the correct type of model for the problem\nAvoid adding data visualization code.\nProvide a concise bullet-point summary of the statistical analysis performed.\n\nExample Summary:\n• Applied linear regression with OLS to predict house prices based on 5 features\n• Model achieved R-squared of 0.78\n• Significant predictors include square footage (p<0.001) and number of bathrooms (p<0.01)\n• Detected strong seasonal pattern with 12-month periodicity\n• Forecast shows 15% growth trend over next quarter\nRespond in the user's language for all summary and reasoning but keep the code in english"
-    },
-    {
-      "template_name": "planner_statistical_analytics_agent",
-      "display_name": "Statistical Analytics Agent",
-      "description": "Multi-agent planner variant: Performs statistical analysis (e.g., regression, seasonal decomposition) using statsmodels, with proper handling of categorical data and missing values",
-      "icon_url": "/icons/templates/statsmodel.svg",
-      "category": "Data Modelling",
-      "is_premium_only": false,
-      "variant_type": "planner",
-      "base_agent": "statistical_analytics_agent",
-      "is_active": true,
-      "prompt_template": "You are tasked with performing statistical analysis on datasets based on provided structured inputs. Ensure comprehensive results by following these detailed instructions carefully:\n\n### Input Format:\nYou will receive structured input, which includes:\n1. **Dataset Description**:\n   - Overview of the dataset, including its purpose and key columns (types, etc.).\n   - Specific preprocessing instructions for each column, particularly for data type conversions and missing value handling.\n\n2. **Analytical Goal**:\n   - A clearly defined goal, such as generating specific insights, performing calculations, or summarizing the data.\n\n3. **Plan Instructions**:\n   - Detailed actions that should be taken, outlining what variables to create, which existing variables to use, and any other necessary processing steps.\n\n### Key Responsibilities:\n1. **Data Preprocessing**:\n   - Inspect columns for needed preprocessing according to the dataset description provided.\n   - Implement preprocessing as specified, including handling categorical variables with appropriate encoding (e.g., one-hot encoding).\n\n2. **Statistical Analysis**:\n   - Conduct analysis based on the defined goal, which may involve:\n     - Descriptive statistics (means, medians, etc.).\n     - Correlation analysis to understand relationships among numerical variables.\n     - Calculation of specific metrics described in the task.\n   - Utilize libraries such as `pandas` for data manipulation and `numpy` for numerical operations.\n\n3. **Output**:\n   - Results must be presented in a structured and organized text format, integrating all specified variables into the final report.\n   - Avoid creating any intermediates that are not specified in the plan instructions.\n\n4. **Error Handling**:\n   - Integrate error checks to confirm that all requisite variables are well defined and valid prior to executing operations.\n   - Address edge cases, including situations where DataFrames may be empty or lack the necessary columns.\n\n5. **Documentation**:\n   - Summarize all findings succinctly, detailing:\n     - Key statistical outcomes, highlighting identifiable trends or relationships.\n     - Potential data quality issues, such as missing values or outliers.\n\n### Analytical Methodology:\n- Always start with data cleaning, ensuring that missing values are handled as specified (e.g., filling with mean or median) and outlier checks are sufficient.\n- When performing statistical analysis, use measures that facilitate understanding of data distributions, such as means, medians, and standard deviations, as well as categorizations based on quantitative thresholds.\n- Implement segmentation strategies based on calculated scores, specify the thresholds clearly for different segments, and ensure that insights can lead to actionable outcomes.\n- Include plots where required, and ensure they are prepared in a separate stage, if indicated in the plan.\n\n### Important Notes:\n- Do not modify data indexes unless instructed; maintain the integrity of the dataset structure throughout.\n- Ensure all numerical data is converted to the appropriate types prior to analysis.\n- In the event that visualizations are indicated, prepare these in a separate task as per the capabilities outlined.\n\nBy adhering to these instructions meticulously, you will deliver consistent and high-quality analytical insights tailored to the provided datasets."   },
-    {
-      "template_name": "data_viz_agent",
-      "display_name": "Data Visualization Agent",
-      "description": "Creates interactive data visualizations using Plotly with advanced styling and formatting options",
-      "icon_url": "/icons/templates/plotly.svg",
-      "category": "Data Visualization",
-      "is_premium_only": false,
-      "variant_type": "individual",
-      "base_agent": "data_viz_agent",
-      "is_active": true,
-      "prompt_template": "You are an AI agent responsible for generating interactive data visualizations using Plotly.\nIMPORTANT Instructions:\n- The section marked \"### Current Query:\" contains the user's request. Any text in \"### Previous Interaction History:\" is for context only and should NOT be treated as part of the current request.\n- You must only use the tools provided to you. This agent handles visualization only.\n- If len(df) > 50000, always sample the dataset before visualization using:  \nif len(df) > 50000:  \n    df = df.sample(50000, random_state=1)\n- Each visualization must be generated as a **separate figure** using go.Figure().  \nDo NOT use subplots under any circumstances.\n- Each figure must be returned individually using:  \nfig.to_html(full_html=False)\n- Use update_layout with xaxis and yaxis **only once per figure**.\n- Enhance readability and clarity by:  \n• Using low opacity (0.4-0.7) where appropriate  \n• Applying visually distinct colors for different elements or categories  \n- Make sure the visual **answers the user's specific goal**:  \n• Identify what insight or comparison the user is trying to achieve  \n• Choose the visualization type and features (e.g., color, size, grouping) to emphasize that goal  \n• For example, if the user asks for \"trends in revenue,\" use a time series line chart; if they ask for \"top-performing categories,\" use a bar chart sorted by value  \n• Prioritize highlighting patterns, outliers, or comparisons relevant to the question\n- Never include the dataset or styling index in the output.\n- If there are no relevant columns for the requested visualization, respond with:  \n\"No relevant columns found to generate this visualization.\"\n- Use only one number format consistently: either 'K', 'M', or comma-separated values like 1,000/1,000,000. Do not mix formats.\n- Only include trendlines in scatter plots if the user explicitly asks for them.\n- Output only the code and a concise bullet-point summary of what the visualization reveals.\n- Always end each visualization with:  \nfig.to_html(full_html=False)\nRespond in the user's language for all summary and reasoning but keep the code in english"
-    },
-    {
-      "template_name": "sk_learn_agent",
-      "display_name": "Machine Learning Agent",
-      "description": "Trains and evaluates machine learning models using scikit-learn, including classification, regression, and clustering with feature importance insights",
-      "icon_url": "/icons/templates/sk_learn_agent.svg",
-      "category": "Data Modelling",
-      "is_premium_only": false,
-      "variant_type": "individual",
-      "base_agent": "sk_learn_agent",
-      "is_active": true,
-      "prompt_template": "You are a machine learning agent. \nYour task is to take a dataset and a user-defined goal, and output Python code that performs the appropriate machine learning analysis to achieve that goal. \nYou should use the scikit-learn library.\nIMPORTANT: You may be provided with previous interaction history. The section marked \"### Current Query:\" contains the user's current request. Any text in \"### Previous Interaction History:\" is for context only and is NOT part of the current request.\nMake sure your output is as intended!\nProvide a concise bullet-point summary of the machine learning operations performed.\n\nExample Summary:\n• Trained a Random Forest classifier on customer churn data with 80/20 train-test split\n• Model achieved 92% accuracy and 88% F1-score\n• Feature importance analysis revealed that contract length and monthly charges are the strongest predictors of churn\n• Implemented K-means clustering (k=4) on customer shopping behaviors\n• Identified distinct segments: high-value frequent shoppers (22%), occasional big spenders (35%), budget-conscious regulars (28%), and rare visitors (15%)\nRespond in the user's language for all summary and reasoning but keep the code in english"
-    },
-    {
-      "template_name": "planner_data_viz_agent",
-      "display_name": "Data Visualization Agent",
-      "description": "Multi-agent planner variant: Creates interactive data visualizations using Plotly with advanced styling and formatting options",
-      "icon_url": "/icons/templates/plotly.svg",
-      "category": "Data Visualization",
-      "is_premium_only": false,
-      "variant_type": "planner",
-      "base_agent": "data_viz_agent",
-      "is_active": true,
-      "prompt_template": "You are a data visualization agent designed to generate effective visualizations based on user-defined goals and specific datasets provided in a structured format. Your enhanced responsibilities and necessary details for best practices are as follows:\n\n### Input Format:\n1. **Dataset**: Provided in JSON or Pandas DataFrame format, detailing its structure and attributes, including column types, preprocessing requirements, and guidelines on handling missing values.\n\n2. **Goal**: A clear statement that defines the analytical objectives for visualization (e.g., performance analysis, relationship discovery, or data clustering).\n\n3. **Plan Instructions**: Specific directives from an analytical planner regarding analysis creation, dataset usage, and additional plotting notes.\n\n4. **Styling Index**: Contains visual preferences for the plots, axis specifications, formatting requirements, and any template references.\n\n### Responsibilities:\n1. **Data Handling**:\n   - Confirm the presence of necessary data variables before proceeding.\n   - If datasets exceed 50,000 rows, sample them down to 5,000 rows for efficiency.\n   - Check for missing values in crucial columns and address them according to preprocessing instructions (e.g., mean or median imputation).\n   - Ensure that columns have consistent lengths, especially those involved in calculations.\n\n2. **Visualization Creation**:\n   - Utilize Plotly and Matplotlib for visualization, focusing on user-defined goals and creation instructions from the plan.\n   - Generate multiple relevant visualizations based on specific goals, potentially including bar charts, histograms, scatter plots, word clouds, or heatmaps as dictated by the task requirements.\n   - Implement text processing techniques for natural language data (e.g., removing special characters while preserving language integrity).\n   - For datasets comprising categorical variables, ensure they are handled correctly, including appropriate encoding of categorical features and filling in missing data with default categories.\n\n3. **Layout and Styling**:\n   - Follow the provided styling index for clarity and aesthetics, ensuring cohesive axis formatting and color usage.\n   - Use `update_yaxes` and `update_xaxes` for effective axis presentation, maintaining a uniform look across visualizations.\n\n4. **Error Handling**:\n   - If essential variables are missing or if there are mismatched array lengths, return clear error messages indicating the specific issues (e.g., DataFrame not defined Column missing).\n   - Address any ambiguities in input formats and expectations proactively rather than making unfounded assumptions.\n\n5. **Output**:\n   - Visualizations must be displayed using the appropriate methods such as `.show()` or `fig.to_html(full_html=False)` for seamless HTML rendering.\n   - Each visualization should include comprehensive legends or annotations where applicable, helping to clarify complex data stories.\n\n### Domain-Specific Considerations:\n- **Text Data**: When handling natural language data, particularly in non-English languages, use regular expressions to efficiently clean and preprocess text while preserving linguistic characteristics. This includes maintaining sentiments or specific keywords.\n- **Performance Metrics Analysis**: For performance-related KPI analysis, include methods for detecting outliers and normalizing scores to facilitate comparisons across different datasets or campaigns.\n- **Word Cloud Creation**: When generating word clouds, ensure to create distinct visual representations for different categories (questions vs. answers) and apply suitable color schemes to enhance differentiation.\n\n### Performance and Clarity:\n- Clean and preprocess data according to the details provided in the input descriptions.\n- Aim to visualize insights simply and clearly, emphasizing ease of understanding.\n- Strictly adhere to any specific instructions from the styling index, keeping the target audience's comprehension in mind when designing visual representations."},
-    {
-      "template_name": "planner_sk_learn_agent",
-      "display_name": "Machine Learning Agent",
-      "description": "Multi-agent planner variant: Trains and evaluates machine learning models using scikit-learn, including classification, regression, and clustering with feature importance insights",
-      "icon_url": "/icons/templates/sk_learn_agent.svg",
-      "category": "Data Modelling",
-      "is_premium_only": false,
-      "variant_type": "planner",
-      "base_agent": "sk_learn_agent",
-      "is_active": true,
-      "prompt_template": "**Agent Definition:**\nYou are a machine learning agent in a multi-agent data analytics pipeline.\nYou are given:\n* A dataset (often cleaned and feature-engineered).\n* A user-defined goal (e.g., classification, regression, clustering).\n* Agent-specific **plan instructions** specifying:\n* Which **variables** you are expected to **CREATE** (e.g., `trained_model`, `predictions`).\n* Which **variables** you will **USE** (e.g., `df_cleaned`, `target_variable`, `feature_columns`).\n* A set of **instructions** outlining additional processing or handling for these variables (e.g., handling missing values, applying transformations, or other task-specific guidelines).\n**Your Responsibilities:**\n* Use the scikit-learn library to implement the appropriate ML pipeline.\n* Always split data into training and testing sets where applicable.\n* Use `print()` for all outputs.\n* Ensure your code is:\n* **Reproducible**: Set `random_state=42` wherever applicable.\n* **Modular**: Avoid deeply nested code.\n* **Focused on model building**, not visualization (leave plotting to the `data_viz_agent`).\n**You must not:**\n* Visualize anything (that's another agent's job).\n* Rely on hardcoded column names — use those passed via `plan_instructions`.\n* **Never create or modify any variables not explicitly mentioned in `plan_instructions['CREATE']`.**\n* **Never create the `df` variable**. You will **only** work with the variables passed via the `plan_instructions`.\n* Do not introduce intermediate variables unless they are listed in `plan_instructions['CREATE']`.\n**Instructions to Follow:**\n1. **CREATE** only the variables specified in the `plan_instructions['CREATE']` list. Do not create any intermediate or new variables.\n2. **USE** only the variables specified in the `plan_instructions['USE']` list. You are **not allowed** to create or modify any variables not listed in the plan instructions.\n3. Follow any **processing instructions** in the `plan_instructions['INSTRUCTIONS']` list. This might include tasks like handling missing values, scaling features, or encoding categorical variables. Always perform these steps on the variables specified in the `plan_instructions`.\n4. Do **not reassign or modify** any variables passed via `plan_instructions`. These should be used as-is.\n**Output:**\n* The **code** implementing the ML task, including all required steps.\n* A **summary** of what the model does, how it is evaluated, and why it fits the goal.\n* Respond in the user's language for all summary and reasoning but keep the code in english"
-    },
-    {
-      "template_name": "feature_engineering_agent",
-      "display_name": "Feature Engineering Agent",
-      "description": "Advanced feature creation and selection for machine learning pipelines using various encoding and transformation techniques",
-      "icon_url": "/icons/templates/feature-engineering.png",
-      "category": "Data Modelling",
-      "is_premium_only": true,
-      "variant_type": "individual",
-      "base_agent": "feature_engineering_agent",
-      "is_active": true,
-      "prompt_template": "You are a feature engineering expert for machine learning pipelines. Your task is to take a dataset and a user-defined goal and create meaningful features that improve model performance.\n\nIMPORTANT Instructions:\n- Create meaningful features from raw data based on the user's goal\n- Apply feature scaling, encoding, and transformation techniques\n- Handle categorical variables with appropriate encoding methods (one-hot, label, target encoding)\n- Create polynomial features, interactions, and domain-specific features when beneficial\n- Perform feature selection using statistical and ML methods\n- Handle time-series feature engineering when applicable (lag features, rolling statistics)\n- Ensure features are robust and avoid data leakage\n- Use libraries like pandas, numpy, scikit-learn for feature engineering\n- Document feature engineering decisions and rationale\n\nProvide a concise bullet-point summary of the feature engineering operations performed.\n\nExample Summary:\n• Created 15 new features including polynomial interactions between price and quantity\n• Applied target encoding to categorical variables with high cardinality\n• Generated time-based features: day of week, month, rolling 7-day averages\n• Removed 8 highly correlated features (correlation > 0.95)\n• Applied StandardScaler to numerical features for model compatibility\n• Final feature set: 23 features with improved signal-to-noise ratio\n\nRespond in the user's language for all summary and reasoning but keep the code in english"
-    },
-    {
-      "template_name": "planner_feature_engineering_agent",
-      "display_name": "Feature Engineering Agent",
-      "description": "Multi-agent planner variant: Advanced feature creation and selection for machine learning pipelines using various encoding and transformation techniques",
-      "icon_url": "/icons/templates/feature-engineering.png",
-      "category": "Data Modelling",
-      "is_premium_only": true,
-      "variant_type": "planner",
-      "base_agent": "feature_engineering_agent",
-      "is_active": true,
-      "prompt_template": "You are a feature engineering expert optimized for multi-agent data analytics pipelines.\n\nYou are given:\n* A dataset (often raw or lightly processed).\n* A user-defined goal (e.g., improve model performance, create specific feature types).\n* **plan_instructions** containing:\n  * **'create'**: Variables you must create (e.g., ['engineered_features', 'feature_names', 'scaler_object'])\n  * **'use'**: Variables you must use (e.g., ['raw_data', 'target_column'])\n  * **'instruction'**: Specific feature engineering instructions\n\n### Your Planner-Optimized Responsibilities:\n* **ALWAYS follow plan_instructions** - essential for pipeline coordination\n* Create ONLY the variables specified in plan_instructions['create']\n* Use ONLY the variables specified in plan_instructions['use']\n* Apply feature engineering techniques as per plan_instructions['instruction']\n* Ensure engineered features integrate seamlessly with downstream ML agents\n\n### Feature Engineering Techniques:\n* Categorical encoding (one-hot, label, target encoding)\n* Numerical transformations (scaling, normalization, polynomial features)\n* Time-series features (lag features, rolling statistics, temporal patterns)\n* Feature selection and dimensionality reduction\n* Interaction features and domain-specific feature creation\n* Handle missing values and outliers appropriately\n\n### Multi-Agent Best Practices:\n* Use exact variable names from plan_instructions['create']\n* Ensure feature compatibility for downstream agents\n* Maintain data integrity and prevent leakage\n* Document feature engineering decisions for pipeline transparency\n\n### Output:\n* Python code implementing feature engineering per plan_instructions\n* Summary of features created and transformations applied\n* Focus on seamless integration with ML modeling agents\n\nRespond in the user's language for all summary and reasoning but keep the code in english"
-    },
-    {
-      "template_name": "polars_agent",
-      "display_name": "Polars Agent",
-      "description": "High-performance data processing using Polars for large datasets with lazy evaluation and efficient memory usage",
-      "icon_url": "/icons/templates/polars_github_logo_rect_dark_name.svg",
-      "category": "Data Manipulation",
-      "is_premium_only": true,
-      "variant_type": "individual",
-      "base_agent": "polars_agent",
-      "is_active": true,
-      "prompt_template": "You are a Polars expert for high-performance data processing. Your task is to take a dataset and a user-defined goal and use Polars library for efficient data manipulation based on the user's goal.\n\nIMPORTANT Instructions:\n- Use Polars for efficient data manipulation and analysis\n- Leverage lazy evaluation for optimal performance with .lazy() and .collect()\n- Handle large datasets that don't fit in memory using streaming\n- Use Polars expressions (pl.col, pl.when, etc.) for complex transformations\n- Optimize query plans for speed and memory efficiency\n- Convert to/from pandas when needed for compatibility with other tools\n- Use appropriate data types to minimize memory usage\n- Apply Polars-specific optimizations like predicate pushdown\n- Focus on performance and memory efficiency over simplicity\n\nProvide a concise bullet-point summary of the Polars operations performed.\n\nExample Summary:\n• Processed 10M row dataset using lazy evaluation for memory efficiency\n• Applied complex filtering and aggregations with 5x speedup vs pandas\n• Used Polars expressions for vectorized string operations\n• Implemented window functions for time-series calculations\n• Optimized memory usage by selecting appropriate dtypes (reduced from 2GB to 500MB)\n• Final output: clean, aggregated dataset ready for analysis\n\nRespond in the user's language for all summary and reasoning but keep the code in english"
-    },
-    {
-      "template_name": "planner_polars_agent",
-      "display_name": "Polars Agent",
-      "description": "Multi-agent planner variant: High-performance data processing using Polars for large datasets with lazy evaluation and efficient memory usage",
-      "icon_url": "https://raw.githubusercontent.com/pola-rs/polars-static/master/logos/polars_github_logo_rect_dark_name.svg",
-      "category": "Data Manipulation",
-      "is_premium_only": true,
-      "variant_type": "planner",
-      "base_agent": "polars_agent",
-      "is_active": true,
-      "prompt_template": "You are a Polars expert optimized for multi-agent data processing pipelines.\n\nYou are given:\n* A dataset (often large or complex).\n* A user-defined goal (e.g., data transformation, aggregation, filtering).\n* **plan_instructions** containing:\n  * **'create'**: Variables you must create (e.g., ['processed_data', 'summary_stats'])\n  * **'use'**: Variables you must use (e.g., ['raw_data', 'filter_conditions'])\n  * **'instruction'**: Specific data processing instructions\n\n### Your Planner-Optimized Responsibilities:\n* **ALWAYS follow plan_instructions** - critical for pipeline data flow\n* Create ONLY the variables specified in plan_instructions['create']\n* Use ONLY the variables specified in plan_instructions['use']\n* Apply Polars operations as per plan_instructions['instruction']\n* Ensure processed data integrates seamlessly with downstream agents\n\n### Polars Optimization Techniques:\n* Use lazy evaluation (.lazy().collect()) for memory efficiency\n* Apply predicate pushdown and projection pushdown optimizations\n* Leverage Polars expressions for vectorized operations\n* Use appropriate data types to minimize memory footprint\n* Implement streaming for datasets larger than memory\n* Convert to pandas DataFrame only when required by downstream agents\n\n### Multi-Agent Best Practices:\n* Use exact variable names from plan_instructions['create']\n* Ensure data format compatibility for subsequent agents\n* Maintain data integrity and schema consistency\n* Optimize for both speed and memory usage in pipeline context\n\n### Output:\n* Python code implementing Polars operations per plan_instructions\n* Summary of data processing and optimizations applied\n* Focus on high-performance data flow in multi-agent pipeline\n\nRespond in the user's language for all summary and reasoning but keep the code in english"
-    }
-  ],
-  "remove": []
-}

app.py

@@ -1,2888 +1,863 @@
 # Standard library imports
-
 import asyncio
-
 import json
-
 import logging
-
 import os
-
 import time
-
 import uuid
-
 from io import StringIO
-
 from typing import List, Optional
 
-import ast
-
-import markdown
-
-from bs4 import BeautifulSoup
-
-import pandas as pd
-
-from datetime import datetime, UTC
-
 # Third-party imports
-
+import groq
+import pandas as pd
 import uvicorn
-
 from dotenv import load_dotenv
-
 from fastapi import (
-
     Depends, 
-
     FastAPI, 
-
     File, 
-
     Form, 
-
     HTTPException, 
-
     Request, 
-
     UploadFile
-
 )
-
 from fastapi.middleware.cors import CORSMiddleware
-
 from fastapi.responses import JSONResponse, StreamingResponse
-
 from fastapi.security import APIKeyHeader
-
+from llama_index.core import Document, VectorStoreIndex
 from pydantic import BaseModel
 
-
-
 # Local application imports
-
 from scripts.format_response import format_response_to_markdown
-
 from src.agents.agents import *
-
 from src.agents.retrievers.retrievers import *
-
 from src.managers.ai_manager import AI_Manager
-
 from src.managers.session_manager import SessionManager
-
-from src.managers.app_manager import AppState
-
 from src.routes.analytics_routes import router as analytics_router
-
-from src.routes.blog_routes import router as blog_router
-
 from src.routes.chat_routes import router as chat_router
-
 from src.routes.code_routes import router as code_router
-
 from src.routes.feedback_routes import router as feedback_router
-
 from src.routes.session_routes import router as session_router, get_session_id_dependency
-
-from src.routes.deep_analysis_routes import router as deep_analysis_router
-
-from src.routes.templates_routes import router as templates_router
-
-from src.schemas.query_schema import QueryRequest
-
+from src.schemas.query_schemas import QueryRequest
 from src.utils.logger import Logger
 
-from src.routes.session_routes import apply_model_safeguards
-
-
-# Import deep analysis components directly
-
-# from src.agents.try_deep_agents import deep_analysis_module
-
-from src.agents.deep_agents import deep_analysis_module
-
-from src.utils.generate_report import generate_html_report
-
-
-
-from src.utils.model_registry import MODEL_OBJECTS
-
-
-
-logger = Logger("app", see_time=True, console_log=True)
 
+logger = Logger("app", see_time=True, console_log=False)
 load_dotenv()
 
-
-
-# Request models
-
-class DeepAnalysisRequest(BaseModel):
-
-    goal: str
-
-    
-
-class DeepAnalysisResponse(BaseModel):
-
-    goal: str
-
-    deep_questions: str
-
-    deep_plan: str
-
-    summaries: List[str]
-
-    code: str
-
-    plotly_figs: List
-
-    synthesis: List[str]
-
-    final_conclusion: str
-
-    html_report: Optional[str] = None
-
-
-
-styling_instructions =  [
-    {
-        "category": "line_charts",
-        "description": "Used to visualize trends and changes over time, often with multiple series.",
-        "styling": {
-            "template": "plotly_white",
-            "axes_line_width": 0.2,
-            "grid_width": 1,
-            "title": {
-                "bold_html": True,
-                "include": True
-            },
-            "colors": "use multiple colors if more than one line",
-            "annotations": ["min", "max"],
-            "number_format": {
-                "apply_k_m": True,
-                "thresholds": {"K": 1000, "M": 100000},
-                "percentage_decimals": 2,
-                "percentage_sign": True
-            },
-            "default_size": {"height": 1200, "width": 1000}
-        }
-    },
-    {
-        "category": "bar_charts",
-        "description": "Useful for comparing discrete categories or groups with bars representing values.",
-        "styling": {
-            "template": "plotly_white",
-            "axes_line_width": 0.2,
-            "grid_width": 1,
-            "title": {"bold_html": True, "include": True},
-            "annotations": ["bar values"],
-            "number_format": {
-                "apply_k_m": True,
-                "thresholds": {"K": 1000, "M": 100000},
-                "percentage_decimals": 2,
-                "percentage_sign": True
-            },
-            "default_size": {"height": 1200, "width": 1000}
-        }
-    },
-    {
-        "category": "histograms",
-        "description": "Display the distribution of a data set, useful for returns or frequency distributions.",
-        "styling": {
-            "template": "plotly_white",
-            "bin_size": 50,
-            "axes_line_width": 0.2,
-            "grid_width": 1,
-            "title": {"bold_html": True, "include": True},
-            "annotations": ["x values"],
-            "number_format": {
-                "apply_k_m": True,
-                "thresholds": {"K": 1000, "M": 100000},
-                "percentage_decimals": 2,
-                "percentage_sign": True
-            },
-            "default_size": {"height": 1200, "width": 1000}
-        }
-    },
-    {
-        "category": "pie_charts",
-        "description": "Show composition or parts of a whole with slices representing categories.",
-        "styling": {
-            "template": "plotly_white",
-            "top_categories_to_show": 10,
-            "bundle_rest_as": "Others",
-            "axes_line_width": 0.2,
-            "grid_width": 1,
-            "title": {"bold_html": True, "include": True},
-            "annotations": ["x values"],
-            "number_format": {
-                "apply_k_m": True,
-                "thresholds": {"K": 1000, "M": 100000},
-                "percentage_decimals": 2,
-                "percentage_sign": True
-            },
-            "default_size": {"height": 1200, "width": 1000}
-        }
-    },
-    {
-        "category": "tabular_and_generic_charts",
-        "description": "Applies to charts where number formatting needs flexibility, including mixed or raw data.",
-        "styling": {
-            "template": "plotly_white",
-            "axes_line_width": 0.2,
-            "grid_width": 1,
-            "title": {"bold_html": True, "include": True},
-            "annotations": ["x values"],
-            "number_format": {
-                "apply_k_m": True,
-                "thresholds": {"K": 1000, "M": 100000},
-                "exclude_if_commas_present": True,
-                "exclude_if_not_numeric": True,
-                "percentage_decimals": 2,
-                "percentage_sign": True
-            },
-            "default_size": {"height": 1200, "width": 1000}
-        }
-    },
-    {
-        "category": "heat_maps",
-        "description": "Show data density or intensity using color scales on a matrix or grid.",
-        "styling": {
-            "template": "plotly_white",
-            "axes_styles": {
-                "line_color": "black",
-                "line_width": 0.2,
-                "grid_width": 1,
-                "format_numbers_as_k_m": True,
-                "exclude_non_numeric_formatting": True
-            },
-            "title": {"bold_html": True, "include": True},
-            "default_size": {"height": 1200, "width": 1000}
-        }
-    },
-    {
-        "category": "histogram_distribution",
-        "description": "Specialized histogram for return distributions with opacity control.",
-        "styling": {
-            "template": "plotly_white",
-            "opacity": 0.75,
-            "axes_styles": {
-                "grid_width": 1,
-                "format_numbers_as_k_m": True,
-                "exclude_non_numeric_formatting": True
-            },
-            "title": {"bold_html": True, "include": True},
-            "default_size": {"height": 1200, "width": 1000}
-        }
-    }
+styling_instructions = [
+    """
+        Dont ignore any of these instructions.
+        For a line chart always use plotly_white template, reduce x axes & y axes line to 0.2 & x & y grid width to 1. 
+        Always give a title and make bold using html tag axis label and try to use multiple colors if more than one line
+        Annotate the min and max of the line
+        Display numbers in thousand(K) or Million(M) if larger than 1000/100000 
+        Show percentages in 2 decimal points with '%' sign
+        Default size of chart should be height =1200 and width =1000
+        
+        """
+        
+   , """
+        Dont ignore any of these instructions.
+        For a bar chart always use plotly_white template, reduce x axes & y axes line to 0.2 & x & y grid width to 1. 
+        Always give a title and make bold using html tag axis label 
+        Always display numbers in thousand(K) or Million(M) if larger than 1000/100000. 
+        Annotate the values of the bar chart
+        If variable is a percentage show in 2 decimal points with '%' sign.
+        Default size of chart should be height =1200 and width =1000
+        """
+        ,
+
+          """
+        For a histogram chart choose a bin_size of 50
+        Do not ignore any of these instructions
+        always use plotly_white template, reduce x & y axes line to 0.2 & x & y grid width to 1. 
+        Always give a title and make bold using html tag axis label 
+        Always display numbers in thousand(K) or Million(M) if larger than 1000/100000. Add annotations x values
+        If variable is a percentage show in 2 decimal points with '%'
+        Default size of chart should be height =1200 and width =1000
+        """,
+
+
+          """
+        For a pie chart only show top 10 categories, bundle rest as others
+        Do not ignore any of these instructions
+        always use plotly_white template, reduce x & y axes line to 0.2 & x & y grid width to 1. 
+        Always give a title and make bold using html tag axis label 
+        Always display numbers in thousand(K) or Million(M) if larger than 1000/100000. Add annotations x values
+        If variable is a percentage show in 2 decimal points with '%'
+        Default size of chart should be height =1200 and width =1000
+        """,
+
+          """
+        Do not ignore any of these instructions
+        always use plotly_white template, reduce x & y axes line to 0.2 & x & y grid width to 1. 
+        Always give a title and make bold using html tag axis label 
+        Always display numbers in thousand(K) or Million(M) if larger than 1000/100000. Add annotations x values
+        Don't add K/M if number already in , or value is not a number
+        If variable is a percentage show in 2 decimal points with '%'
+        Default size of chart should be height =1200 and width =1000
+        """,
+"""
+    For a heat map
+    Use the 'plotly_white' template for a clean, white background. 
+    Set a chart title 
+    Style the X-axis with a black line color, 0.2 line width, 1 grid width, format 1000/1000000 as K/M
+    Do not format non-numerical numbers 
+    .style the Y-axis with a black line color, 0.2 line width, 1 grid width format 1000/1000000 as K/M
+    Do not format non-numerical numbers 
+
+    . Set the figure dimensions to a height of 1200 pixels and a width of 1000 pixels.
+""",
+"""
+    For a Histogram, used for returns/distribution plotting
+    Use the 'plotly_white' template for a clean, white background. 
+    Set a chart title 
+    Style the X-axis  1 grid width, format 1000/1000000 as K/M
+    Do not format non-numerical numbers 
+    .style the Y-axis, 1 grid width format 1000/1000000 as K/M
+    Do not format non-numerical numbers 
+    
+    Use an opacity of 0.75
+
+     Set the figure dimensions to a height of 1200 pixels and a width of 1000 pixels.
+"""
 ]
 
-# Convert to list of JSON strings
-styling_instructions = [str(chart_dict) for chart_dict in styling_instructions]
-
-# Output (just show first 2 for readability)
-
-
-
-
 # Add near the top of the file, after imports
-
 DEFAULT_MODEL_CONFIG = {
     "provider": os.getenv("MODEL_PROVIDER", "openai"),
-    "model": os.getenv("MODEL_NAME", "gpt-5-mini"),
+    "model": os.getenv("MODEL_NAME", "gpt-4o-mini"),
     "api_key": os.getenv("OPENAI_API_KEY"),
-    "temperature": min(1.0, max(0.0, float(os.getenv("TEMPERATURE", "1.0")))),  # Clamp to 0..1
-    "max_tokens": int(os.getenv("MAX_TOKENS", 6000)), "cache": False
-
+    "temperature": float(os.getenv("TEMPERATURE", 1.0)),
+    "max_tokens": int(os.getenv("MAX_TOKENS", 6000))
 }
 
-
-
 # Create default LM config but don't set it globally
-
-
-
-default_lm = MODEL_OBJECTS[DEFAULT_MODEL_CONFIG['model']]
-
-    
-
-
-
-    
-
-dspy.configure(lm=default_lm, async_max_workers=1000)
-
-
+if DEFAULT_MODEL_CONFIG["provider"].lower() == "groq":
+    default_lm = dspy.GROQ(
+        model=DEFAULT_MODEL_CONFIG["model"],
+        api_key=DEFAULT_MODEL_CONFIG["api_key"],
+        temperature=DEFAULT_MODEL_CONFIG["temperature"],
+        max_tokens=DEFAULT_MODEL_CONFIG["max_tokens"]
+    )
+elif DEFAULT_MODEL_CONFIG["provider"].lower() == "gemini":
+    default_lm = dspy.LM(
+        model=f"gemini/{DEFAULT_MODEL_CONFIG['model']}",
+        api_key=DEFAULT_MODEL_CONFIG["api_key"],
+        temperature=DEFAULT_MODEL_CONFIG["temperature"],
+        max_tokens=DEFAULT_MODEL_CONFIG["max_tokens"]
+    )
+else:
+    default_lm = dspy.LM(
+        model=DEFAULT_MODEL_CONFIG["model"],
+        api_key=DEFAULT_MODEL_CONFIG["api_key"],
+        temperature=DEFAULT_MODEL_CONFIG["temperature"],
+        max_tokens=DEFAULT_MODEL_CONFIG["max_tokens"]
+    )
 
 # Function to get model config from session or use default
-
 def get_session_lm(session_state):
-
     """Get the appropriate LM instance for a session, or default if not configured"""
-
     # First check if we have a valid session-specific model config 
-
     if session_state and isinstance(session_state, dict) and "model_config" in session_state:
-
         model_config = session_state["model_config"]
-
         if model_config and isinstance(model_config, dict) and "model" in model_config:
-
             # Found valid session-specific model config, use it
-
             provider = model_config.get("provider", "openai").lower()
-
-            model_name = model_config.get("model", DEFAULT_MODEL_CONFIG["model"])
-
-            # Import and apply centralized safeguards (temperature + max_tokens)
-            
-
-            requested_temp = model_config.get("temperature", DEFAULT_MODEL_CONFIG["temperature"])
-            requested_max = model_config.get("max_tokens", DEFAULT_MODEL_CONFIG["max_tokens"])
-
-            safe_params = apply_model_safeguards(
-                model_name=model_name,
-                provider=provider,
-                temperature=requested_temp,
-                max_tokens=requested_max,
-            )
-
-            # Apply the safeguarded parameters
-            MODEL_OBJECTS[model_name].__dict__['kwargs']['max_tokens'] = safe_params["max_tokens"]
-            MODEL_OBJECTS[model_name].__dict__['kwargs']['temperature'] = safe_params["temperature"]
-
-
-
+            if provider == "groq":
+                return dspy.GROQ(
+                    model=model_config.get("model", DEFAULT_MODEL_CONFIG["model"]),
+                    api_key=model_config.get("api_key", DEFAULT_MODEL_CONFIG["api_key"]),
+                    temperature=model_config.get("temperature", DEFAULT_MODEL_CONFIG["temperature"]),
+                    max_tokens=model_config.get("max_tokens", DEFAULT_MODEL_CONFIG["max_tokens"])
+                )
+            elif provider == "anthropic":
+                return dspy.LM(
+                    model=model_config.get("model", DEFAULT_MODEL_CONFIG["model"]),
+                    api_key=model_config.get("api_key", DEFAULT_MODEL_CONFIG["api_key"]),
+                    temperature=model_config.get("temperature", DEFAULT_MODEL_CONFIG["temperature"]),
+                    max_tokens=model_config.get("max_tokens", DEFAULT_MODEL_CONFIG["max_tokens"])
+                )
+            elif provider == "gemini":
+                return dspy.LM(
+                    model=f"gemini/{model_config.get('model', DEFAULT_MODEL_CONFIG['model'])}",
+                    api_key=model_config.get("api_key", DEFAULT_MODEL_CONFIG["api_key"]),
+                    temperature=model_config.get("temperature", DEFAULT_MODEL_CONFIG["temperature"]),
+                    max_tokens=model_config.get("max_tokens", DEFAULT_MODEL_CONFIG["max_tokens"])
+                )
+            else:  # OpenAI is the default
+                return dspy.LM(
+                    model=model_config.get("model", DEFAULT_MODEL_CONFIG["model"]),
+                    api_key=model_config.get("api_key", DEFAULT_MODEL_CONFIG["api_key"]),
+                    temperature=model_config.get("temperature", DEFAULT_MODEL_CONFIG["temperature"]),
+                    max_tokens=model_config.get("max_tokens", DEFAULT_MODEL_CONFIG["max_tokens"])
+                )
     
-
     # If no valid session config, use default
-
-    return MODEL_OBJECTS[model_name]
-
-
+    return default_lm
 
 # Initialize retrievers with empty data first
-
-
+def initialize_retrievers(styling_instructions: List[str], doc: List[str]):
+    try:
+        style_index = VectorStoreIndex.from_documents([Document(text=x) for x in styling_instructions])
+        data_index = VectorStoreIndex.from_documents([Document(text=x) for x in doc])
+        return {"style_index": style_index, "dataframe_index": data_index}
+    except Exception as e:
+        logger.log_message(f"Error initializing retrievers: {str(e)}", level=logging.ERROR)
+        raise e
 
 # clear console
-
 def clear_console():
-
     os.system('cls' if os.name == 'nt' else 'clear')
 
 
-
-
-
 # Check for Housing.csv
-
 housing_csv_path = "Housing.csv"
-
 if not os.path.exists(housing_csv_path):
-
     logger.log_message(f"Housing.csv not found at {os.path.abspath(housing_csv_path)}", level=logging.ERROR)
-
     raise FileNotFoundError(f"Housing.csv not found at {os.path.abspath(housing_csv_path)}")
 
+AVAILABLE_AGENTS = {
+    "data_viz_agent": data_viz_agent,
+    "sk_learn_agent": sk_learn_agent,
+    "statistical_analytics_agent": statistical_analytics_agent,
+    "preprocessing_agent": preprocessing_agent,
+}
 
-
-# All agents are now loaded from database - no hardcoded dictionaries needed
-
-
+PLANNER_AGENTS = {
+    "planner_preprocessing_agent": planner_preprocessing_agent,
+    "planner_sk_learn_agent": planner_sk_learn_agent,
+    "planner_statistical_analytics_agent": planner_statistical_analytics_agent,
+    "planner_data_viz_agent": planner_data_viz_agent,
+}
 
 # Add session header
-
 X_SESSION_ID = APIKeyHeader(name="X-Session-ID", auto_error=False)
 
-
-
 # Update AppState class to use SessionManager
+class AppState:
+    def __init__(self):
+        self._session_manager = SessionManager(styling_instructions, PLANNER_AGENTS)
+        self.model_config = DEFAULT_MODEL_CONFIG.copy()
+        # Update the SessionManager with the current model_config
+        self._session_manager._app_model_config = self.model_config
+        self.ai_manager = AI_Manager()
+        self.chat_name_agent = chat_history_name_agent
+    
+    def get_session_state(self, session_id: str):
+        """Get or create session-specific state using the SessionManager"""
+        return self._session_manager.get_session_state(session_id)
 
-# The AppState class is now in src.managers.app_manager
+    def clear_session_state(self, session_id: str):
+        """Clear session-specific state using the SessionManager"""
+        self._session_manager.clear_session_state(session_id)
 
+    def update_session_dataset(self, session_id: str, df, name, desc):
+        """Update dataset for a specific session using the SessionManager"""
+        self._session_manager.update_session_dataset(session_id, df, name, desc)
 
+    def reset_session_to_default(self, session_id: str):
+        """Reset a session to use the default dataset using the SessionManager"""
+        self._session_manager.reset_session_to_default(session_id)
+    
+    def set_session_user(self, session_id: str, user_id: int, chat_id: int = None):
+        """Associate a user with a session using the SessionManager"""
+        return self._session_manager.set_session_user(session_id, user_id, chat_id)
+    
+    def get_ai_manager(self):
+        """Get the AI Manager instance"""
+        return self.ai_manager
+    
+    def get_provider_for_model(self, model_name):
+        return self.ai_manager.get_provider_for_model(model_name)
+    
+    def calculate_cost(self, model_name, input_tokens, output_tokens):
+        return self.ai_manager.calculate_cost(model_name, input_tokens, output_tokens)
+    
+    def save_usage_to_db(self, user_id, chat_id, model_name, provider, prompt_tokens, completion_tokens, total_tokens, query_size, response_size, cost, request_time_ms, is_streaming=False):
+        return self.ai_manager.save_usage_to_db(user_id, chat_id, model_name, provider, prompt_tokens, completion_tokens, total_tokens, query_size, response_size, round(cost, 7), request_time_ms, is_streaming)
+    
+    def get_tokenizer(self):
+        return self.ai_manager.tokenizer
+    
+    def get_chat_history_name_agent(self):
+        return dspy.Predict(self.chat_name_agent)
 
 # Initialize FastAPI app with state
-
 app = FastAPI(title="AI Analytics API", version="1.0")
-
-# Pass required parameters to AppState
-app.state = AppState(styling_instructions, chat_history_name_agent, DEFAULT_MODEL_CONFIG)
-
-
-
-
+app.state = AppState()
 
 # Configure middleware
-
 # Use a wildcard for local development or read from environment
-
 is_development = os.getenv("ENVIRONMENT", "development").lower() == "development"
 
-
-
 allowed_origins = []
-
 frontend_url = os.getenv("FRONTEND_URL", "").strip()
-
 print(f"FRONTEND_URL: {frontend_url}")
-
 if is_development:
-
     allowed_origins = ["*"]
-
 elif frontend_url:
-
     allowed_origins = [frontend_url]
-
 else:
-
     logger.log_message("CORS misconfigured: FRONTEND_URL not set", level=logging.ERROR)
-
     allowed_origins = []  # or set a default safe origin
 
-
-
 # Add a strict origin verification middleware
-
 @app.middleware("http")
-
 async def verify_origin_middleware(request: Request, call_next):
-
     # Skip origin check in development mode
-
     if is_development:
-
         return await call_next(request)
-
     
-
     # Get the origin from the request headers
-
     origin = request.headers.get("origin")
-
     
-
     # Log the origin for debugging
-
     if origin:
-
         print(f"Request from origin: {origin}")
-
     
-
     # If no origin header or origin not in allowed list, reject the request
-
     if origin and frontend_url and origin != frontend_url:
-
         print(f"Blocked request from unauthorized origin: {origin}")
-
         return JSONResponse(
-
             status_code=403,
-
             content={"detail": "Not authorized"}
-
         )
-
     
-
     # Continue processing the request if origin is allowed
-
     return await call_next(request)
 
-
-
 # CORS middleware (still needed for browser preflight)
-
 app.add_middleware(
-
     CORSMiddleware,
-
     allow_origins=allowed_origins,
-
     allow_origin_regex=None,
-
     allow_credentials=True,
-
     allow_methods=["*"],
-
     allow_headers=["*"],
-
     expose_headers=["*"],
-
     max_age=600  # Cache preflight requests for 10 minutes (for performance)
-
 )
 
-
-
 # Add these constants at the top of the file with other imports/constants
-
 RESPONSE_ERROR_INVALID_QUERY = "Please provide a valid query..."
-
 RESPONSE_ERROR_NO_DATASET = "No dataset is currently loaded. Please link a dataset before proceeding with your analysis."
-
 DEFAULT_TOKEN_RATIO = 1.5
-
-REQUEST_TIMEOUT_SECONDS = 90 # Timeout for LLM requests
-
-MAX_RECENT_MESSAGES = 5
-
+REQUEST_TIMEOUT_SECONDS = 60  # Timeout for LLM requests
+MAX_RECENT_MESSAGES = 3
 DB_BATCH_SIZE = 10  # For future batch DB operations
 
-
-
+# Replace the existing chat_with_agent function
 @app.post("/chat/{agent_name}", response_model=dict)
-
 async def chat_with_agent(
-
     agent_name: str, 
-
     request: QueryRequest,
-
     request_obj: Request,
-
     session_id: str = Depends(get_session_id_dependency)
-
 ):
-
     session_state = app.state.get_session_state(session_id)
-
-    logger.log_message(f"[DEBUG] chat_with_agent called with agent: '{agent_name}', query: '{request.query[:100]}...'", level=logging.DEBUG)
-
     
-
     try:
-
         # Extract and validate query parameters
-
-        logger.log_message(f"[DEBUG] Updating session from query params", level=logging.DEBUG)
-
         _update_session_from_query_params(request_obj, session_state)
-
-        logger.log_message(f"[DEBUG] Session state after query params: user_id={session_state.get('user_id')}, chat_id={session_state.get('chat_id')}", level=logging.DEBUG)
-
         
-
         # Validate dataset and agent name
-
-        if session_state["datasets"] is None:
-            logger.log_message(f"[DEBUG] No dataset loaded", level=logging.DEBUG)
-
+        if session_state["current_df"] is None:
             raise HTTPException(status_code=400, detail=RESPONSE_ERROR_NO_DATASET)
 
-
-
-        # Log the dataset being used for analysis with detailed information
-        datasets = session_state["datasets"]
-        dataset_names = list(datasets.keys())
-        if dataset_names:
-            current_dataset_name = dataset_names[-1]  # Get the last (most recent) dataset
-            dataset_shape = datasets[current_dataset_name].shape
-            
-            # Check if this is the default dataset and explain why
-            session_name = session_state.get("name", "")
-            is_default_dataset = (current_dataset_name == "df" and session_name == "Housing.csv") or current_dataset_name == "Housing.csv"
-            
-            if is_default_dataset:
-                logger.log_message(f"[ANALYSIS] Using DEFAULT dataset 'Housing.csv' for analysis (shape: {dataset_shape[0]} rows, {dataset_shape[1]} columns)", level=logging.INFO)
-                logger.log_message(f"[ANALYSIS] Reason: No custom dataset uploaded yet - using default Housing.csv dataset", level=logging.INFO)
-            else:
-                logger.log_message(f"[ANALYSIS] Using CUSTOM dataset '{current_dataset_name}' for analysis (shape: {dataset_shape[0]} rows, {dataset_shape[1]} columns)", level=logging.INFO)
-                logger.log_message(f"[ANALYSIS] This is a user-uploaded dataset, not the default", level=logging.INFO)
-        else:
-            logger.log_message(f"[ANALYSIS] No datasets available in session {session_id}", level=logging.WARNING)
-
-        logger.log_message(f"[DEBUG] About to validate agent name: '{agent_name}'", level=logging.DEBUG)
-
-        _validate_agent_name(agent_name, session_state)
-
-        logger.log_message(f"[DEBUG] Agent validation completed successfully", level=logging.DEBUG)
-
+        _validate_agent_name(agent_name)
         
-
         # Record start time for timing
-
         start_time = time.time()
-
         
-
         # Get chat context and prepare query
-
-        logger.log_message(f"[DEBUG] Preparing query with context", level=logging.DEBUG)
-
         enhanced_query = _prepare_query_with_context(request.query, session_state)
-
-        logger.log_message(f"[DEBUG] Enhanced query length: {len(enhanced_query)}", level=logging.DEBUG)
-
         
-
-        # Initialize agent - handle standard, template, and custom agents
-
+        # Initialize agent
         if "," in agent_name:
-
-            logger.log_message(f"[DEBUG] Processing multiple agents: {agent_name}", level=logging.DEBUG)
-
-            # Multiple agents case
-
-            agent_list = [agent.strip() for agent in agent_name.split(",")]
-
-            
-
-            # Categorize agents
-
-            standard_agents = [agent for agent in agent_list if _is_standard_agent(agent)]
-
-            template_agents = [agent for agent in agent_list if _is_template_agent(agent)]
-
-            custom_agents = [agent for agent in agent_list if not _is_standard_agent(agent) and not _is_template_agent(agent)]
-
-            
-
-            logger.log_message(f"[DEBUG] Agent categorization - standard: {standard_agents}, template: {template_agents}, custom: {custom_agents}", level=logging.DEBUG)
-
-            
-
-            if custom_agents:
-
-                # If any custom agents, use session AI system for all
-
-                ai_system = session_state["ai_system"]
-
-                session_lm = get_session_lm(session_state)
-
-                logger.log_message(f"[DEBUG] Using custom agent execution path", level=logging.DEBUG)
-
-                with dspy.context(lm=session_lm):
-
-                    response = await asyncio.wait_for(
-
-                        _execute_custom_agents(ai_system, agent_list, enhanced_query),
-
-                        timeout=REQUEST_TIMEOUT_SECONDS
-
-                    )
-
-                    logger.log_message(f"[DEBUG] Custom agents response type: {type(response)}, keys: {list(response.keys()) if isinstance(response, dict) else 'not a dict'}", level=logging.DEBUG)
-
-            else:
-
-                # All standard/template agents - use auto_analyst_ind which loads from DB
-
-                user_id = session_state.get("user_id")
-
-                logger.log_message(f"[DEBUG] Using auto_analyst_ind for multiple standard/template agents with user_id: {user_id}", level=logging.DEBUG)
-
-        
-
-                # Create database session for agent loading
-
-                from src.db.init_db import session_factory
-
-                db_session = session_factory()
-
-                try:
-
-                    # auto_analyst_ind will load all agents from database
-
-                    logger.log_message(f"[DEBUG] Creating auto_analyst_ind instance", level=logging.DEBUG)
-
-                    agent = auto_analyst_ind(agents=[], retrievers=session_state["retrievers"], user_id=user_id, db_session=db_session)
-
-                    session_lm = get_session_lm(session_state)
-
-                    logger.log_message(f"[DEBUG] About to call agent.forward with query and agent list", level=logging.DEBUG)
-
-                    with dspy.context(lm=session_lm):
-
-                        response = await asyncio.wait_for(
-
-                            agent(enhanced_query, ",".join(agent_list)),
-
-                            timeout=REQUEST_TIMEOUT_SECONDS
-
-                        )
-
-                        logger.log_message(f"[DEBUG] auto_analyst_ind response type: {type(response)}, content: {str(response)[:200]}...", level=logging.DEBUG)
-
-                finally:
-
-                    db_session.close()
-
+            agent_list = [AVAILABLE_AGENTS[agent.strip()] for agent in agent_name.split(",")]
+            agent = auto_analyst_ind(agents=agent_list, retrievers=session_state["retrievers"])
         else:
-
-            logger.log_message(f"[DEBUG] Processing single agent: {agent_name}", level=logging.DEBUG)
-
-            # Single agent case
-
-            if _is_standard_agent(agent_name) or _is_template_agent(agent_name):
-
-                # Standard or template agent - use auto_analyst_ind which loads from DB
-
-                user_id = session_state.get("user_id")
-
-                logger.log_message(f"[DEBUG] Using auto_analyst_ind for single standard/template agent '{agent_name}' with user_id: {user_id}", level=logging.DEBUG)
-
-                
-
-                # Create database session for agent loading
-
-                from src.db.init_db import session_factory
-
-                db_session = session_factory()
-
-                try:
-
-                    # auto_analyst_ind will load all agents from database
-
-                    logger.log_message(f"[DEBUG] Creating auto_analyst_ind instance for single agent", level=logging.DEBUG)
-
-                    agent = auto_analyst_ind(agents=[], retrievers=session_state["retrievers"], user_id=user_id, db_session=db_session)
-
-                    session_lm = get_session_lm(session_state)
-
-                    logger.log_message(f"[DEBUG] About to call agent.forward for single agent '{agent_name}'", level=logging.DEBUG)
-
-                    with dspy.context(lm=session_lm):
-
-                        response = await asyncio.wait_for(
-
-                            agent(enhanced_query, agent_name),
-
-                            timeout=REQUEST_TIMEOUT_SECONDS
-
-                        )
-
-                        logger.log_message(f"[DEBUG] Single agent response type: {type(response)}, content: {str(response)[:200]}...", level=logging.DEBUG)
-
-                finally:
-
-                    db_session.close()
-
-            else:
-
-                # Custom agent - use session AI system
-
-                ai_system = session_state["ai_system"]
-
-                session_lm = get_session_lm(session_state)
-
-                logger.log_message(f"[DEBUG] Using custom agent execution for '{agent_name}'", level=logging.DEBUG)
-
-                with dspy.context(lm=session_lm):
-
-                    response = await asyncio.wait_for(
-
-                        _execute_custom_agents(ai_system, [agent_name], enhanced_query),
-
-                        timeout=REQUEST_TIMEOUT_SECONDS
-
-                    )
-
-                    logger.log_message(f"[DEBUG] Custom single agent response type: {type(response)}, content: {str(response)[:200]}...", level=logging.DEBUG)
-
+            agent = auto_analyst_ind(agents=[AVAILABLE_AGENTS[agent_name]], retrievers=session_state["retrievers"])
         
-
-        logger.log_message(f"[DEBUG] About to format response to markdown. Response type: {type(response)}", level=logging.DEBUG)
-
-        formatted_response = format_response_to_markdown(response, agent_name, session_state["datasets"])
-        logger.log_message(f"[DEBUG] Formatted response type: {type(formatted_response)}, length: {len(str(formatted_response))}", level=logging.DEBUG)
-
+        # Execute agent with timeout
+        try:
+            # Get session-specific model
+            session_lm = get_session_lm(session_state)
+            
+            # Use session-specific model for this request
+            with dspy.context(lm=session_lm):
+                response = await asyncio.wait_for(
+                    asyncio.to_thread(agent, enhanced_query, agent_name),
+                    timeout=REQUEST_TIMEOUT_SECONDS
+                )
+        except asyncio.TimeoutError:
+            logger.log_message(f"Agent execution timed out for {agent_name}", level=logging.WARNING)
+            raise HTTPException(status_code=504, detail="Request timed out. Please try a simpler query.")
+        except Exception as agent_error:
+            logger.log_message(f"Agent execution failed: {str(agent_error)}", level=logging.ERROR)
+            raise HTTPException(status_code=500, detail="Failed to process query. Please try again.")
+        
+        formatted_response = format_response_to_markdown(response, agent_name, session_state["current_df"])
         
-
         if formatted_response == RESPONSE_ERROR_INVALID_QUERY:
-
-            logger.log_message(f"[DEBUG] Response was invalid query error", level=logging.DEBUG)
-
             return {
-
                 "agent_name": agent_name,
-
                 "query": request.query,
-
                 "response": formatted_response,
-
                 "session_id": session_id
-
             }
-
         
-
         # Track usage statistics
-
         if session_state.get("user_id"):
-
-            logger.log_message(f"[DEBUG] Tracking model usage", level=logging.DEBUG)
-
             _track_model_usage(
-
                 session_state=session_state,
-
                 enhanced_query=enhanced_query,
-
                 response=response,
-
                 processing_time_ms=int((time.time() - start_time) * 1000)
-
             )
-
         
-
-        logger.log_message(f"[DEBUG] chat_with_agent completed successfully", level=logging.DEBUG)
-
         return {
-
             "agent_name": agent_name,
-
             "query": request.query,  # Return original query without context
-
             "response": formatted_response,
-
             "session_id": session_id
-
         }
-
     except HTTPException:
-
         # Re-raise HTTP exceptions to preserve status codes
-
-        logger.log_message(f"[DEBUG] HTTPException caught and re-raised", level=logging.DEBUG)
-
         raise
-
-    except asyncio.TimeoutError:
-
-        logger.log_message(f"[ERROR] Timeout error in chat_with_agent", level=logging.ERROR)
-
-        raise HTTPException(status_code=504, detail="Request timed out. Please try a simpler query.")
-
     except Exception as e:
-
-        logger.log_message(f"[ERROR] Unexpected error in chat_with_agent: {str(e)}", level=logging.ERROR)
-
-        logger.log_message(f"[ERROR] Exception type: {type(e)}, traceback: {str(e)}", level=logging.ERROR)
-
-        import traceback
-
-        logger.log_message(f"[ERROR] Full traceback: {traceback.format_exc()}", level=logging.ERROR)
-
+        logger.log_message(f"Unexpected error in chat_with_agent: {str(e)}", level=logging.ERROR)
         raise HTTPException(status_code=500, detail="An unexpected error occurred. Please try again later.")
-
-
-
-
-
+    
+    
 @app.post("/chat", response_model=dict)
-
 async def chat_with_all(
-
     request: QueryRequest,
-
     request_obj: Request,
-
     session_id: str = Depends(get_session_id_dependency)
-
 ):
-
     session_state = app.state.get_session_state(session_id)
 
-
-
     try:
-
         # Extract and validate query parameters
-
         _update_session_from_query_params(request_obj, session_state)
-
         
-
         # Validate dataset
-
-        if session_state["datasets"] is None:
+        if session_state["current_df"] is None:
             raise HTTPException(status_code=400, detail=RESPONSE_ERROR_NO_DATASET)
-
         
-
         if session_state["ai_system"] is None:
-
             raise HTTPException(status_code=500, detail="AI system not properly initialized.")
 
-
-
         # Get session-specific model
-
         session_lm = get_session_lm(session_state)
 
-
-
         # Create streaming response
-
         return StreamingResponse(
-
             _generate_streaming_responses(session_state, request.query, session_lm),
-
             media_type='text/event-stream',
-
             headers={
-
                 'Cache-Control': 'no-cache',
-
                 'Connection': 'keep-alive',
-
                 'Content-Type': 'text/event-stream',
-
                 'Access-Control-Allow-Origin': '*',
-
                 'X-Accel-Buffering': 'no'
-
             }
-
         )
-
     except HTTPException:
-
         # Re-raise HTTP exceptions to preserve status codes
-
         raise
-
     except Exception as e:
-
+        logger.log_message(f"Unexpected error in chat_with_all: {str(e)}", level=logging.ERROR)
         raise HTTPException(status_code=500, detail="An unexpected error occurred. Please try again later.")
 
 
-
-
-
 # Helper functions to reduce duplication and improve modularity
-
 def _update_session_from_query_params(request_obj: Request, session_state: dict):
-
     """Extract and validate chat_id and user_id from query parameters"""
-
     # Check for chat_id in query parameters
-
     if "chat_id" in request_obj.query_params:
-
         try:
-
             chat_id_param = int(request_obj.query_params.get("chat_id"))
-
             # Update session state with this chat ID
-
             session_state["chat_id"] = chat_id_param
-
         except (ValueError, TypeError):
-
             logger.log_message("Invalid chat_id parameter", level=logging.WARNING)
-
             # Continue without updating chat_id
 
-
-
     # Check for user_id in query parameters
-
     if "user_id" in request_obj.query_params:
-
         try:
-
             user_id = int(request_obj.query_params["user_id"])
-
             session_state["user_id"] = user_id
-
         except (ValueError, TypeError):
-
             raise HTTPException(
-
                 status_code=400,
-
                 detail="Invalid user_id in query params. Please provide a valid integer."
-
             )
 
 
-
-
-
-def _validate_agent_name(agent_name: str, session_state: dict = None):
-
-    """Validate that the agent name(s) are available"""
-
-    logger.log_message(f"[DEBUG] Validating agent name: '{agent_name}'", level=logging.DEBUG)
-
-    
-
+def _validate_agent_name(agent_name: str):
+    """Validate that the requested agent(s) exist"""
     if "," in agent_name:
-
-        # Multiple agents
-
         agent_list = [agent.strip() for agent in agent_name.split(",")]
-
-        logger.log_message(f"[DEBUG] Multiple agents detected: {agent_list}", level=logging.DEBUG)
-
         for agent in agent_list:
-
-            is_available = _is_agent_available(agent, session_state)
-
-            logger.log_message(f"[DEBUG] Agent '{agent}' availability: {is_available}", level=logging.DEBUG)
-
-            if not is_available:
-
-                available_agents = _get_available_agents_list(session_state)
-
-                logger.log_message(f"[DEBUG] Agent '{agent}' not found. Available: {available_agents}", level=logging.DEBUG)
-
+            if agent not in AVAILABLE_AGENTS:
+                available = list(AVAILABLE_AGENTS.keys())
                 raise HTTPException(
-
-                    status_code=400, 
-
-                    detail=f"Agent '{agent}' not found. Available agents: {available_agents}"
-
-                )
-
-    else:
-
-        # Single agent
-
-        is_available = _is_agent_available(agent_name, session_state)
-
-        logger.log_message(f"[DEBUG] Single agent '{agent_name}' availability: {is_available}", level=logging.DEBUG)
-
-        if not is_available:
-
-            available_agents = _get_available_agents_list(session_state)
-
-            logger.log_message(f"[DEBUG] Agent '{agent_name}' not found. Available: {available_agents}", level=logging.DEBUG)
-
-            raise HTTPException(
-
-                status_code=400, 
-
-                detail=f"Agent '{agent_name}' not found. Available agents: {available_agents}"
-
-            )
-
-    
-
-    logger.log_message(f"[DEBUG] Agent validation passed for: '{agent_name}'", level=logging.DEBUG)
-
-
-
-def _is_agent_available(agent_name: str, session_state: dict = None) -> bool:
-
-    """Check if an agent is available (standard, template, or custom)"""
-
-    # Check if it's a standard agent
-
-    if _is_standard_agent(agent_name):
-
-        return True
-
-    
-
-    # Check if it's a template agent
-
-    if _is_template_agent(agent_name):
-
-        return True
-
-    
-
-    # Check if it's a custom agent in session
-
-    if session_state and "ai_system" in session_state:
-
-        ai_system = session_state["ai_system"]
-
-        if hasattr(ai_system, 'agents') and agent_name in ai_system.agents:
-
-            return True
-
-    
-
-    return False
-
-
-
-def _get_available_agents_list(session_state: dict = None) -> list:
-
-    """Get list of all available agents from database"""
-
-    from src.db.init_db import session_factory
-
-    from src.agents.agents import load_all_available_templates_from_db
-
-    
-
-    # Core agents (always available)
-
-    available = ["preprocessing_agent", "statistical_analytics_agent", "sk_learn_agent", "data_viz_agent"]
-
-    
-
-    # Add template agents from database
-
-    db_session = session_factory()
-
-    try:
-
-        template_agents_dict = load_all_available_templates_from_db(db_session)
-
-        # template_agents_dict is a dict with template_name as keys
-
-        template_names = [template_name for template_name in template_agents_dict.keys() 
-
-                         if template_name not in available and template_name != 'basic_qa_agent']
-
-        available.extend(template_names)
-
-    except Exception as e:
-
-        logger.log_message(f"Error loading template agents: {str(e)}", level=logging.ERROR)
-
-    finally:
-
-        db_session.close()
-
-    
-
-    return available
-
-
-
-def _is_standard_agent(agent_name: str) -> bool:
-
-    """Check if agent is one of the 4 core standard agents"""
-
-    standard_agents = ["preprocessing_agent", "statistical_analytics_agent", "sk_learn_agent", "data_viz_agent"]
-
-    return agent_name in standard_agents
-
-
-
-def _is_template_agent(agent_name: str) -> bool:
-
-    """Check if agent is a template agent"""
-
-    try:
-
-        from src.db.init_db import session_factory
-
-        from src.db.schemas.models import AgentTemplate
-
-        
-
-        db_session = session_factory()
-
-        try:
-
-            template = db_session.query(AgentTemplate).filter(
-
-                AgentTemplate.template_name == agent_name,
-
-                AgentTemplate.is_active == True
-
-            ).first()
-
-            return template is not None
-
-        finally:
-
-            db_session.close()
-
-    except Exception as e:
-
-        logger.log_message(f"Error checking if {agent_name} is template: {str(e)}", level=logging.ERROR)
-
-        return False
-
-
-
-async def _execute_custom_agents(ai_system, agent_names: list, query: str):
-
-    """Execute custom agents using the session's AI system"""
-
-    try:
-
-        # For custom agents, we need to use the AI system's execute_agent method
-
-
-
-        agent_results = [ai_system]
-
-        if len(agent_names) == 1:
-
-            # Single custom agent
-
-            agent_name = agent_names[0]
-
-            # Prepare inputs for the custom agent (similar to standard agents like data_viz_agent)
-
-            dict_ = {}
-
-            dict_['dataset'] = ai_system.dataset.retrieve(query)[0].text
-
-            dict_['styling_index'] = ai_system.styling_index.retrieve(query)[0].text
-
-            dict_['goal'] = query
-
-            dict_['Agent_desc'] = str(ai_system.agent_desc)
-
-
-
-            # Get input fields for this agent
-
-            if agent_name in ai_system.agent_inputs:
-
-                inputs = {x: dict_[x] for x in ai_system.agent_inputs[agent_name] if x in dict_}
-
-                
-
-                # Execute the custom agent
-
-                agent_name_result, result_dict = await ai_system.agents[agent_name](**inputs)
-
-                return {agent_name_result: result_dict}
-
-            else:
-
-                logger.log_message(f"Agent '{agent_name}' not found in ai_system.agent_inputs", level=logging.ERROR)
-
-                return {"error": f"Agent '{agent_name}' input configuration not found"}
-
-        else:
-
-            # Multiple agents - execute sequentially
-
-            results = {}
-
-            for agent_name in agent_names:
-
-                single_result = await _execute_custom_agents(ai_system, [agent_name], query)
-
-                results.update(single_result)
-
-            return results
-
-            
-
-    except Exception as e:
-
-        logger.log_message(f"Error in _execute_custom_agents: {str(e)}", level=logging.ERROR)
-
-        return {"error": f"Error executing custom agents: {str(e)}"}
-
-
-
-def _prepare_query_with_context(query: str, session_state: dict) -> str:
-
-    """Prepare the query with chat context from previous messages"""
-
-    chat_id = session_state.get("chat_id")
-
-    if not chat_id:
-
-        return query
-
-        
-
-    # Get chat manager from app state
-
-    chat_manager = app.state._session_manager.chat_manager
-
-    # Get recent messages
-
-    recent_messages = chat_manager.get_recent_chat_history(chat_id, limit=MAX_RECENT_MESSAGES)
-
-    # Extract response history
-
-    chat_context = chat_manager.extract_response_history(recent_messages)
-
-    
-
-    # Append context to the query if available
-
-    if chat_context:
-
-        return f"### Current Query:\n{query}\n\n{chat_context}"
-
-    return query
-
+                    status_code=404, 
+                    detail=f"Agent '{agent}' not found. Available agents: {available}"
+                )
+    elif agent_name not in AVAILABLE_AGENTS:
+        available = list(AVAILABLE_AGENTS.keys())
+        raise HTTPException(
+            status_code=404, 
+            detail=f"Agent '{agent_name}' not found. Available agents: {available}"
+        )
 
 
+def _prepare_query_with_context(query: str, session_state: dict) -> str:
+    """Prepare the query with chat context from previous messages"""
+    chat_id = session_state.get("chat_id")
+    if not chat_id:
+        return query
+        
+    # Get chat manager from app state
+    chat_manager = app.state._session_manager.chat_manager
+    # Get recent messages
+    recent_messages = chat_manager.get_recent_chat_history(chat_id, limit=MAX_RECENT_MESSAGES)
+    # Extract response history
+    chat_context = chat_manager.extract_response_history(recent_messages)
+    
+    # Append context to the query if available
+    if chat_context:
+        return f"### Current Query:\n{query}\n\n{chat_context}"
+    return query
 
 
 def _track_model_usage(session_state: dict, enhanced_query: str, response, processing_time_ms: int):
-
     """Track model usage statistics in the database"""
-
     try:
-
         ai_manager = app.state.get_ai_manager()
-
         
-
         # Get model configuration
-
         model_config = session_state.get("model_config", DEFAULT_MODEL_CONFIG)
-
         model_name = model_config.get("model", DEFAULT_MODEL_CONFIG["model"])
-
         provider = ai_manager.get_provider_for_model(model_name)
-
         
-
         # Calculate token usage
-
         try:
-
             # Try exact tokenization
-
             prompt_tokens = len(ai_manager.tokenizer.encode(enhanced_query))
-
             completion_tokens = len(ai_manager.tokenizer.encode(str(response)))
-
             total_tokens = prompt_tokens + completion_tokens
-
         except Exception as token_error:
-
             # Fall back to estimation
-
             logger.log_message(f"Tokenization error: {str(token_error)}", level=logging.WARNING)
-
             prompt_words = len(enhanced_query.split())
-
             completion_words = len(str(response).split())
-
             prompt_tokens = int(prompt_words * DEFAULT_TOKEN_RATIO)
-
             completion_tokens = int(completion_words * DEFAULT_TOKEN_RATIO)
-
             total_tokens = prompt_tokens + completion_tokens
-
         
-
         # Calculate cost
-
         cost = ai_manager.calculate_cost(model_name, prompt_tokens, completion_tokens)
-
         
-
         # Save usage to database
-
         ai_manager.save_usage_to_db(
-
             user_id=session_state.get("user_id"),
-
             chat_id=session_state.get("chat_id"),
-
             model_name=model_name,
-
             provider=provider,
-
             prompt_tokens=int(prompt_tokens),
-
             completion_tokens=int(completion_tokens),
-
             total_tokens=int(total_tokens),
-
             query_size=len(enhanced_query),
-
             response_size=len(str(response)),
-
             cost=round(cost, 7),
-
             request_time_ms=processing_time_ms,
-
             is_streaming=False
-
         )
-
     except Exception as e:
-
         # Log but don't fail the request if usage tracking fails
-
         logger.log_message(f"Failed to track model usage: {str(e)}", level=logging.ERROR)
 
 
-
-
-
 async def _generate_streaming_responses(session_state: dict, query: str, session_lm):
-
     """Generate streaming responses for chat_with_all endpoint"""
-
     overall_start_time = time.time()
-
     total_response = ""
-
     total_inputs = ""
-
     usage_records = []
 
+    try:
+        # Add chat context from previous messages
+        enhanced_query = _prepare_query_with_context(query, session_state)
+        
+        # Use the session model for this specific request
+        with dspy.context(lm=session_lm):
+            try:
+                # Get the plan
+                plan_response = await asyncio.wait_for(
+                    asyncio.to_thread(session_state["ai_system"].get_plan, enhanced_query),
+                    timeout=REQUEST_TIMEOUT_SECONDS
+                )
+                
+                plan_description = format_response_to_markdown(
+                    {"analytical_planner": plan_response}, 
+                    dataframe=session_state["current_df"]
+                )
+                
+                # Check if plan is valid
+                if plan_description == RESPONSE_ERROR_INVALID_QUERY:
+                    yield json.dumps({
+                        "agent": "Analytical Planner",
+                        "content": plan_description,
+                        "status": "error"
+                    }) + "\n"
+                    return
+                
+                yield json.dumps({
+                    "agent": "Analytical Planner",
+                    "content": plan_description,
+                    "status": "success" if plan_description else "error"
+                }) + "\n"
+                
+                # Track planner usage
+                if session_state.get("user_id"):
+                    planner_tokens = _estimate_tokens(ai_manager=app.state.ai_manager, 
+                                                    input_text=enhanced_query, 
+                                                    output_text=plan_description)
+                    
+                    usage_records.append(_create_usage_record(
+                        session_state=session_state,
+                        model_name=session_state.get("model_config", DEFAULT_MODEL_CONFIG)["model"],
+                        prompt_tokens=planner_tokens["prompt"],
+                        completion_tokens=planner_tokens["completion"],
+                        query_size=len(enhanced_query),
+                        response_size=len(plan_description),
+                        processing_time_ms=int((time.time() - overall_start_time) * 1000),
+                        is_streaming=False
+                    ))
+                
+                # Execute the plan with well-managed concurrency
+                async for agent_name, inputs, response in _execute_plan_with_timeout(
+                    session_state["ai_system"], enhanced_query, plan_response):
+                    
+                    if agent_name == "plan_not_found":
+                        yield json.dumps({
+                            "agent": "Analytical Planner",
+                            "content": "**No plan found**\n\nPlease try again with a different query or try using a different model.",
+                            "status": "error"
+                        }) + "\n"
+                        return
+                    
+                    formatted_response = format_response_to_markdown(
+                        {agent_name: response}, 
+                        dataframe=session_state["current_df"]
+                    ) or "No response generated"
 
-
-    # Add chat context from previous messages
-
-    enhanced_query = _prepare_query_with_context(query, session_state)
-
-    
-
-    # try:
-
-        # Get the plan - planner is now async, so we need to await it
-
-    plan_response = await session_state["ai_system"].get_plan(enhanced_query)
-
-    
-
-    plan_description = format_response_to_markdown(
-
-        {"analytical_planner": plan_response}, 
-
-        datasets=session_state["datasets"]
-    )
-
-    
-
-    # Check if plan is valid
-
-    if plan_description == RESPONSE_ERROR_INVALID_QUERY:
-
-        yield json.dumps({
-
-            "agent": "Analytical Planner",
-
-            "content": plan_description,
-
-            "status": "error"
-
-        }) + "\n"
-
-        return
-
-    
-
-    yield json.dumps({
-
-        "agent": "Analytical Planner",
-
-        "content": plan_description,
-
-        "status": "success" if plan_description else "error"
-
-    }) + "\n"
-
-    
-
-    # Track planner usage
-
-    if session_state.get("user_id"):
-
-        planner_tokens = _estimate_tokens(ai_manager=app.state.ai_manager, 
-
-                                        input_text=enhanced_query, 
-
-                                        output_text=plan_description)
-
-        
-
-        usage_records.append(_create_usage_record(
-
-            session_state=session_state,
-
-            model_name=session_state.get("model_config", DEFAULT_MODEL_CONFIG)["model"],
-
-            prompt_tokens=planner_tokens["prompt"],
-
-            completion_tokens=planner_tokens["completion"],
-
-            query_size=len(enhanced_query),
-
-            response_size=len(plan_description),
-
-            processing_time_ms=int((time.time() - overall_start_time) * 1000),
-
-            is_streaming=False
-
-        ))
-
-    
-
-    logger.log_message(f"Plan response: {plan_response}", level=logging.INFO)
-
-    logger.log_message(f"Plan response type: {type(plan_response)}", level=logging.INFO)
-
-
-
-    # Check if plan_response is valid
-
-    # if not plan_response or not isinstance(plan_response, dict):
-
-    #     yield json.dumps({
-
-    #         "agent": "Analytical Planner",
-
-    #         "content": "**Error: Invalid plan response**\n\nResponse: " + str(plan_response),
-
-    #         "status": "error"
-
-    #     }) + "\n"
-
-    #     return
-
-    
-
-    # Execute the plan with well-managed concurrency
-
-    with dspy.context(lm = session_lm):
-
-        # try:
-
-            
-
-        async for agent_name, inputs, response in session_state["ai_system"].execute_plan(enhanced_query, plan_response):
-
-            
-
-            if agent_name == "plan_not_found":
-
-                yield json.dumps({
-
-                    "agent": "Analytical Planner",
-
-                    "content": "**No plan found**\n\nPlease try again with a different query or try using a different model.",
-
-                    "status": "error"
-
-                }) + "\n"
-
-                return
-
-                
-
-            if agent_name == "plan_not_formated_correctly":
-
+                    if formatted_response == RESPONSE_ERROR_INVALID_QUERY:
+                        yield json.dumps({
+                            "agent": agent_name,
+                            "content": formatted_response,
+                            "status": "error"
+                        }) + "\n"
+                        return
+                        
+                    # if "code_combiner_agent" in agent_name:
+                    #     # logger.log_message(f"[>] Code combiner response: {response}", level=logging.INFO)
+                    #     total_response += str(response) if response else ""
+                    #     total_inputs += str(inputs) if inputs else ""
+
+                    # Send response chunk
+                    yield json.dumps({
+                        "agent": agent_name.split("__")[0] if "__" in agent_name else agent_name,
+                        "content": formatted_response,
+                        "status": "success" if response else "error"
+                    }) + "\n"
+                    
+                    # Track agent usage for future batch DB write
+                    if session_state.get("user_id"):
+                        agent_tokens = _estimate_tokens(
+                            ai_manager=app.state.ai_manager,
+                            input_text=str(inputs),
+                            output_text=str(response)
+                        )
+                        
+                        # Get appropriate model name for code combiner
+                        if "code_combiner_agent" in agent_name and "__" in agent_name:
+                            provider = agent_name.split("__")[1]
+                            model_name = _get_model_name_for_provider(provider)
+                        else:
+                            model_name = session_state.get("model_config", DEFAULT_MODEL_CONFIG)["model"]
+
+                        usage_records.append(_create_usage_record(
+                            session_state=session_state,
+                            model_name=model_name,
+                            prompt_tokens=agent_tokens["prompt"],
+                            completion_tokens=agent_tokens["completion"],
+                            query_size=len(str(inputs)),
+                            response_size=len(str(response)),
+                            processing_time_ms=int((time.time() - overall_start_time) * 1000),
+                            is_streaming=True
+                        ))
+                        
+            except asyncio.TimeoutError:
                 yield json.dumps({
-
-                    "agent": "Analytical Planner",
-
-                    "content": "**Something went wrong with formatting, retry the query!**",
-
+                    "agent": "planner",
+                    "content": "The request timed out. Please try a simpler query.",
                     "status": "error"
-
                 }) + "\n"
-
                 return
-
-            
-
-
-
-            formatted_response = format_response_to_markdown(
-
-                {agent_name: response}, 
-
-                datasets=session_state["datasets"]
-            ) 
-
-
-
-            yield json.dumps({
-
-                "agent": agent_name.split("__")[0] if "__" in agent_name else agent_name,
-
-                "content": formatted_response,
-
-                "status": "success" if response else "error"
-
-            }) + "\n"
-
-
-
-            # Handle agent errors
-
-            if isinstance(response, dict) and "error" in response:
-
-                yield json.dumps({
-
-                    "agent": agent_name,
-
-                    "content": f"**Error in {agent_name}**: {response['error']}",
-
-                    "status": "error"
-
-                }) + "\n"
-
-                continue  # Continue with next agent instead of returning
-
-
-
-
-
-
-
-            if formatted_response == RESPONSE_ERROR_INVALID_QUERY:
-
+            except Exception as e:
+                logger.log_message(f"Error in streaming response: {str(e)}", level=logging.ERROR)
                 yield json.dumps({
-
-                    "agent": agent_name,
-
-                    "content": formatted_response,
-
+                    "agent": "planner",
+                    "content": "An error occurred while generating responses. Please try again!",
                     "status": "error"
-
                 }) + "\n"
-
-                continue  # Continue with next agent instead of returning
-
-
-
-            # Send response chunk
-
-
-
-            
-
-            # Track agent usage for future batch DB write
-
-            if session_state.get("user_id"):
-
-                agent_tokens = _estimate_tokens(
-
-                    ai_manager=app.state.ai_manager,
-
-                    input_text=str(inputs),
-
-                    output_text=str(response)
-
-                )
-
-                
-
-                # Get appropriate model name for code combiner
-
-                if "code_combiner_agent" in agent_name and "__" in agent_name:
-
-                    provider = agent_name.split("__")[1]
-
-                    model_name = _get_model_name_for_provider(provider)
-
-                else:
-
-                    model_name = session_state.get("model_config", DEFAULT_MODEL_CONFIG)["model"]
-
-
-
-                usage_records.append(_create_usage_record(
-
-                    session_state=session_state,
-
-                    model_name=model_name,
-
-                    prompt_tokens=agent_tokens["prompt"],
-
-                    completion_tokens=agent_tokens["completion"],
-
-                    query_size=len(str(inputs)),
-
-                    response_size=len(str(response)),
-
-                    processing_time_ms=int((time.time() - overall_start_time) * 1000),
-
-                    is_streaming=True
-
-                ))
-
-                    
-
-        # except asyncio.TimeoutError:
-
-        #     yield json.dumps({
-
-        #         "agent": "planner",
-
-        #         "content": "The request timed out. Please try a simpler query.",
-
-        #         "status": "error"
-
-        #     }) + "\n"
-
-        #     return
-
-            
-
-        # except Exception as e:
-
-        #     logger.log_message(f"Error executing plan: {str(e)}", level=logging.ERROR)
-
-        #     yield json.dumps({
-
-        #         "agent": "planner",
-
-        #         "content": f"An error occurred while executing the plan: {str(e)}",
-
-        #         "status": "error"
-
-        #     }) + "\n"
-
-        #     return
-
                 
-
-    # except Exception as e:
-
-    #         logger.log_message(f"Error in streaming response: {str(e)}", level=logging.ERROR)
-
-    #         yield json.dumps({
-
-    #             "agent": "planner",
-
-    #             "content": "An error occurred while generating responses. Please try again!" + str(e) + str({k: v for k, v in session_lm.__dict__['kwargs'].items() if k != 'api_key'}),
-
-    #             "status": "error"
-
-    #         }) + "\n"
-
-
-
+            # Batch write usage records to DB
+            if usage_records and session_state.get("user_id"):
+                try:
+                    # In a real implementation, you would batch these writes
+                    # For now, we're writing them one by one but could be optimized
+                    ai_manager = app.state.get_ai_manager()
+                    for record in usage_records:
+                        ai_manager.save_usage_to_db(**record)
+                except Exception as db_error:
+                    logger.log_message(f"Failed to save usage records: {str(db_error)}", level=logging.ERROR)
+           
+    except Exception as e:
+        logger.log_message(f"Streaming response generation failed: {str(e)}", level=logging.ERROR)
+        yield json.dumps({
+            "agent": "planner",
+            "content": "An error occurred while generating responses. Please try again!",
+            "status": "error"
+        }) + "\n"
 
 
 def _estimate_tokens(ai_manager, input_text: str, output_text: str) -> dict:
-
     """Estimate token counts, with fallback for tokenization errors"""
-
     try:
-
         # Try exact tokenization
-
         prompt_tokens = len(ai_manager.tokenizer.encode(input_text))
-
         completion_tokens = len(ai_manager.tokenizer.encode(output_text))
-
     except Exception:
-
         # Fall back to estimation
-
         prompt_words = len(input_text.split())
-
         completion_words = len(output_text.split())
-
         prompt_tokens = int(prompt_words * DEFAULT_TOKEN_RATIO)
-
         completion_tokens = int(completion_words * DEFAULT_TOKEN_RATIO)
-
     
-
     return {
-
         "prompt": prompt_tokens,
-
         "completion": completion_tokens,
-
         "total": prompt_tokens + completion_tokens
-
     }
 
 
-
-
-
 def _create_usage_record(session_state: dict, model_name: str, prompt_tokens: int, 
-
                         completion_tokens: int, query_size: int, response_size: int,
-
                         processing_time_ms: int, is_streaming: bool) -> dict:
-
     """Create a usage record for the database"""
-
     ai_manager = app.state.get_ai_manager()
-
     provider = ai_manager.get_provider_for_model(model_name)
-
     cost = ai_manager.calculate_cost(model_name, prompt_tokens, completion_tokens)
-
     
-
     return {
-
         "user_id": session_state.get("user_id"),
-
         "chat_id": session_state.get("chat_id"),
-
         "model_name": model_name,
-
         "provider": provider,
-
         "prompt_tokens": int(prompt_tokens),
-
         "completion_tokens": int(completion_tokens),
-
         "total_tokens": int(prompt_tokens + completion_tokens),
-
         "query_size": query_size,
-
         "response_size": response_size,
-
         "cost": round(cost, 7),
-
         "request_time_ms": processing_time_ms,
-
         "is_streaming": is_streaming
-
     }
 
 
-
-
-
 def _get_model_name_for_provider(provider: str) -> str:
-
     """Get the model name for a provider"""
-
     provider_model_map = {
-
-        "openai": "gpt-5.4",
-        "anthropic": "claude-sonnet-4-6",
+        "openai": "o3-mini",
+        "anthropic": "claude-3-7-sonnet-latest",
         "gemini": "gemini-2.5-pro-preview-03-25"
     }
-    return provider_model_map.get(provider, "gpt-5.4")
-
-
-
-
-
-
+    return provider_model_map.get(provider, "o3-mini")
 
-# Add an endpoint to list available agents
-
-@app.get("/agents", response_model=dict)
-
-async def list_agents(request: Request, session_id: str = Depends(get_session_id_dependency)):
-
-    """Get all available agents (standard, template, and custom)"""
-
-    session_state = app.state.get_session_state(session_id)
-
-    
 
+async def _execute_plan_with_timeout(ai_system, enhanced_query, plan_response):
+    """Execute the plan with timeout handling for each step"""
     try:
-
-        # Get all available agents from database and session
-
-        available_agents_list = _get_available_agents_list(session_state)
-
-        
-
-        # Categorize agents
-
-        standard_agents = ["preprocessing_agent", "statistical_analytics_agent", "sk_learn_agent", "data_viz_agent"]
-
-        
-
-        # Get template agents from database
-
-        from src.db.init_db import session_factory
-
-        from src.agents.agents import load_all_available_templates_from_db
-
-        
-
-        db_session = session_factory()
-
-        try:
-
-            template_agents_dict = load_all_available_templates_from_db(db_session)
-
-            # template_agents_dict is a dict with template_name as keys
-
-            template_agents = [template_name for template_name in template_agents_dict.keys() 
-
-                             if template_name not in standard_agents and template_name != 'basic_qa_agent']
-
-        except Exception as e:
-
-            logger.log_message(f"Error loading template agents in /agents endpoint: {str(e)}", level=logging.ERROR)
-
-            template_agents = []
-
-        finally:
-
-            db_session.close()
-
-        
-
-        # Get custom agents from session
-
-        custom_agents = []
-
-        if session_state and "ai_system" in session_state:
-
-            ai_system = session_state["ai_system"]
-
-            if hasattr(ai_system, 'agents'):
-
-                custom_agents = [agent for agent in available_agents_list
-
-                               if agent not in standard_agents and agent not in template_agents]
-
-        
-
-        # Ensure template agents are in the available list
-
-        for template_agent in template_agents:
-
-            if template_agent not in available_agents_list:
-
-                available_agents_list.append(template_agent)
-
-        
-
-        return {
-
-            "available_agents": available_agents_list,
-
-            "standard_agents": standard_agents,
-
-            "template_agents": template_agents,
-
-            "custom_agents": custom_agents
-
-        }
-
+        # Use asyncio.create_task to run the execute_plan coroutine
+        async for agent_name, inputs, response in ai_system.execute_plan(enhanced_query, plan_response):
+            # Yield results as they come
+            yield agent_name, inputs, response
     except Exception as e:
-
-        logger.log_message(f"Error getting agents list: {str(e)}", level=logging.ERROR)
-
-        raise HTTPException(status_code=500, detail=f"Error getting agents list: {str(e)}")
+        logger.log_message(f"Error executing plan: {str(e)}", level=logging.ERROR)
+        yield "error", None, {"error": "An error occurred during plan execution"}
 
 
+# Add an endpoint to list available agents
+@app.get("/agents", response_model=dict)
+async def list_agents():
+    return {
+        "available_agents": list(AVAILABLE_AGENTS.keys()),
+        "description": "List of available specialized agents that can be called using @agent_name"
+    }
 
 @app.get("/health", response_model=dict)
-
 async def health():
-
     return {"message": "API is healthy and running"}
 
-
-
 @app.get("/")
-
 async def index():
-
     return {
-
         "title": "Welcome to the AI Analytics API",
-
         "message": "Explore our API for advanced analytics and visualization tools designed to empower your data-driven decisions.",
-
         "description": "Utilize our powerful agents and models to gain insights from your data effortlessly.",
-
         "colors": {
-
             "primary": "#007bff",
-
             "secondary": "#6c757d",
-
             "success": "#28a745",
-
             "danger": "#dc3545",
-
         },
-
         "features": [
-
             "Real-time data processing",
-
             "Customizable visualizations",
-
             "Seamless integration with various data sources",
-
             "User-friendly interface for easy navigation",
-
             "Custom Analytics",
-
         ],
-
     }
 
-
-
 @app.post("/chat_history_name")
-
 async def chat_history_name(request: dict, session_id: str = Depends(get_session_id_dependency)):
-
     query = request.get("query")
-
     name = None
-
     
-
-    lm = dspy.LM(model="openai/gpt-5-nano", max_tokens=300, temperature=0.5, api_key=os.getenv("OPENAI_API_KEY"))
-
+    lm = dspy.LM(model="gpt-4o-mini", max_tokens=300, temperature=0.5)
     
-
     with dspy.context(lm=lm):
-
         name = app.state.get_chat_history_name_agent()(query=str(query))
-
         
-
     return {"name": name.name if name else "New Chat"}
 
-
-
-@app.post("/deep_analysis_streaming")
-
-async def deep_analysis_streaming(
-
-    request: DeepAnalysisRequest,
-
-    request_obj: Request,
-
-    session_id: str = Depends(get_session_id_dependency)
-
-):
-
-    """Perform streaming deep analysis with real-time updates"""
-
-    session_state = app.state.get_session_state(session_id)
-
-    
-
-    try:
-
-        # Extract and validate query parameters
-
-        _update_session_from_query_params(request_obj, session_state)
-
-        
-
-        # Validate dataset
-
-        if session_state["datasets"] is None:
-            raise HTTPException(status_code=400, detail=RESPONSE_ERROR_NO_DATASET)
-
-        
-
-        # Get user_id from session state (if available)
-
-        user_id = session_state.get("user_id")
-
-        
-
-        # Generate a UUID for this report
-
-        import uuid
-
-        report_uuid = str(uuid.uuid4())
-
-        
-
-        # Create initial pending report in the database
-
-        try:
-
-            from src.db.init_db import session_factory
-
-            from src.db.schemas.models import DeepAnalysisReport
-
-            
-
-            db_session = session_factory()
-
-            
-
-            try:
-
-                # Create a pending report entry
-
-                new_report = DeepAnalysisReport(
-
-                    report_uuid=report_uuid,
-
-                    user_id=user_id,
-
-                    goal=request.goal,
-
-                    status="pending",
-
-                    start_time=datetime.now(UTC),
-
-                    progress_percentage=0
-
-                )
-
-                
-
-                db_session.add(new_report)
-
-                db_session.commit()
-
-                db_session.refresh(new_report)
-
-                
-
-                # Store the report ID in session state for later updates
-
-                session_state["current_deep_analysis_id"] = new_report.report_id
-
-                session_state["current_deep_analysis_uuid"] = report_uuid
-
-                
-
-            except Exception as e:
-
-                logger.log_message(f"Error creating initial deep analysis report: {str(e)}", level=logging.ERROR)
-
-                # Continue even if DB storage fails
-
-            finally:
-
-                db_session.close()
-
-                
-
-        except Exception as e:
-
-            logger.log_message(f"Database operation failed: {str(e)}", level=logging.ERROR)
-
-            # Continue even if DB operation fails
-
-        
-
-        # Get session-specific model
-
-        # session_lm = get_session_lm(session_state)
-
-        session_lm = dspy.LM(model="anthropic/claude-sonnet-4-6", max_tokens=7000, temperature=0.5)
-
-        
-
-        return StreamingResponse(
-
-            _generate_deep_analysis_stream(session_state, request.goal, session_lm, session_id),
-
-            media_type='text/event-stream',
-
-            headers={
-
-                'Cache-Control': 'no-cache',
-
-                'Connection': 'keep-alive',
-
-                'Content-Type': 'text/event-stream',
-
-                'Access-Control-Allow-Origin': '*',
-
-                'X-Accel-Buffering': 'no'
-
-            }
-
-        )
-
-        
-
-    except HTTPException:
-
-        raise
-
-    except Exception as e:
-
-        logger.log_message(f"Streaming deep analysis failed: {str(e)}", level=logging.ERROR)
-
-        raise HTTPException(status_code=500, detail=f"Streaming deep analysis failed: {str(e)}")
-
-
-
-async def _generate_deep_analysis_stream(session_state: dict, goal: str, session_lm, session_id: str):
-
-    """Generate streaming responses for deep analysis"""
-
-    # Track the start time for duration calculation
-
-    start_time = datetime.now(UTC)
-
-    
-
-    try:
-
-        # Get dataset info
-        datasets = session_state["datasets"]
-        desc = session_state['description']
-        
-        # Generate dataset info for all datasets
-        logger.log_message(f"🔍 DEEP ANALYSIS START - datasets type: {type(datasets)}, keys: {list(datasets.keys()) if datasets else 'None'}", level=logging.DEBUG)
-        
-        dataset_info = desc
-        logger.log_message(f"🔍 DEEP ANALYSIS - dataset_info type: {type(dataset_info)}, length: {len(dataset_info) if isinstance(dataset_info, str) else 'N/A'}", level=logging.DEBUG)
-        logger.log_message(f"🔍 DEEP ANALYSIS - dataset_info content: {dataset_info[:200]}...", level=logging.DEBUG)
-
-        
-
-        # Get report info from session state
-
-        report_id = session_state.get("current_deep_analysis_id")
-
-        report_uuid = session_state.get("current_deep_analysis_uuid")
-
-        user_id = session_state.get("user_id")
-
-        
-
-        # Helper function to update report in database
-
-        async def update_report_in_db(status, progress, step=None, content=None):
-
-            if not report_id:
-
-                return
-
-                
-
-            try:
-
-                from src.db.init_db import session_factory
-
-                from src.db.schemas.models import DeepAnalysisReport
-
-                
-
-                db_session = session_factory()
-
-                
-
-                try:
-
-                    report = db_session.query(DeepAnalysisReport).filter(DeepAnalysisReport.report_id == report_id).first()
-
-                    
-
-                    if report:
-
-                        report.status = status
-
-                        report.progress_percentage = progress
-
-                        
-
-                        # Update step-specific fields if provided
-
-                        if step == "questions" and content:
-
-                            report.deep_questions = content
-
-                        elif step == "planning" and content:
-
-                            report.deep_plan = content
-
-                        elif step == "analysis" and content:
-
-                            # For analysis step, we get the full object with multiple fields
-
-                            if isinstance(content, dict):
-
-                                # Update fields from content if they exist
-
-                                if "deep_questions" in content and content["deep_questions"]:
-
-                                    report.deep_questions = content["deep_questions"]
-
-                                if "deep_plan" in content and content["deep_plan"]:
-
-                                    report.deep_plan = content["deep_plan"]
-
-                                if "code" in content and content["code"]:
-
-                                    report.analysis_code = content["code"]
-
-                                if "final_conclusion" in content and content["final_conclusion"]:
-
-                                    report.final_conclusion = content["final_conclusion"]
-
-                                    # Also update summary from conclusion
-
-                                    conclusion = content["final_conclusion"]
-
-                                    conclusion = conclusion.replace("**Conclusion**", "")
-
-                                    report.report_summary = conclusion[:200] + "..." if len(conclusion) > 200 else conclusion
-
-                                
-
-                                # Handle JSON fields
-
-                                if "summaries" in content and content["summaries"]:
-
-                                    report.summaries = json.dumps(content["summaries"])
-
-                                if "plotly_figs" in content and content["plotly_figs"]:
-
-                                    report.plotly_figures = json.dumps(content["plotly_figs"])
-
-                                if "synthesis" in content and content["synthesis"]:
-
-                                    report.synthesis = json.dumps(content["synthesis"])
-
-                        
-
-                        # For the final step, update the HTML report
-
-                        if step == "completed":
-
-                            if content:
-
-                                report.html_report = content
-
-                            else:
-
-                                logger.log_message("No HTML content provided for completed step", level=logging.WARNING)
-
-                                
-
-                            report.end_time = datetime.now(UTC)
-
-                            # Ensure start_time is timezone-aware before calculating duration
-
-                            if report.start_time.tzinfo is None:
-
-                                start_time_utc = report.start_time.replace(tzinfo=UTC)
-
-                            else:
-
-                                start_time_utc = report.start_time
-
-                            report.duration_seconds = int((report.end_time - start_time_utc).total_seconds())
-
-                            
-
-                        report.updated_at = datetime.now(UTC)
-
-                        db_session.commit()
-
-                        
-
-                except Exception as e:
-
-                    db_session.rollback()
-
-                    logger.log_message(f"Error updating deep analysis report: {str(e)}", level=logging.ERROR)
-
-                finally:
-
-                    db_session.close()
-
-            except Exception as e:
-
-                logger.log_message(f"Database operation failed: {str(e)}", level=logging.ERROR)
-
-        
-
-        # Use session model for this request
-
-        with dspy.context(lm=session_lm):
-
-            # Send initial status
-
-            yield json.dumps({
-
-                "step": "initialization",
-
-                "status": "starting",
-
-                "message": "Initializing deep analysis...",
-
-                "progress": 5
-
-            }) + "\n"
-
-            
-
-            # Update DB status to running
-
-            await update_report_in_db("running", 5)
-
-            
-
-            # Get deep analyzer - use the correct session_id from the session_state
-
-            logger.log_message(f"Getting deep analyzer for session_id: {session_id}, user_id: {user_id}", level=logging.INFO)
-
-            deep_analyzer = app.state.get_deep_analyzer(session_id)
-
-            
-
-            # Make all datasets available globally for code execution
-
-            for dataset_name, dataset_df in datasets.items():
-                globals()[dataset_name] = dataset_df
-            
-            # Use the new streaming method and forward all progress updates
-            final_result = None
-
-            logger.log_message(f"🔍 CALLING DEEP ANALYSIS - goal: {goal[:100]}...", level=logging.DEBUG)
-            logger.log_message(f"🔍 CALLING DEEP ANALYSIS - dataset_info type: {type(dataset_info)}, length: {len(dataset_info) if isinstance(dataset_info, str) else 'N/A'}", level=logging.DEBUG)
-            logger.log_message(f"🔍 CALLING DEEP ANALYSIS - session_datasets type: {type(datasets)}, keys: {list(datasets.keys()) if datasets else 'None'}", level=logging.DEBUG)
-            
-            async for update in deep_analyzer.execute_deep_analysis_streaming(
-                goal=goal,
-                dataset_info=dataset_info,
-                session_datasets=datasets  # Pass all datasets instead of single df
-            ):
-
-                # Convert the update to the expected format and yield it
-
-                if update.get("step") == "questions" and update.get("status") == "completed":
-
-                    # Update DB with questions
-
-                    await update_report_in_db("running", update.get("progress", 0), "questions", update.get("content"))
-
-                elif update.get("step") == "planning" and update.get("status") == "completed":
-
-                    # Update DB with planning
-
-                    await update_report_in_db("running", update.get("progress", 0), "planning", update.get("content"))
-
-                elif update.get("step") == "conclusion" and update.get("status") == "completed":
-
-                    # Store the final result for later processing
-
-                    final_result = update.get("final_result")
-
-                    
-
-                    # Convert Plotly figures to JSON format for network transmission
-
-                    if final_result:
-
-                        import plotly.io
-
-                        serialized_return_dict = final_result.copy()
-
-                        
-
-                        # Convert plotly_figs to JSON format
-
-                        if 'plotly_figs' in serialized_return_dict and serialized_return_dict['plotly_figs']:
-
-                            json_figs = []
-
-                            for fig_list in serialized_return_dict['plotly_figs']:
-
-                                if isinstance(fig_list, list):
-
-                                    json_fig_list = []
-
-                                    for fig in fig_list:
-
-                                        if hasattr(fig, 'to_json'):  # Check if it's a Plotly figure
-
-                                            json_fig_list.append(plotly.io.to_json(fig))
-
-                                        else:
-
-                                            json_fig_list.append(fig)  # Already JSON or other format
-
-                                    json_figs.append(json_fig_list)
-
-                                else:
-
-                                    # Single figure case
-
-                                    if hasattr(fig_list, 'to_json'):
-
-                                        json_figs.append(plotly.io.to_json(fig_list))
-
-                                    else:
-
-                                        json_figs.append(fig_list)
-
-                            serialized_return_dict['plotly_figs'] = json_figs
-
-                        
-
-                        # Update DB with analysis results
-
-                        await update_report_in_db("running", update.get("progress", 0), "analysis", serialized_return_dict)
-
-                        
-
-                        # Generate HTML report using the original final_result with Figure objects
-
-                        html_report = None
-
-                        try:
-
-                            html_report = generate_html_report(final_result)
-
-                        except Exception as e:
-
-                            logger.log_message(f"Error generating HTML report: {str(e)}", level=logging.ERROR)
-
-                            # Continue even if HTML generation fails
-
-                        
-
-                        # Send the analysis results
-
-                        yield json.dumps({
-
-                            "step": "analysis",
-
-                            "status": "completed",
-
-                            "content": serialized_return_dict,
-
-                            "progress": 90
-
-                        }) + "\n"
-
-                        
-
-                        # Send report generation status
-
-                        yield json.dumps({
-
-                            "step": "report",
-
-                            "status": "processing",
-
-                            "message": "Generating final report...",
-
-                            "progress": 95
-
-                        }) + "\n"
-
-                        
-
-                        # Send final completion
-
-                        yield json.dumps({
-
-                            "step": "completed",
-
-                            "status": "success",
-
-                            "analysis": serialized_return_dict,
-
-                            "html_report": html_report,
-
-                            "progress": 100
-
-                        }) + "\n"
-
-                        
-
-                        # Update DB with completed report (with HTML if generated)
-
-                        if html_report:
-
-                            logger.log_message(f"Saving HTML report to database, length: {len(html_report)}", level=logging.INFO)
-
-                        else:
-
-                            logger.log_message("No HTML report to save to database", level=logging.WARNING)
-
-                        await update_report_in_db("completed", 100, "completed", html_report)
-
-                elif update.get("step") == "error":
-
-                    # Forward error directly
-
-                    yield json.dumps(update) + "\n"
-
-                    await update_report_in_db("failed", 0)
-
-                    return
-
-                else:
-
-                    # Forward all other progress updates
-
-                    yield json.dumps(update) + "\n"
-
-            
-
-            # If we somehow exit the loop without getting a final result, that's an error
-
-            if not final_result:
-
-                yield json.dumps({
-
-                    "step": "error",
-
-                    "status": "failed",
-
-                    "message": "Deep analysis completed without final result",
-
-                    "progress": 0
-
-                }) + "\n"
-
-                await update_report_in_db("failed", 0)
-
-        
-
-    except Exception as e:
-
-        logger.log_message(f"Error in deep analysis stream: {str(e)}", level=logging.ERROR)
-
-        yield json.dumps({
-
-            "step": "error",
-
-            "status": "failed",
-
-            "message": f"Deep analysis failed: {str(e)}",
-
-            "progress": 0
-
-        }) + "\n"
-
-        
-
-        # Update DB with error status
-
-        if 'update_report_in_db' in locals() and session_state.get("current_deep_analysis_id"):
-
-            await update_report_in_db("failed", 0)
-
-
-
-@app.post("/deep_analysis/download_report")
-
-async def download_html_report(
-
-    request: dict,
-
-    session_id: str = Depends(get_session_id_dependency)
-
-):
-
-    """Download HTML report from previous deep analysis"""
-
-    try:
-
-        analysis_data = request.get("analysis_data")
-
-        if not analysis_data:
-
-            raise HTTPException(status_code=400, detail="No analysis data provided")
-
-        
-
-        # Get report UUID from request if available (for saving to DB)
-
-        report_uuid = request.get("report_uuid")
-
-        session_state = app.state.get_session_state(session_id)
-
-        
-
-        # If no report_uuid in request, try to get it from session state
-
-        if not report_uuid and session_state.get("current_deep_analysis_uuid"):
-
-            report_uuid = session_state.get("current_deep_analysis_uuid")
-
-            
-
-        # Convert JSON-serialized Plotly figures back to Figure objects for HTML generation
-
-        processed_data = analysis_data.copy()
-
-        
-
-        if 'plotly_figs' in processed_data and processed_data['plotly_figs']:
-
-            import plotly.io
-
-            import plotly.graph_objects as go
-
-            
-
-            figure_objects = []
-
-            for fig_list in processed_data['plotly_figs']:
-
-                if isinstance(fig_list, list):
-
-                    fig_obj_list = []
-
-                    for fig_json in fig_list:
-
-                        if isinstance(fig_json, str):
-
-                            # Convert JSON string back to Figure object
-
-                            try:
-
-                                fig_obj = plotly.io.from_json(fig_json)
-
-                                fig_obj_list.append(fig_obj)
-
-                            except Exception as e:
-
-                                logger.log_message(f"Error parsing Plotly JSON: {str(e)}", level=logging.WARNING)
-
-                                continue
-
-                        elif hasattr(fig_json, 'to_html'):
-
-                            # Already a Figure object
-
-                            fig_obj_list.append(fig_json)
-
-                    figure_objects.append(fig_obj_list)
-
-                else:
-
-                    # Single figure case
-
-                    if isinstance(fig_list, str):
-
-                        try:
-
-                            fig_obj = plotly.io.from_json(fig_list)
-
-                            figure_objects.append(fig_obj)
-
-                        except Exception as e:
-
-                            logger.log_message(f"Error parsing Plotly JSON: {str(e)}", level=logging.WARNING)
-
-                            continue
-
-                    elif hasattr(fig_list, 'to_html'):
-
-                        figure_objects.append(fig_list)
-
-            
-
-            processed_data['plotly_figs'] = figure_objects
-
-        
-
-        # Generate HTML report
-
-        html_report = generate_html_report(processed_data)
-
-        
-
-        # Save report to database if we have a UUID
-
-        if report_uuid:
-
-            try:
-
-                from src.db.init_db import session_factory
-
-                from src.db.schemas.models import DeepAnalysisReport
-
-                
-
-                db_session = session_factory()
-
-                try:
-
-                    # Try to find existing report by UUID
-
-                    report = db_session.query(DeepAnalysisReport).filter(DeepAnalysisReport.report_uuid == report_uuid).first()
-
-                    
-
-                    if report:
-
-                        # Update existing report with HTML content
-
-                        report.html_report = html_report
-
-                        report.updated_at = datetime.now(UTC)
-
-                        db_session.commit()
-
-                except Exception as e:
-
-                    db_session.rollback()
-
-                finally:
-
-                    db_session.close()
-
-            except Exception as e:
-
-                logger.log_message(f"Database operation failed when storing HTML report: {str(e)}", level=logging.ERROR)
-
-                # Continue even if DB storage fails
-
-        
-
-        # Create a filename with timestamp
-
-        timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
-
-        filename = f"deep_analysis_report_{timestamp}.html"
-
-        
-
-        # Return as downloadable file
-
-        return StreamingResponse(
-
-            iter([html_report.encode('utf-8')]),
-
-            media_type='text/html',
-
-            headers={
-
-                'Content-Disposition': f'attachment; filename="{filename}"',
-
-                'Content-Type': 'text/html; charset=utf-8'
-
-            }
-
-        )
-
-        
-
-    except Exception as e:
-
-        logger.log_message(f"Failed to generate HTML report: {str(e)}", level=logging.ERROR)
-
-        raise HTTPException(status_code=500, detail=f"Failed to generate report: {str(e)}")
-
-
-
-
-
-# In the section where routers are included, add the session_router
-
-app.include_router(chat_router)
-
-app.include_router(analytics_router)
-
-app.include_router(code_router)
-
-app.include_router(session_router)
-
-app.include_router(feedback_router)
-
-app.include_router(deep_analysis_router)
-
-app.include_router(templates_router)
-
-app.include_router(blog_router)
-
-
+# In the section where routers are included, add the session_router
+app.include_router(chat_router)
+app.include_router(analytics_router)
+app.include_router(code_router)
+app.include_router(session_router)
+app.include_router(feedback_router)
 
 if __name__ == "__main__":
-
-    port = int(os.environ.get("PORT", 8000))
-
-    uvicorn.run(app, host="0.0.0.0", port=port)
-
+    uvicorn.run(app, host="0.0.0.0", port=8000)

chat_database.db

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6403c5e1c61ed80d24f1b33d27a5a734121e1bad55e311bab41cf7349e7af23
+size 36864

docs/README.md

@@ -1,251 +0,0 @@
-# Auto-Analyst Backend Documentation
-
-This directory contains comprehensive documentation for the Auto-Analyst backend - a sophisticated multi-agent AI platform for data analysis built with FastAPI, DSPy, and modern Python technologies.
-
-## 📁 Documentation Structure
-
-### **🏗️ Architecture** (`/architecture/`)
-- **[System Architecture](./architecture/architecture.md)** - Comprehensive overview of backend system design, components, and data flow patterns
-
-### **🚀 Development** (`/development/`)
-- **[Development Workflow](./development/development_workflow.md)** - Complete development guide with patterns, best practices, and code organization principles
-
-### **🔧 System** (`/system/`)
-- **[Database Schema](./system/database-schema.md)** - Complete database schema with all tables, relationships, and performance optimization
-- **[Shared DataFrame System](./system/shared_dataframe.md)** - Inter-agent data sharing and session management
-
-### **🌐 API** (`/api/`)
-- **[API Endpoints Overview](./api/README.md)** - Main API reference hub
-- **[Route Documentation](./api/routes/)** - Detailed endpoint documentation:
-  - **[Core Routes](./api/routes/session.md)** - File uploads, sessions, authentication
-  - **[Chat Routes](./api/routes/chats.md)** - Chat and messaging endpoints
-  - **[Code Routes](./api/routes/code.md)** - Code execution and processing
-  - **[Analytics Routes](./api/routes/analytics.md)** - Usage analytics and monitoring
-  - **[Deep Analysis Routes](./api/routes/deep_analysis.md)** - Multi-agent analysis system
-  - **[Template Routes](./api/routes/templates.md)** - Agent template management
-  - **[Feedback Routes](./api/routes/feedback.md)** - User feedback and rating system
-
-### **🐛 Troubleshooting** (`/troubleshooting/`)
-- **[Troubleshooting Guide](./troubleshooting/troubleshooting.md)** - Common issues, debugging tools, and solutions
-
-## 🎯 Backend Overview
-
-### **Tech Stack**
-- **FastAPI** - Modern async Python web framework
-- **DSPy** - AI agent orchestration and LLM integration
-- **SQLAlchemy** - Database ORM with PostgreSQL/SQLite support
-- **Plotly** - Interactive data visualizations
-- **Pandas/NumPy** - Data manipulation and analysis
-- **Scikit-learn** - Machine learning models
-- **Statsmodels** - Statistical analysis
-
-### **Core Features**
-- **Multi-Agent System** - 4+ specialized AI agents for different analysis tasks
-- **Template System** - User-customizable agent configurations
-- **Deep Analysis** - Multi-step analytical workflows with streaming progress
-- **Session Management** - Stateful user sessions with shared data context
-- **Code Execution** - Safe Python code execution environment
-- **Real-time Streaming** - WebSocket support for live analysis updates
-
-### **Agent Types**
-1. **Data Preprocessing Agent** - Data cleaning and preparation
-2. **Statistical Analytics Agent** - Statistical analysis using statsmodels
-3. **Machine Learning Agent** - ML modeling with scikit-learn
-4. **Data Visualization Agent** - Interactive charts with Plotly
-5. **Feature Engineering Agent** (Premium) - Advanced feature creation
-6. **Polars Agent** (Premium) - High-performance data processing
-
-## 🚀 Quick Start Guide
-
-### **1. Environment Setup**
-
-```bash
-# Navigate to backend directory
-cd Auto-Analyst-CS/auto-analyst-backend
-
-# Create virtual environment
-python -m venv venv
-source venv/bin/activate  # Linux/Mac
-venv\Scripts\activate     # Windows
-
-# Install dependencies
-pip install -r requirements.txt
-```
-
-### **2. Environment Configuration**
-
-Create `.env` file with required variables:
-
-```env
-# Database Configuration
-DATABASE_URL=sqlite:///./chat_database.db
-
-# AI Model Configuration
-OPENAI_API_KEY=your-openai-api-key
-MODEL_PROVIDER=openai  # openai, anthropic, groq, gemini
-MODEL_NAME=gpt-4o-mini
-TEMPERATURE=0.7
-MAX_TOKENS=6000
-
-# Optional: Additional AI Providers
-ANTHROPIC_API_KEY=your-anthropic-key
-GROQ_API_KEY=your-groq-key
-GEMINI_API_KEY=your-gemini-key
-
-# Security
-ADMIN_API_KEY=your-admin-key
-
-# Application Settings
-ENVIRONMENT=development
-FRONTEND_URL=http://localhost:3000/
-```
-
-### **3. Database Initialization**
-
-```bash
-# Initialize database and default agents
-python -c "
-from src.db.init_db import init_db
-init_db()
-print('✅ Database and agents initialized successfully')
-"
-```
-
-### **4. Start Development Server**
-
-```bash
-# Start the FastAPI server
-python -m app
-
-# Or with uvicorn for more control
-uvicorn app:app --reload --host 0.0.0.0 --port 8000
-```
-
-### **5. Verify Installation**
-
-- **API Documentation**: `http://localhost:8000/docs`
-- **Health Check**: `http://localhost:8000/health`
-
-## 🔧 Development Workflow
-
-### **Adding New Agents**
-
-1. **Define Agent Signature** in `src/agents/agents.py`
-2. **Add Configuration** to `agents_config.json`
-3. **Register Agent** in loading system
-4. **Test Integration** with multi-agent pipeline
-
-### **Adding New API Endpoints**
-
-1. **Create Route File** in `src/routes/`
-2. **Define Pydantic Models** for request/response
-3. **Implement Endpoints** with proper error handling
-4. **Register Router** in `app.py`
-5. **Update Documentation**
-
-### **Database Changes**
-
-1. **Modify Models** in `src/db/schemas/models.py`
-2. **Create Migration**: `alembic revision --autogenerate -m "description"`
-3. **Apply Migration**: `alembic upgrade head`
-4. **Update Documentation**
-
-## 📊 System Architecture
-
-### **Request Processing Flow**
-```
-HTTP Request → FastAPI Router → Route Handler → Business Logic → 
-Database/Agent System → AI Model → Response Processing → JSON Response
-```
-
-### **Agent Execution Flow**
-```
-User Query → Session Manager → Agent Selection → Context Preparation → 
-DSPy Chain → AI Model → Code Generation → Execution → Response Formatting
-```
-
-### **Deep Analysis Workflow**
-```
-Goal Input → Question Generation → Planning → Multi-Agent Execution → 
-Code Synthesis → Result Compilation → HTML Report Generation
-```
-
-## 🧪 Testing & Validation
-
-### **API Testing**
-```bash
-# Interactive documentation
-open http://localhost:8000/docs
-
-# cURL examples
-curl -X GET "http://localhost:8000/health"
-curl -X POST "http://localhost:8000/chat/preprocessing_agent" \
-  -H "Content-Type: application/json" \
-  -d '{"query": "Clean this dataset", "session_id": "test"}'
-```
-
-### **Agent Testing**
-```python
-# Test individual agents
-from src.agents.agents import preprocessing_agent
-import dspy
-
-# Configure DSPy
-lm = dspy.LM('openai/gpt-4o-mini', api_key='your-key')
-dspy.configure(lm=lm)
-
-# Test agent
-agent = dspy.ChainOfThought(preprocessing_agent)
-result = agent(goal='clean data', dataset='test dataset')
-print(result)
-```
-
-## 🔒 Security & Production
-
-### **Security Features**
-- **Session-based authentication** with secure session management
-- **API key protection** for admin endpoints
-- **Input validation** using Pydantic models
-- **Error handling** with proper HTTP status codes
-- **CORS configuration** for frontend integration
-
-### **Production Considerations**
-- **PostgreSQL database** for production deployment
-- **Environment variable management** for secrets
-- **Logging configuration** for monitoring
-- **Rate limiting** for API protection
-- **Performance optimization** for large datasets
-
-## 📈 Monitoring & Analytics
-
-The backend includes comprehensive analytics for:
-- **Usage tracking** - API endpoint usage and performance
-- **Model usage** - AI model consumption and costs
-- **User analytics** - User behavior and engagement
-- **Error monitoring** - System health and error tracking
-- **Performance metrics** - Response times and throughput
-
-## 🤝 Contributing
-
-1. **Follow coding standards** defined in development workflow
-2. **Add comprehensive tests** for new features
-3. **Update documentation** for all changes
-4. **Use proper error handling** patterns
-5. **Submit detailed pull requests** with clear descriptions
-
----
-
-## 📖 Detailed Documentation
-
-For specific implementation details, refer to the organized documentation in each subdirectory:
-
-- **[Getting Started Guide](./getting_started.md)** - Complete setup walkthrough
-- **[Architecture Documentation](./architecture/)** - System design and components
-- **[Development Guides](./development/)** - Workflow and best practices
-- **[API Reference](./api/)** - Complete endpoint documentation
-- **[System Documentation](./system/)** - Database and core systems
-- **[Troubleshooting](./troubleshooting/)** - Debugging and solutions
-
----
-
-**Need help?** Check the troubleshooting guide or refer to the comprehensive documentation in each section. 

docs/api/routes/analytics.md

@@ -1,562 +0,0 @@
-# Analytics Routes Documentation
-
-These routes provide comprehensive analytics functionality for the Auto-Analyst backend, including dashboard summaries, user analytics, model performance metrics, cost analysis, and system monitoring.
-
-## Authentication
-
-All analytics endpoints require admin authentication via API key:
-
-```python
-ADMIN_API_KEY = os.getenv("ADMIN_API_KEY", "default-admin-key-change-me")
-```
-
-The API key can be provided via:
-- **Header:** `X-Admin-API-Key`
-- **Query parameter:** `admin_api_key`
-
----
-
-## Dashboard Endpoints
-
-### **GET /analytics/dashboard**
-Returns comprehensive dashboard data combining usage statistics, model performance, and user activity.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-
-**Response:**
-```json
-{
-  "total_tokens": 123456,
-  "total_cost": 25.50,
-  "total_requests": 1000,
-  "total_users": 50,
-  "daily_usage": [
-    {
-      "date": "2023-05-01",
-      "tokens": 5000,
-      "cost": 1.25,
-      "requests": 100
-    }
-  ],
-  "model_usage": [
-    {
-      "model_name": "claude-3-sonnet-20241022",
-      "tokens": 10000,
-      "cost": 10.00,
-      "requests": 200
-    }
-  ],
-  "top_users": [
-    {
-      "user_id": "123",
-      "tokens": 5000,
-      "cost": 5.00,
-      "requests": 50
-    }
-  ],
-  "start_date": "2023-04-01",
-  "end_date": "2023-05-01"
-}
-```
-
-### **WebSocket /analytics/dashboard/realtime**
-WebSocket endpoint for real-time dashboard updates. Accepts connections and maintains them for broadcasting live data updates.
-
----
-
-## User Analytics Endpoints
-
-### **GET /analytics/users**
-Returns user list with usage statistics from the past 7 days.
-
-**Query Parameters:**
-- `limit` (optional): Maximum users to return (default: `100`)
-- `offset` (optional): Pagination offset (default: `0`)
-
-**Response:**
-```json
-{
-  "users": [
-    {
-      "user_id": "123",
-      "tokens": 5000,
-      "cost": 5.00,
-      "requests": 50,
-      "first_seen": "2023-04-01T12:00:00Z",
-      "last_seen": "2023-05-01T12:00:00Z"
-    }
-  ],
-  "total": 200,
-  "limit": 100,
-  "offset": 0
-}
-```
-
-### **GET /analytics/users/activity**
-Returns daily user activity metrics with new user tracking.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-
-**Response:**
-```json
-{
-  "user_activity": [
-    {
-      "date": "2023-05-01",
-      "activeUsers": 20,
-      "newUsers": 5,
-      "sessions": 30
-    }
-  ]
-}
-```
-
-### **GET /analytics/users/sessions/stats**
-Returns session statistics including total users, active users today, average queries per session, and average session time.
-
-**Response:**
-```json
-{
-  "totalUsers": 500,
-  "activeToday": 25,
-  "avgQueriesPerSession": 3.2,
-  "avgSessionTime": 300
-}
-```
-
-### **WebSocket /analytics/realtime**
-WebSocket endpoint for real-time user analytics updates.
-
----
-
-## Model Analytics Endpoints
-
-### **GET /analytics/usage/models**
-Returns model usage breakdown with performance metrics.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-
-**Response:**
-```json
-{
-  "model_usage": [
-    {
-      "model_name": "claude-3-sonnet-20241022",
-      "tokens": 10000,
-      "cost": 10.00,
-      "requests": 200,
-      "avg_response_time": 1.5
-    }
-  ]
-}
-```
-
-### **GET /analytics/models/history**
-Returns daily model usage history with trend data.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-
-**Response:**
-```json
-{
-  "model_history": [
-    {
-      "date": "2023-05-01",
-      "models": [
-        {
-          "name": "claude-3-sonnet-20241022",
-          "tokens": 5000,
-          "requests": 100
-        }
-      ]
-    }
-  ]
-}
-```
-
-### **GET /analytics/models/metrics**
-Returns model performance metrics including success rates and response times.
-
-**Response:**
-```json
-{
-  "model_metrics": [
-    {
-      "name": "claude-3-sonnet-20241022",
-      "avg_tokens": 250.5,
-      "avg_response_time": 1.2,
-      "success_rate": 0.95
-    }
-  ]
-}
-```
-
----
-
-## Cost Analytics Endpoints
-
-### **GET /analytics/costs/summary**
-Returns cost summary with averages and totals.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-
-**Response:**
-```json
-{
-  "totalCost": 25.50,
-  "totalTokens": 100000,
-  "totalRequests": 1000,
-  "avgDailyCost": 0.85,
-  "costPerThousandTokens": 0.255,
-  "daysInPeriod": 30,
-  "startDate": "2023-04-01",
-  "endDate": "2023-05-01"
-}
-```
-
-### **GET /analytics/costs/daily**
-Returns daily cost breakdown with filled gaps for missing dates.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-
-**Response:**
-```json
-{
-  "daily_costs": [
-    {
-      "date": "2023-05-01",
-      "cost": 1.25,
-      "tokens": 5000
-    }
-  ]
-}
-```
-
-### **GET /analytics/costs/models**
-Returns cost breakdown by model.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-
-**Response:**
-```json
-{
-  "model_costs": [
-    {
-      "model_name": "claude-3-sonnet-20241022",
-      "cost": 15.50,
-      "tokens": 50000,
-      "requests": 500
-    }
-  ]
-}
-```
-
-### **GET /analytics/costs/projections**
-Returns cost projections based on last 30 days usage.
-
-**Response:**
-```json
-{
-  "nextMonth": 75.00,
-  "next3Months": 225.00,
-  "nextYear": 900.00,
-  "tokensNextMonth": 300000,
-  "dailyCost": 2.50,
-  "dailyTokens": 10000,
-  "baselineDays": 30
-}
-```
-
-### **GET /analytics/costs/today**
-Returns today's cost data.
-
-**Response:**
-```json
-{
-  "date": "2023-05-01",
-  "cost": 2.50,
-  "tokens": 10000,
-  "requests": 100
-}
-```
-
----
-
-## Tier Analytics Endpoints
-
-### **GET /analytics/tiers/usage**
-Returns usage data categorized by model tiers with aggregated statistics.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-
-**Response:**
-```json
-{
-  "tier_data": {
-    "tier_1": {
-      "name": "Basic",
-      "credits": 1,
-      "total_tokens": 50000,
-      "total_requests": 500,
-      "total_cost": 5.00,
-      "avg_tokens_per_query": 100,
-      "cost_per_1k_tokens": 0.10,
-      "total_credit_cost": 500,
-      "cost_per_credit": 0.01,
-      "models": [...]
-    }
-  },
-  "period": "30d",
-  "start_date": "2023-04-01",
-  "end_date": "2023-05-01"
-}
-```
-
-### **GET /analytics/tiers/projections**
-Returns tier-based cost and usage projections.
-
-**Response:**
-```json
-{
-  "daily_usage": {...},
-  "projections": {
-    "monthly": {...},
-    "quarterly": {...},
-    "yearly": {...}
-  },
-  "tier_definitions": {...}
-}
-```
-
-### **GET /analytics/tiers/efficiency**
-Returns efficiency metrics by tier including cost per credit and tokens per credit.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-
-**Response:**
-```json
-{
-  "efficiency_data": {...},
-  "most_efficient_tier": "tier_2",
-  "best_value_tier": "tier_1",
-  "period": "30d",
-  "start_date": "2023-04-01",
-  "end_date": "2023-05-01"
-}
-```
-
----
-
-## Code Execution Analytics Endpoints
-
-### **GET /analytics/code-executions/summary**
-Returns code execution statistics including success rates and model performance.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-
-**Response:**
-```json
-{
-  "period": "30d",
-  "start_date": "2023-04-01",
-  "end_date": "2023-05-01",
-  "overall_stats": {
-    "total_executions": 1000,
-    "successful_executions": 950,
-    "failed_executions": 50,
-    "success_rate": 0.95,
-    "total_users": 100,
-    "total_chats": 200
-  },
-  "model_performance": [...],
-  "failed_agents": [...]
-}
-```
-
-### **GET /analytics/code-executions/detailed**
-Returns detailed code execution records with filtering options.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-- `success_filter` (optional): Filter by success status (boolean)
-- `user_id` (optional): Filter by user ID
-- `model_name` (optional): Filter by model name
-- `limit` (optional): Maximum results (default: `100`)
-
-**Response:**
-```json
-{
-  "period": "30d",
-  "start_date": "2023-04-01", 
-  "end_date": "2023-05-01",
-  "count": 50,
-  "executions": [...]
-}
-```
-
-### **GET /analytics/code-executions/users**
-Returns code execution statistics grouped by user.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-- `limit` (optional): Maximum users (default: `50`)
-
-**Response:**
-```json
-{
-  "period": "30d",
-  "start_date": "2023-04-01",
-  "end_date": "2023-05-01", 
-  "users": [...]
-}
-```
-
-### **GET /analytics/code-executions/error-analysis**
-Returns error analysis with categorized error types and agent failure patterns.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-
-**Response:**
-```json
-{
-  "period": "30d",
-  "start_date": "2023-04-01",
-  "end_date": "2023-05-01",
-  "total_failed_executions": 50,
-  "error_types": [...],
-  "error_by_agent": [...]
-}
-```
-
----
-
-## Feedback Analytics Endpoints
-
-### **GET /analytics/feedback/summary**
-Returns feedback summary statistics including rating distributions and trends.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-
-**Response:**
-```json
-{
-  "period": "30d",
-  "start_date": "2023-04-01",
-  "end_date": "2023-05-01",
-  "total_feedback": 500,
-  "avg_rating": 4.2,
-  "chats_with_feedback": 200,
-  "ratings_distribution": [
-    {"rating": 1, "count": 10},
-    {"rating": 2, "count": 20},
-    {"rating": 3, "count": 50},
-    {"rating": 4, "count": 200},
-    {"rating": 5, "count": 220}
-  ],
-  "models_data": [...],
-  "feedback_trend": [...]
-}
-```
-
-### **GET /analytics/feedback/detailed**
-Returns detailed feedback records with filtering and pagination.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-- `min_rating` (optional): Minimum rating filter
-- `max_rating` (optional): Maximum rating filter  
-- `model_name` (optional): Filter by model name
-- `limit` (optional): Maximum results (default: `100`)
-- `offset` (optional): Pagination offset (default: `0`)
-
-**Response:**
-```json
-{
-  "period": "30d",
-  "start_date": "2023-04-01",
-  "end_date": "2023-05-01", 
-  "total": 500,
-  "count": 100,
-  "offset": 0,
-  "limit": 100,
-  "feedback": [...]
-}
-```
-
----
-
-## Public Endpoints
-
-### **GET /analytics/public/ticker**
-Returns public ticker data for landing page statistics. **No authentication required.**
-
-**Response:**
-```json
-{
-  "total_signups": 1000,
-  "total_tokens": 5000000,
-  "total_requests": 50000,
-  "last_updated": "2023-05-01T12:00:00Z"
-}
-```
-
----
-
-## Utility Endpoints
-
-### **GET /analytics/usage/summary**
-Returns overall usage summary (legacy endpoint, calls dashboard with 30d period).
-
-### **GET /analytics/debug/model_usage**
-Debug endpoint for testing admin API key validation.
-
-**Response:**
-```json
-{
-  "status": "success",
-  "message": "Admin API key validated successfully"
-}
-```
-
----
-
-## Error Categorization
-
-The system automatically categorizes code execution errors into the following types:
-
-- **NameError**: Variable or function name not found
-- **SyntaxError**: Invalid Python syntax
-- **TypeError**: Type-related errors
-- **AttributeError**: Attribute access errors
-- **IndexError/KeyError**: Index or key access errors
-- **ImportError**: Module import errors
-- **ValueError**: Invalid values passed to functions
-- **OperationError**: Unsupported operations
-- **IndentationError**: Python indentation errors
-- **PermissionError**: File/system permission errors
-- **FileNotFoundError**: File access errors
-- **MemoryError**: Memory allocation errors
-- **TimeoutError**: Operation timeout errors
-- **OtherError**: Uncategorized errors
-
-## Real-time Updates
-
-The analytics system supports real-time updates through WebSocket connections:
-
-- **Dashboard updates**: Broadcasted when new model usage is recorded
-- **User activity updates**: Broadcasted for user activity changes
-- **Model performance updates**: Broadcasted for model-specific metrics
-
-All real-time updates are sent as JSON messages with `type` field indicating the update category and `metrics` containing the delta or new values.

docs/api/routes/deep_analysis.md

@@ -1,348 +0,0 @@
-# Deep Analysis API Documentation
-
-## Overview
-
-The Deep Analysis system provides advanced multi-agent analytical capabilities that automatically generate comprehensive reports based on user goals. The system uses DSPy (Declarative Self-improving Language Programs) to orchestrate multiple AI agents and create detailed analytical insights.
-
-## Key Features
-
-- **Multi-Agent Analysis**: Orchestrates multiple specialized agents (preprocessing, statistical analysis, machine learning, visualization)
-- **Template Integration**: Uses the user's active templates/agents for analysis
-- **Streaming Progress**: Real-time progress updates during analysis execution
-- **Report Persistence**: Stores complete analysis reports in database with metadata
-- **HTML Export**: Generates downloadable HTML reports with visualizations
-- **Credit Tracking**: Monitors token usage, costs, and credits consumed
-
-## Template Integration
-
-The deep analysis system integrates with the user's active templates through the agent system:
-
-1. **Agent Selection**: Uses agents from the user's active template preferences (configured via `/templates` endpoints)
-2. **Default Agents**: Falls back to system default agents if user hasn't configured preferences:
-   - `preprocessing` (both individual and planner variants)
-   - `statistical_analytics` (both individual and planner variants) 
-   - `sk_learn` (both individual and planner variants)
-   - `data_viz` (both individual and planner variants)
-3. **Template Limits**: Respects the 10-template limit for planner performance optimization
-4. **Dynamic Planning**: The planner automatically selects the most appropriate agents based on the analysis goal and available templates
-
-## Analysis Flow
-
-The deep analysis process follows these steps:
-
-1. **Question Generation** (20% progress): Generates 5 targeted analytical questions based on the user's goal
-2. **Planning** (40% progress): Creates an optimized execution plan using available agents
-3. **Agent Execution** (60% progress): Executes analysis using user's active templates
-4. **Code Synthesis** (80% progress): Combines and optimizes code from all agents
-5. **Code Execution** (85% progress): Runs the synthesized analysis code
-6. **Synthesis** (90% progress): Synthesizes results into coherent insights
-7. **Conclusion** (100% progress): Generates final conclusions and recommendations
-
----
-
-## API Endpoints
-
-### Create Deep Analysis Report
-
-**POST** `/deep_analysis/reports`
-
-Creates a new deep analysis report in the database.
-
-**Request Body:**
-```json
-{
-  "report_uuid": "string",
-  "user_id": 123,
-  "goal": "Analyze customer churn patterns",
-  "status": "completed",
-  "deep_questions": "1. What factors...\n2. How does...",
-  "deep_plan": "{\n  \"@preprocessing\": {\n    \"create\": [...],\n    \"use\": [...],\n    \"instruction\": \"...\"\n  }\n}",
-  "summaries": ["Agent summary 1", "Agent summary 2"],
-  "analysis_code": "import pandas as pd\n# Analysis code...",
-  "plotly_figures": [{"data": [...], "layout": {...}}],
-  "synthesis": ["Synthesis result 1"],
-  "final_conclusion": "## Conclusion\nThe analysis reveals...",
-  "html_report": "<html>...</html>",
-  "report_summary": "Brief summary of findings",
-  "progress_percentage": 100,
-  "duration_seconds": 120,
-  "credits_consumed": 5,
-  "error_message": null,
-  "model_provider": "anthropic",
-  "model_name": "claude-sonnet-4-20250514",
-  "total_tokens_used": 15000,
-  "estimated_cost": 0.25,
-  "steps_completed": ["questions", "planning", "execution", "synthesis", "conclusion"]
-}
-```
-
-**Response:**
-```json
-{
-  "report_id": 1,
-  "report_uuid": "uuid-string",
-  "user_id": 123,
-  "goal": "Analyze customer churn patterns",
-  "status": "completed",
-  "start_time": "2024-01-01T12:00:00Z",
-  "end_time": "2024-01-01T12:02:00Z",
-  "duration_seconds": 120,
-  "report_summary": "Brief summary of findings",
-  "created_at": "2024-01-01T12:02:00Z",
-  "updated_at": "2024-01-01T12:02:00Z"
-}
-```
-
-### Get Deep Analysis Reports
-
-**GET** `/deep_analysis/reports`
-
-Retrieves a list of deep analysis reports with optional filtering.
-
-**Query Parameters:**
-- `user_id` (optional): Filter by user ID
-- `limit` (optional): Number of reports to return (1-100, default: 10)
-- `offset` (optional): Number of reports to skip (default: 0)
-- `status` (optional): Filter by status ("pending", "running", "completed", "failed")
-
-**Response:**
-```json
-[
-  {
-    "report_id": 1,
-    "report_uuid": "uuid-string",
-    "user_id": 123,
-    "goal": "Analyze customer churn patterns",
-    "status": "completed",
-    "start_time": "2024-01-01T12:00:00Z",
-    "end_time": "2024-01-01T12:02:00Z",
-    "duration_seconds": 120,
-    "report_summary": "Brief summary of findings",
-    "created_at": "2024-01-01T12:02:00Z",
-    "updated_at": "2024-01-01T12:02:00Z"
-  }
-]
-```
-
-### Get User Historical Reports
-
-**GET** `/deep_analysis/reports/user_historical`
-
-Retrieves all historical deep analysis reports for a specific user.
-
-**Query Parameters:**
-- `user_id`: User ID (required)
-- `limit` (optional): Number of reports to return (1-100, default: 50)
-
-### Get Report by ID
-
-**GET** `/deep_analysis/reports/{report_id}`
-
-Retrieves a complete deep analysis report by ID.
-
-**Query Parameters:**
-- `user_id` (optional): Ensures report belongs to specified user
-
-**Response:**
-```json
-{
-  "report_id": 1,
-  "report_uuid": "uuid-string",
-  "user_id": 123,
-  "goal": "Analyze customer churn patterns",
-  "status": "completed",
-  "start_time": "2024-01-01T12:00:00Z",
-  "end_time": "2024-01-01T12:02:00Z",
-  "duration_seconds": 120,
-  "deep_questions": "1. What factors contribute to churn?\n2. How does churn vary by segment?",
-  "deep_plan": "{\n  \"@preprocessing\": {...},\n  \"@statistical_analytics\": {...}\n}",
-  "summaries": ["Agent performed data cleaning...", "Statistical analysis revealed..."],
-  "analysis_code": "import pandas as pd\n# Complete analysis code",
-  "plotly_figures": [{"data": [...], "layout": {...}}],
-  "synthesis": ["The analysis shows clear patterns..."],
-  "final_conclusion": "## Conclusion\nCustomer churn is primarily driven by...",
-  "html_report": "<html>...</html>",
-  "report_summary": "Analysis of customer churn patterns reveals...",
-  "progress_percentage": 100,
-  "credits_consumed": 5,
-  "error_message": null,
-  "model_provider": "anthropic",
-  "model_name": "claude-sonnet-4-20250514",
-  "total_tokens_used": 15000,
-  "estimated_cost": 0.25,
-  "steps_completed": ["questions", "planning", "execution", "synthesis", "conclusion"],
-  "created_at": "2024-01-01T12:02:00Z",
-  "updated_at": "2024-01-01T12:02:00Z"
-}
-```
-
-### Get Report by UUID
-
-**GET** `/deep_analysis/reports/uuid/{report_uuid}`
-
-Retrieves a complete deep analysis report by UUID. Same response format as get by ID.
-
-### Delete Report
-
-**DELETE** `/deep_analysis/reports/{report_id}`
-
-Deletes a deep analysis report.
-
-**Query Parameters:**
-- `user_id` (optional): Ensures report belongs to specified user
-
-**Response:**
-```json
-{
-  "message": "Report 1 deleted successfully"
-}
-```
-
-### Update Report Status
-
-**PUT** `/deep_analysis/reports/{report_id}/status`
-
-Updates the status of a deep analysis report.
-
-**Request Body:**
-```json
-{
-  "status": "completed"
-}
-```
-
-**Valid Status Values:**
-- `pending`: Analysis queued but not started
-- `running`: Analysis in progress
-- `completed`: Analysis finished successfully
-- `failed`: Analysis encountered errors
-
-### Get HTML Report
-
-**GET** `/deep_analysis/reports/uuid/{report_uuid}/html`
-
-Retrieves only the HTML report content for a specific analysis.
-
-**Query Parameters:**
-- `user_id` (optional): Ensures report belongs to specified user
-
-**Response:**
-```json
-{
-  "html_report": "<html>...</html>",
-  "filename": "deep_analysis_report_20240101_120200.html"
-}
-```
-
-### Download HTML Report
-
-**POST** `/deep_analysis/download_from_db/{report_uuid}`
-
-Downloads the HTML report as a file attachment.
-
-**Query Parameters:**
-- `user_id` (optional): Ensures report belongs to specified user
-
-**Response:**
-- Content-Type: `text/html; charset=utf-8`
-- Content-Disposition: `attachment; filename="deep_analysis_report_TIMESTAMP.html"`
-
----
-
-## Deep Analysis Module Architecture
-
-### DSPy Signatures
-
-The system uses several DSPy signatures for different analysis phases:
-
-#### 1. `deep_questions`
-Generates 5 targeted analytical questions based on the user's goal and dataset structure.
-
-#### 2. `deep_planner` 
-Creates an optimized execution plan using the user's active templates/agents. The planner:
-- Verifies feasibility using available datasets and agent descriptions
-- Batches similar questions per agent call for efficiency
-- Reuses outputs across questions to minimize agent calls
-- Defines clear variable flow and dependencies between agents
-
-#### 3. `deep_code_synthesizer`
-Combines and optimizes code from multiple agents:
-- Fixes errors and inconsistencies between agent outputs
-- Ensures proper data flow and type handling
-- Converts all visualizations to Plotly format
-- Adds comprehensive error handling and validation
-
-#### 4. `deep_synthesizer`
-Synthesizes analysis results into coherent insights and findings.
-
-#### 5. `final_conclusion`
-Generates final conclusions and strategic recommendations based on all analysis results.
-
-### Streaming Analysis
-
-The `execute_deep_analysis_streaming` method provides real-time progress updates:
-
-```python
-async for update in deep_analysis.execute_deep_analysis_streaming(goal, dataset_info, session_df):
-    if update["step"] == "questions":
-        # Handle questions generation progress
-    elif update["step"] == "planning":
-        # Handle planning progress
-    elif update["step"] == "agent_execution":
-        # Handle agent execution progress
-    # ... handle other steps
-```
-
-### Integration with User Templates
-
-The deep analysis system integrates with user templates in several ways:
-
-1. **Agent Discovery**: Retrieves user's active template preferences from the database
-2. **Dynamic Planning**: The planner uses available agents to create optimal execution plans
-3. **Template Validation**: Ensures all referenced agents exist in the user's active templates
-4. **Fallback Handling**: Uses default agents if user preferences are incomplete
-5. **Performance Optimization**: Respects template limits for efficient execution
-
-### Error Handling
-
-The system includes comprehensive error handling:
-
-- **Code Execution Errors**: Automatically attempts to fix and retry failed code
-- **Template Missing**: Falls back to default agents if user templates are unavailable
-- **Timeout Protection**: Includes timeouts for long-running operations
-- **Memory Management**: Handles large datasets and visualization efficiently
-- **Unicode Handling**: Cleans problematic characters that might cause encoding issues
-
-### Visualization Integration
-
-All visualizations are standardized to Plotly format:
-- Consistent styling and color schemes
-- Interactive features (zoom, pan, hover)
-- Accessibility compliance (colorblind-friendly palettes)
-- Export capabilities for reports
-- Responsive design for different screen sizes
-
----
-
-## Frontend Integration
-
-The deep analysis system includes React components for:
-
-- **DeepAnalysisSidebar**: Main interface for starting and managing analyses
-- **NewAnalysisForm**: Form for initiating new deep analyses
-- **CurrentAnalysisView**: Real-time progress tracking during analysis
-- **HistoryView**: Browse and access historical analysis reports
-- **AnalysisStep**: Individual step progress visualization
-
-The frontend integrates with the streaming API to provide real-time feedback and uses the user's active template configuration for personalized analysis capabilities.
-
-## Credit and Cost Tracking
-
-The system tracks detailed usage metrics:
-- **Credits Consumed**: Number of credits deducted from user account
-- **Token Usage**: Total tokens used across all model calls
-- **Estimated Cost**: Dollar cost estimate based on model pricing
-- **Model Information**: Provider and model name used for analysis
-- **Execution Time**: Duration of analysis for performance monitoring
-
-This information helps users understand resource consumption and optimize their analysis strategies. 

docs/api/routes/feedback.md

@@ -1,153 +0,0 @@
-# Feedback Routes Documentation
-
-This document describes the API endpoints available for managing user feedback on AI-generated messages in the Auto-Analyst backend.
-
-## Base URL
-
-All feedback-related endpoints are prefixed with `/feedback`.
-
-## Endpoints
-
-### Create or Update Message Feedback
-Creates new feedback or updates existing feedback for a specific message.
-
-**Endpoint:** `POST /feedback/message/{message_id}`
-
-**Path Parameters:**
-- `message_id`: ID of the message to provide feedback for
-
-**Request Body:**
-```json
-{
-    "rating": 5,                           // Required: Star rating (1-5)
-    "model_name": "gpt-4o-mini",          // Optional: Model used for the message
-    "model_provider": "openai",           // Optional: Provider of the model
-    "temperature": 0.7,                   // Optional: Temperature setting
-    "max_tokens": 6000                    // Optional: Max tokens setting
-}
-```
-
-**Response:**
-```json
-{
-    "feedback_id": 123,
-    "message_id": 456,
-    "rating": 5,
-    "feedback_comment": null,
-    "model_name": "gpt-4o-mini",
-    "model_provider": "openai",
-    "temperature": 0.7,
-    "max_tokens": 6000,
-    "created_at": "2023-05-01T12:00:00Z",
-    "updated_at": "2023-05-01T12:00:00Z"
-}
-```
-
-**Error Responses:**
-- `404 Not Found`: Message with specified ID not found
-- `500 Internal Server Error`: Failed to create/update feedback
-
-### Get Message Feedback
-Retrieves feedback for a specific message.
-
-**Endpoint:** `GET /feedback/message/{message_id}`
-
-**Path Parameters:**
-- `message_id`: ID of the message to get feedback for
-
-**Response:**
-```json
-{
-    "feedback_id": 123,
-    "message_id": 456,
-    "rating": 5,
-    "feedback_comment": null,
-    "model_name": "gpt-4o-mini",
-    "model_provider": "openai",
-    "temperature": 0.7,
-    "max_tokens": 6000,
-    "created_at": "2023-05-01T12:00:00Z",
-    "updated_at": "2023-05-01T12:00:00Z"
-}
-```
-
-**Error Responses:**
-- `404 Not Found`: No feedback found for the specified message
-- `500 Internal Server Error`: Failed to retrieve feedback
-
-### Get Chat Feedback
-Retrieves all feedback for messages in a specific chat.
-
-**Endpoint:** `GET /feedback/chat/{chat_id}`
-
-**Path Parameters:**
-- `chat_id`: ID of the chat to get feedback for
-
-**Response:**
-```json
-[
-    {
-        "feedback_id": 123,
-        "message_id": 456,
-        "rating": 5,
-        "feedback_comment": null,
-        "model_name": "gpt-4o-mini",
-        "model_provider": "openai",
-        "temperature": 0.7,
-        "max_tokens": 6000,
-        "created_at": "2023-05-01T12:00:00Z",
-        "updated_at": "2023-05-01T12:00:00Z"
-    }
-]
-```
-
-**Note:** Returns an empty array if no feedback exists for the chat.
-
-**Error Responses:**
-- `500 Internal Server Error`: Failed to retrieve chat feedback
-
-## Feedback Features
-
-### Rating System
-- **Scale:** 1-5 star rating system
-- **Required:** Rating is the only required field for feedback
-- **Purpose:** Allows users to rate the quality of AI responses
-
-### Model Context Tracking
-The system optionally tracks:
-- **Model Name:** The specific AI model used (e.g., "gpt-4o-mini")
-- **Model Provider:** The provider of the model (e.g., "openai", "anthropic")
-- **Temperature:** The creativity/randomness setting used
-- **Max Tokens:** The maximum response length setting
-
-### Update Behavior
-- **Upsert Operation:** The POST endpoint either creates new feedback or updates existing feedback
-- **Partial Updates:** When updating, only provided fields are modified
-- **Timestamp Tracking:** Both creation and update timestamps are maintained
-
-## Data Management
-
-### Database Operations
-- **Atomic Operations:** Feedback creation/updates are handled in database transactions
-- **Referential Integrity:** Feedback is linked to specific messages via foreign keys
-- **Soft Handling:** Missing optional fields are handled gracefully
-
-### Error Handling
-- **Comprehensive Logging:** All operations are logged for debugging
-- **User-Friendly Messages:** Error responses provide clear information
-- **Transaction Safety:** Failed operations are rolled back to maintain data consistency
-
-## Usage Patterns
-
-### Typical Workflow
-1. User receives an AI-generated message
-2. User provides rating (1-5 stars) via the frontend
-3. Frontend calls `POST /feedback/message/{message_id}` with rating and model context
-4. System stores or updates the feedback
-5. Feedback can be retrieved later for analytics or user review
-
-### Analytics Integration
-Feedback data is used by the analytics system to:
-- Track model performance across different configurations
-- Identify patterns in user satisfaction
-- Generate insights for model optimization 

docs/api/routes/session.md

@@ -1,273 +0,0 @@
-# **Auto-Analyst API Documentation**
-
-The core application routes are designed to manage the data and AI analysis capabilities of the Auto-Analyst application.
-
-## **1. Core Application Routes**
-### **Data Management**
-
-#### **POST /upload_dataframe**  
-Uploads a CSV dataset for analysis.  
-**Request:**  
-- `file`: CSV file  
-- `name`: Dataset name  
-- `description`: Dataset description  
-**Headers:**
-- `X-Force-Refresh`: "true" (optional) - Forces session reset before upload
-**Response:**  
-```json
-{ "message": "Dataframe uploaded successfully", "session_id": "abc123" }
-```
-
-#### **POST /upload_excel**  
-Uploads an Excel file with a specific sheet for analysis.  
-**Request:**  
-- `file`: Excel file  
-- `name`: Dataset name  
-- `description`: Dataset description  
-- `sheet_name`: Name of the Excel sheet to use
-**Headers:**
-- `X-Force-Refresh`: "true" (optional) - Forces session reset before upload
-**Response:**  
-```json
-{ "message": "Excel file processed successfully", "session_id": "abc123", "sheet": "Sheet1" }
-```
-
-#### **POST /api/excel-sheets**  
-Gets the list of sheet names from an Excel file.  
-**Request:**  
-- `file`: Excel file  
-**Response:**  
-```json
-{ "sheets": ["Sheet1", "Sheet2", "Data"] }
-```
-
-#### **GET /api/default-dataset**  
-Gets the default dataset.  
-**Response:**  
-```json
-{
-  "headers": ["column1", "column2", ...],
-  "rows": [[val1, val2, ...], ...],
-  "name": "Housing Dataset",
-  "description": "A comprehensive dataset containing housing information..."
-}
-```
-
-#### **POST /reset-session**  
-Resets session to default dataset.  
-**Request Body:**  
-```json
-{
-  "name": "optional name",
-  "description": "optional description",
-  "preserveModelSettings": false
-}
-```
-**Response:**  
-```json
-{
-  "message": "Session reset to default dataset",
-  "session_id": "abc123",
-  "dataset": "Housing.csv"
-}
-```
-
-#### **GET /api/preview-csv** / **POST /api/preview-csv**  
-Preview the current dataset in the session.
-**Response:**  
-```json
-{
-  "headers": ["column1", "column2", ...],
-  "rows": [[val1, val2, ...], ...],
-  "name": "Dataset Name",
-  "description": "Dataset description..."
-}
-```
-
----
-
-### **2. AI Analysis**
-
-#### **POST /chat/{agent_name}**  
-Processes a query using a specific AI agent.  
-**Path Parameters:** `agent_name`  
-**Request Body:**  
-```json
-{ "query": "Analyze the relationship between price and size" }
-```
-**Query Parameters:** `user_id` (optional), `chat_id` (optional)  
-**Response:**  
-```json
-{
-  "agent_name": "data_viz_agent",
-  "query": "Analyze the relationship between price and size",
-  "response": "# Analysis\n\nThere appears to be a strong positive correlation...",
-  "session_id": "abc123"
-}
-```
-
-#### **POST /chat**  
-Processes a query using multiple AI agents with streaming responses.  
-**Request Body:**  
-```json
-{ "query": "Analyze the housing data" }
-```
-**Query Parameters:** `user_id` (optional), `chat_id` (optional)  
-**Response:** *Streaming JSON objects:*  
-```json
-{"agent": "data_viz_agent", "content": "# Visualization\n\n...", "status": "success"}
-{"agent": "statistical_analytics_agent", "content": "# Statistical Analysis\n\n...", "status": "success"}
-```
-
-#### **POST /chat_history_name**  
-Generates a name for a chat based on the query.  
-**Request Body:**  
-```json
-{ "query": "Analyze sales data for Q4" }
-```
-**Response:**  
-```json
-{ "name": "Chat about sales data analysis" }
-```
-
-#### **GET /agents**  
-Lists available AI agents.  
-**Response:**  
-```json
-{
-  "available_agents": ["data_viz_agent", "sk_learn_agent", "statistical_analytics_agent", "preprocessing_agent"],
-  "standard_agents": ["preprocessing_agent", "statistical_analytics_agent", "sk_learn_agent", "data_viz_agent"],
-  "template_agents": ["custom_template_1", "custom_template_2"],
-  "custom_agents": []
-}
-```
-
----
-
-### **3. Deep Analysis**
-
-#### **POST /deep_analysis_streaming**  
-Performs comprehensive deep analysis with real-time streaming updates.  
-**Request Body:**  
-```json
-{ "goal": "Perform comprehensive analysis of the sales data" }
-```
-**Query Parameters:** `user_id` (optional), `chat_id` (optional)  
-**Response:** *Streaming JSON objects with progress updates*
-
-#### **POST /deep_analysis/download_report**  
-Downloads an HTML report from deep analysis results.  
-**Request Body:**  
-```json
-{
-  "analysis_data": { ... },
-  "report_uuid": "optional-uuid"
-}
-```
-**Response:** HTML file download
-
----
-
-### **4. Model Settings**
-
-#### **GET /api/model-settings**  
-Fetches current model settings.  
-**Response:**  
-```json
-{
-  "provider": "openai",
-  "model": "gpt-4o-mini",
-  "hasCustomKey": true,
-  "temperature": 1.0,
-  "maxTokens": 6000
-}
-```
-
-#### **POST /settings/model**  
-Updates model settings.  
-**Request Body:**  
-```json
-{
-  "provider": "openai",
-  "model": "gpt-4",
-  "api_key": "sk-...",
-  "temperature": 0.7,
-  "max_tokens": 8000
-}
-```
-**Response:**  
-```json
-{ "message": "Model settings updated successfully" }
-```
-
----
-
-### **5. Session Management**
-
-#### **GET /api/session-info**  
-Gets information about the current session.  
-**Response:**  
-```json
-{
-  "session_id": "abc123",
-  "dataset_name": "Housing Dataset",
-  "dataset_description": "...",
-  "model_config": { ... }
-}
-```
-
-#### **POST /set-message-info**  
-Associates message tracking information with the session.  
-**Request Body:**  
-```json
-{
-  "chat_id": 123,
-  "message_id": 456,
-  "user_id": 789
-}
-```
-
-#### **POST /create-dataset-description**  
-Creates an AI-generated description for a dataset.  
-**Request Body:**  
-```json
-{
-  "df_preview": "column1,column2\nvalue1,value2\n...",
-  "name": "Dataset Name"
-}
-```
-
----
-
-### **6. System Endpoints**
-
-#### **GET /**  
-Returns API welcome information and feature list.
-
-#### **GET /health**  
-Health check endpoint.  
-**Response:**  
-```json
-{ "message": "API is healthy and running" }
-```
-
----
-
----
-
-### **7. Authentication & Session Management**
-- **Session ID Sources:**  
-  - Query parameter: `session_id`  
-  - Header: `X-Session-ID`  
-  - Auto-generated if not provided  
-- **Session State Includes:**  
-  - Current dataset  
-  - AI system instance  
-  - Model configuration  
-  - User and chat associations
-
-### **9. Error Handling**
-- Comprehensive error handling with appropriate HTTP status codes
-- Detailed error messages for debugging
-- Fallback encoding support for CSV files (UTF-8, unicode_escape, ISO-8859-1)
-- Session state preservation during errors

docs/api/routes/templates.md

@@ -1,363 +0,0 @@
-# Templates and Agent Loading Documentation
-
-This document describes how the Auto-Analyst template system works, including agent loading, user preferences, and template management.
-
-## Overview
-
-The Auto-Analyst system uses a flexible template-based approach for managing AI agents. Templates define specialized agents with specific capabilities, and users can customize which agents are available for their analysis workflows.
-
-## Template System Architecture
-
-### Template Types
-
-Templates come in different **variant types** that determine how they can be used:
-
-- **`individual`**: Templates available for single-agent queries (e.g., `@preprocessing_agent`)
-- **`planner`**: Templates available for multi-agent planning workflows
-- **`both`**: Templates available in both individual and planner contexts
-
-### Default Agents
-
-The system includes four core default agents that are **enabled by default** for all users:
-
-**For Individual Use:**
-- `preprocessing_agent`: Data cleaning and preprocessing
-- `statistical_analytics_agent`: Statistical analysis and insights  
-- `sk_learn_agent`: Machine learning with scikit-learn
-- `data_viz_agent`: Data visualization with Plotly
-
-**For Planner Use:**
-- `planner_preprocessing_agent`: Planning version of preprocessing agent
-- `planner_statistical_analytics_agent`: Planning version of statistical agent
-- `planner_sk_learn_agent`: Planning version of ML agent
-- `planner_data_viz_agent`: Planning version of visualization agent
-
-## Template Management Endpoints
-
-### Get All Templates
-
-**Endpoint:** `GET /templates/`
-
-**Query Parameters:**
-- `variant_type`: Filter by `"individual"`, `"planner"`, or `"all"` (default: `"all"`)
-
-**Response:**
-```json
-[
-  {
-    "template_id": 1,
-    "template_name": "preprocessing_agent",
-    "display_name": "Data Preprocessing Agent",
-    "description": "Handles data cleaning, missing values, and preprocessing tasks",
-    "prompt_template": "You are a data preprocessing specialist...",
-    "template_category": "Data Processing",
-    "icon_url": "/icons/templates/preprocessing_agent.svg",
-    "is_premium_only": false,
-    "is_active": true,
-    "usage_count": 12,
-    "created_at": "2023-05-01T12:00:00Z",
-    "updated_at": "2023-05-01T12:00:00Z"
-  }
-]
-```
-
-### Get Templates by Category
-
-**Endpoint:** `GET /templates/categories`
-
-**Query Parameters:**
-- `variant_type`: Filter by `"individual"`, `"planner"`, or `"all"` (default: `"individual"`)
-
-**Response:**
-```json
-[
-  {
-    "category": "Data Processing", 
-    "templates": [
-      {
-        "agent_id": 1,
-        "agent_name": "preprocessing_agent",
-        "display_name": "Data Preprocessing Agent",
-        "description": "Handles data cleaning and preprocessing",
-        "icon_url": "/icons/templates/preprocessing_agent.svg",
-        "usage_count": 1234
-      }
-    ]
-  }
-]
-```
-
-### Get Template by ID
-
-**Endpoint:** `GET /templates/template/{template_id}`
-
-**Response:**
-```json
-{
-  "template_id": 1,
-  "template_name": "preprocessing_agent",
-  "display_name": "Data Preprocessing Agent",
-  "description": "Handles data cleaning, missing values, and preprocessing tasks",
-  "prompt_template": "You are a data preprocessing specialist...",
-  "template_category": "Data Processing",
-  "icon_url": "/icons/templates/preprocessing_agent.svg",
-  "is_premium_only": false,
-  "is_active": true,
-  "usage_count": 1234,
-  "created_at": "2023-05-01T12:00:00Z",
-  "updated_at": "2023-05-01T12:00:00Z"
-}
-```
-
-### Get Template Categories List
-
-**Endpoint:** `GET /templates/categories/list`
-
-**Response:**
-```json
-{
-  "categories": [
-    "Data Processing",
-    "Machine Learning", 
-    "Visualization",
-    "Statistics"
-  ]
-}
-```
-
-### Get Templates by Specific Category
-
-**Endpoint:** `GET /templates/category/{category}`
-
-**Path Parameters:**
-- `category`: Name of the category to filter by
-
-**Response:**
-```json
-[
-  {
-    "template_id": 1,
-    "template_name": "preprocessing_agent",
-    "display_name": "Data Preprocessing Agent",
-    "description": "Handles data cleaning and preprocessing",
-    "template_category": "Data Processing",
-    "icon_url": "/icons/templates/preprocessing_agent.svg",
-    "usage_count": 1234
-  }
-]
-```
-
-## User Template Preferences
-
-### How Agent Loading Works for Users
-
-1. **Default Behavior**: New users automatically have the 4 core default agents enabled
-2. **Custom Preferences**: Users can enable/disable additional templates through preferences
-3. **Variant-Specific**: Individual and planner variants are managed separately
-4. **Usage Tracking**: System tracks which templates users actually use
-
-### Get User Template Preferences
-
-**Endpoint:** `GET /templates/user/{user_id}`
-
-**Query Parameters:**
-- `variant_type`: Filter by `"individual"`, `"planner"`, or `"all"` (default: `"planner"`)
-
-**Response:**
-```json
-[
-  {
-    "template_id": 1,
-    "template_name": "preprocessing_agent", 
-    "display_name": "Data Preprocessing Agent",
-    "description": "Handles data cleaning and preprocessing",
-    "template_category": "Data Processing",
-    "icon_url": "/icons/templates/preprocessing_agent.svg",
-    "is_premium_only": false,
-    "is_active": true,
-    "is_enabled": true,
-    "usage_count": 15,
-    "last_used_at": "2023-05-01T12:00:00Z",
-    "created_at": "2023-04-01T12:00:00Z",
-    "updated_at": "2023-05-01T12:00:00Z"
-  }
-]
-```
-
-### Get Only Enabled Templates
-
-**Endpoint:** `GET /templates/user/{user_id}/enabled`
-
-Returns only templates that are currently enabled for the user.
-
-### Get Enabled Templates for Planner
-
-**Endpoint:** `GET /templates/user/{user_id}/enabled/planner`
-
-Returns enabled planner templates with the following restrictions:
-- **Maximum 10 templates** for planner use
-- **Sorted by usage** (most used first)
-- **Only planner variants** (`planner` or `both` types)
-
-## Template Preference Management
-
-### Toggle Single Template
-
-**Endpoint:** `POST /templates/user/{user_id}/template/{template_id}/toggle`
-
-**Request Body:**
-```json
-{
-  "is_enabled": true
-}
-```
-
-**Restrictions:**
-- Cannot disable all templates (at least 1 must remain enabled)
-- Cannot enable more than 10 templates for planner use
-
-### Bulk Toggle Templates
-
-**Endpoint:** `POST /templates/user/{user_id}/bulk-toggle`
-
-**Request Body:**
-```json
-{
-  "preferences": [
-    {
-      "template_id": 1,
-      "is_enabled": true
-    },
-    {
-      "template_id": 2, 
-      "is_enabled": false
-    }
-  ]
-}
-```
-
-**Response:**
-```json
-{
-  "results": [
-    {
-      "template_id": 1,
-      "success": true,
-      "message": "Template enabled successfully",
-      "is_enabled": true
-    }
-  ]
-}
-```
-
-## Template Categories and Icons
-
-### Available Categories
-
-Templates are organized into categories such as:
-- **Data Processing**: Preprocessing, cleaning, feature engineering
-- **Machine Learning**: Various ML frameworks and algorithms  
-- **Visualization**: Plotting and chart generation
-- **Statistics**: Statistical analysis and modeling
-- **Custom**: User or organization-specific templates
-
-### Icon System
-
-Templates include visual icons stored in `/public/icons/templates/`:
-
-**Core Agent Icons:**
-- `preprocessing_agent.svg`: Data preprocessing 
-- `sk_learn_agent.svg`: Machine learning
-- `matplotlib_agent.png`: Plotting with matplotlib
-- `polars_agent.svg`: Data manipulation with Polars
-
-**Library-Specific Icons:**
-- `numpy.svg`, `scipy.png`: Scientific computing
-- `plotly.svg`, `seaborn.svg`: Advanced visualization
-- `lightgbm.png`, `xgboost.png`: Gradient boosting
-- `pymc.png`, `statsmodel.svg`: Statistical modeling
-
-**Special Purpose Icons:**
-- `data-cleaning.png`: Data cleaning workflows
-- `feature-engineering.png`: Feature engineering tasks
-
-## Agent Loading Process
-
-### For Individual Queries
-
-When a user makes a query like `@preprocessing_agent analyze my data`:
-
-1. **Check User Preferences**: System looks up user's enabled individual templates
-2. **Apply Defaults**: If no preference exists, default agents are enabled
-3. **Load Agent**: System loads the specific agent template and executes the query
-4. **Track Usage**: Usage count is incremented for analytics
-
-### For Planner Workflows  
-
-When a user makes a general query that triggers the planner:
-
-1. **Get Enabled Planner Templates**: System queries user's enabled planner variants
-2. **Apply 10-Template Limit**: Maximum 10 templates for performance
-3. **Sort by Usage**: Most-used templates get priority
-4. **Create Plan**: Planner selects appropriate agents for the analysis
-5. **Execute Workflow**: Selected agents execute in sequence
-6. **Update Usage**: Usage statistics updated for selected agents
-
-### Default Agent Behavior
-
-```python
-# Default agents enabled for new users
-individual_defaults = [
-    "preprocessing_agent",
-    "statistical_analytics_agent", 
-    "sk_learn_agent",
-    "data_viz_agent"
-]
-
-planner_defaults = [
-    "planner_preprocessing_agent",
-    "planner_statistical_analytics_agent",
-    "planner_sk_learn_agent", 
-    "planner_data_viz_agent"
-]
-```
-
-## Usage Analytics
-
-### Global Usage Tracking
-
-The system tracks global usage statistics across all users:
-- **Total usage count** per template
-- **User-specific usage** for personalization
-- **Last used timestamps** for sorting
-
-### Usage-Based Features
-
-- **Template Recommendations**: Popular templates shown first
-- **Personalized Ordering**: User's most-used templates prioritized
-- **Analytics Dashboard**: Usage patterns for administrators
-
-## Template Restrictions
-
-### User Limitations
-
-- **Minimum 1 Agent**: Cannot disable all templates
-- **Maximum 10 for Planner**: Performance optimization
-- **Premium Templates**: Some templates require premium access
-
-### System Limitations
-
-- **Active Templates Only**: Inactive templates not available
-- **Variant Compatibility**: Individual/planner variants managed separately
-- **Category Organization**: Templates must belong to valid categories
-
-## Integration with Deep Analysis
-
-The deep analysis system uses the template preference system:
-
-1. **Load User Preferences**: Gets enabled planner templates for user
-2. **Create Agent Pool**: Instantiates agents from enabled templates  
-3. **Execute Analysis**: Uses available agents for comprehensive analysis
-4. **Fallback Behavior**: Uses default agents if no preferences found
-
-This ensures users get personalized deep analysis based on their template preferences while maintaining system performance through the 10-template limit. 

docs/architecture/architecture.md

@@ -1,427 +0,0 @@
-# Auto-Analyst Backend System Architecture
-
-## Overview
-
-Auto-Analyst is a sophisticated multi-agent AI platform designed for comprehensive data analysis. The backend system orchestrates specialized AI agents, manages user sessions, and provides a robust API for data processing and analysis workflows.
-
-## 🏗️ High-Level Architecture
-
-```
-┌─────────────────┐    ┌──────────────────┐    ┌─────────────────┐
-│   Frontend      │    │     Backend      │    │    Database     │
-│   (Next.js)     │◄──►│    (FastAPI)     │◄──►│ (PostgreSQL/    │
-│                 │    │                  │    │  SQLite)        │
-└─────────────────┘    └──────────────────┘    └─────────────────┘
-                              │
-                              ▼
-                       ┌──────────────────┐
-                       │   AI Models      │
-                       │   (DSPy/LLMs)    │
-                       └──────────────────┘
-                              │
-                              ▼
-                       ┌──────────────────┐
-                       │ Agent System     │
-                       │ [Processing]     │
-                       │ [Analytics]      │
-                       │ [Visualization]  │
-                       └──────────────────┘
-```
-
-## 🎯 Core Components
-
-### 1. Application Layer (`app.py`)
-
-**FastAPI Application Server**
-- **Role**: Main HTTP server and request router
-- **Responsibilities**:
-  - Request/response handling
-  - Session-based authentication
-  - Route registration and middleware
-  - Error handling and logging
-  - Static file serving
-  - CORS configuration
-
-**Key Features**:
-- Async/await support for high concurrency
-- Automatic API documentation generation
-- Request validation with Pydantic
-- Session management for user tracking
-
-### 2. Agent System (`src/agents/`)
-
-**Multi-Agent Orchestra**
-- **Core Agents**: Specialized AI agents for different analysis tasks
-- **Deep Analysis**: Advanced multi-agent coordination system
-- **Template System**: User-customizable agent configurations
-
-#### Agent Types
-
-1. **Individual Agents** (`agents.py`):
-   ```python
-   - preprocessing_agent         # Data cleaning and preparation
-   - statistical_analytics_agent # Statistical analysis
-   - sk_learn_agent             # Machine learning with scikit-learn
-   - data_viz_agent             # Data visualization
-   - basic_qa_agent             # General Q&A
-   ```
-
-2. **Planner Agents** (Multi-agent coordination):
-   ```python
-   - planner_preprocessing_agent
-   - planner_statistical_analytics_agent
-   - planner_sk_learn_agent
-   - planner_data_viz_agent
-   ```
-
-3. **Deep Analysis System** (`deep_agents.py`):
-   ```python
-   - deep_questions         # Question generation
-   - deep_planner          # Execution planning
-   - deep_code_synthesizer # Code combination
-   - deep_synthesizer      # Result synthesis
-   - final_conclusion      # Report generation
-   ```
-
-#### Agent Architecture Pattern
-
-```python
-class AgentSignature(dspy.Signature):
-    """Agent description and purpose"""
-    goal = dspy.InputField(desc="Analysis objective")
-    dataset = dspy.InputField(desc="Dataset information")
-    plan_instructions = dspy.InputField(desc="Execution plan")
-    
-    summary = dspy.OutputField(desc="Analysis summary")
-    code = dspy.OutputField(desc="Generated code")
-```
-
-### 3. Database Layer (`src/db/`)
-
-**Data Persistence and Management**
-
-#### Database Models (`schemas/models.py`):
-
-```python
-# Core Models
-User              # User accounts and authentication
-Chat              # Conversation sessions
-Message           # Individual messages in chats
-ModelUsage        # AI model usage tracking
-
-# Template System
-AgentTemplate     # Agent definitions and configurations
-UserTemplatePreference  # User's enabled/disabled agents
-
-# Deep Analysis
-DeepAnalysisReport     # Analysis reports and results
-
-# Analytics
-CodeExecution     # Code execution tracking
-UserAnalytics     # User behavior analytics
-```
-
-#### Database Architecture:
-
-```
-Users (1) ──────── (Many) Chats
-  │                        │
-  │                        ▼
-  └─── (Many) ModelUsage ──┘
-  │
-  └─── (Many) UserTemplatePreference
-               │
-               ▼
-         AgentTemplate
-```
-
-### 4. Route Handlers (`src/routes/`)
-
-**RESTful API Endpoints**
-
-| Module | Purpose | Key Endpoints |
-|--------|---------|---------------|
-| `session_routes.py` | Core functionality | `/upload_excel`, `/session_info` |
-| `chat_routes.py` | Chat management | `/chats`, `/messages`, `/delete_chat` |
-| `code_routes.py` | Code operations | `/execute_code`, `/get_latest_code` |
-| `templates_routes.py` | Agent templates | `/templates`, `/user/{id}/enabled` |
-| `deep_analysis_routes.py` | Deep analysis | `/reports`, `/download_from_db` |
-| `analytics_routes.py` | System analytics | `/usage`, `/feedback`, `/costs` |
-| `feedback_routes.py` | User feedback | `/feedback`, `/message/{id}/feedback` |
-
-NOTE: Make sure to add a router prefix when calling these endpoints, such as to get dashboard, you'll use `http://localhost:8000/templates/dashboard`
-
-### 5. Business Logic Layer (`src/managers/`)
-
-**Service Layer for Complex Operations**
-
-#### Manager Components:
-
-1. **`chat_manager.py`**:
-   ```python
-   - Session management
-   - Message handling
-   - Context preservation
-   - Agent orchestration
-   ```
-
-2. **`ai_manager.py`**:
-   ```python
-   - Model selection and routing
-   - Token tracking and cost calculation
-   - Error handling and retries
-   - Response formatting
-   ```
-
-3. **`session_manager.py`**:
-   ```python
-   - Session lifecycle management
-   - Data sharing between agents
-   - Memory management
-   - Cleanup operations
-   ```
-
-### 6. Utility Layer (`src/utils/`)
-
-**Shared Services and Helpers**
-
-- **`logger.py`**: Centralized logging system
-- **`generate_report.py`**: HTML report generation
-- **`model_registry.py`**: AI model configuration
-
-## 🔄 Data Flow Architecture
-
-### 1. Request Processing Flow
-
-```
-HTTP Request → FastAPI Router → Route Handler → Manager/Business Logic → 
-Database/Agent System → AI Model → Response Processing → JSON Response
-```
-
-### 2. Agent Execution Flow
-
-```
-User Query → Session Creation → Template Selection → Agent Loading → 
-Code Generation → Code Execution → Result Processing → Response Formatting
-```
-
-### 3. Deep Analysis Flow
-
-```
-Analysis Goal → Question Generation → Planning Phase → Agent Coordination → 
-Code Synthesis → Execution → Result Synthesis → Final Report Generation
-```
-
-### 4. Template System Flow
-
-```
-User Preferences → Template Loading → Agent Registration → 
-Capability Mapping → Execution Routing → Usage Tracking
-```
-
-## 🎨 Design Patterns
-
-### 1. **Module Pattern**
-- Clear separation of concerns
-- Each module has specific responsibilities
-- Minimal dependencies between modules
-
-### 2. **Repository Pattern**
-- Database access abstracted through SQLAlchemy
-- Session management centralized
-- Clean separation of data and business logic
-
-### 3. **Strategy Pattern**
-- Multiple AI models supported through unified interface
-- Agent selection based on user preferences
-- Dynamic template loading
-
-### 4. **Observer Pattern**
-- Usage tracking and analytics
-- Event-driven model updates
-- Real-time progress notifications
-
-## 🔧 Configuration Management
-
-### Environment Configuration
-
-```python
-# Database
-DATABASE_URL: str           # Database connection string
-POSTGRES_PASSWORD: str      # PostgreSQL password (optional)
-
-# AI Models
-ANTHROPIC_API_KEY: str      # Claude API key
-OPENAI_API_KEY: str         # OpenAI API key
-
-# Authentication
-ADMIN_API_KEY: str          # Admin operations key (optional)
-
-# Deployment
-PORT: int = 8000            # Server port
-DEBUG: bool = False         # Debug mode
-```
-
-### Agent Configuration (`agents_config.json`)
-
-```json
-{
-  "default_agents": [
-    {
-      "template_name": "preprocessing_agent",
-      "description": "Data cleaning and preparation",
-      "variant_type": "both",
-      "is_premium": false,
-      "usage_count": 0,
-      "icon_url": "preprocessing.svg"
-    }
-  ],
-  "premium_templates": [...],
-  "remove": [...]
-}
-```
-
-## 🔒 Security Architecture
-
-### Authentication & Authorization
-
-1. **Session-based Authentication**:
-   - Session IDs for user identification
-   - Optional API key authentication for admin endpoints
-
-2. **Input Validation**:
-   - Pydantic models for request validation
-   - SQL injection prevention through SQLAlchemy
-   - File upload restrictions and validation
-
-3. **Resource Protection**:
-   - User-specific data isolation
-   - Usage tracking and monitoring
-   - Rate limiting considerations
-
-### Data Security
-
-1. **Database Security**:
-   - Encrypted connections for PostgreSQL
-   - Parameterized queries
-   - Regular backup procedures
-
-2. **Code Execution Security**:
-   - Sandboxed code execution environment
-   - Limited library imports
-   - Timeout protection
-
-## 📊 Performance Architecture
-
-### Scalability Features
-
-1. **Async Architecture**:
-   - Non-blocking I/O operations
-   - Concurrent agent execution
-   - Streaming responses for long operations
-
-2. **Database Optimization**:
-   - Connection pooling
-   - Query optimization
-   - Indexed frequently accessed columns
-
-3. **Caching Strategy**:
-   - In-memory caching for templates
-   - Result caching for expensive operations
-   - Session data management
-
-### Performance Monitoring
-
-1. **Usage Analytics**:
-   - Request/response time tracking
-   - Token usage monitoring
-   - Error rate analysis
-
-2. **Resource Monitoring**:
-   - Database query performance
-   - Memory usage tracking
-   - Agent execution time analysis
-
-## 🚀 Deployment Architecture
-
-### Development Environment
-
-```
-Local Development → SQLite Database → File-based Logging → 
-Direct Model API Calls → Hot Reloading
-```
-
-### Production Environment
-
-```
-Load Balancer → Multiple FastAPI Instances → PostgreSQL Database → 
-Centralized Logging → Monitoring & Alerting
-```
-
-### Container Architecture
-
-```dockerfile
-# Multi-stage build for optimization
-FROM python:3.11-slim as base
-# Dependencies and application setup
-# Health checks and graceful shutdown
-# Environment-specific configurations
-```
-
-## 🔄 Integration Patterns
-
-### External Service Integration
-
-1. **AI Model Providers**:
-   - Anthropic (Claude)
-   - OpenAI (GPT models)
-   - Unified interface through DSPy
-
-2. **Database Systems**:
-   - PostgreSQL (production)
-   - SQLite (development)
-   - Migration support through Alembic
-
-### Frontend Integration
-
-1. **REST API**:
-   - Standard HTTP endpoints
-   - JSON request/response format
-   - Session-based communication
-
-2. **Data Exchange**:
-   - File upload capabilities
-   - Real-time analysis results
-   - Report generation and download
-
-### Third-Party Integration
-
-1. **Python Data Science Stack (For Agentic Use only)**:
-   - Pandas for data manipulation
-   - NumPy for numerical computing
-   - Scikit-learn for machine learning
-   - Plotly for visualization
-   - Statsmodels for statistical analysis
-
-2. **Development Tools**:
-   - Alembic for database migrations
-   - SQLAlchemy for ORM
-   - FastAPI for web framework
-   - Pydantic for data validation
-
-## 📝 Documentation Architecture
-
-### API Documentation
-
-1. **Auto-generated Docs**: Available at `/docs` endpoint
-2. **Schema Definitions**: Pydantic models with descriptions
-3. **Endpoint Documentation**: Detailed parameter and response docs
-
-### Code Documentation
-
-1. **Inline Documentation**: Comprehensive docstrings
-2. **Architecture Guides**: High-level system design documentation
-3. **Getting Started**: Developer onboarding documentation
-4. **Troubleshooting**: Common issues and solutions
-
-This architecture provides a robust, scalable foundation for multi-agent AI analysis while maintaining clean separation of concerns and supporting both development and production deployment scenarios. 

docs/development/development_workflow.md

@@ -1,506 +0,0 @@
-# Auto-Analyst Backend Development Workflow
-
-## 🎯 Development Philosophy
-
-The Auto-Analyst backend follows modern Python development practices with emphasis on:
-- **Modularity**: Clear separation of concerns across components
-- **Async-First**: Non-blocking operations for scalability
-- **Type Safety**: Comprehensive type hints and validation
-- **Documentation**: Self-documenting code and comprehensive docs
-- **Testing**: Robust testing at multiple levels
-- **Performance**: Optimized for real-world usage patterns
-
-## 🏗️ Code Organization Principles
-
-### 1. **Directory Structure Standards**
-
-```
-src/
-├── agents/           # AI agent implementations
-│   ├── agents.py    # Core agent definitions
-│   ├── deep_agents.py # Deep analysis system
-│   └── retrievers/  # Information retrieval components
-├── db/              # Database layer
-│   ├── init_db.py   # Database initialization
-│   └── schemas/     # SQLAlchemy models
-├── managers/        # Business logic layer
-│   ├── chat_manager.py    # Chat operations
-│   ├── ai_manager.py      # AI model management
-│   └── session_manager.py # Session lifecycle
-├── routes/          # FastAPI route handlers
-│   ├── session_routes.py     # Core functionality
-│   ├── chat_routes.py     # Chat endpoints
-│   └── [feature]_routes.py # Feature-specific routes
-├── utils/           # Shared utilities
-│   ├── logger.py    # Centralized logging
-│   └── helpers.py   # Common functions
-└── schemas/         # Pydantic models
-    ├── chat_schemas.py    # Chat data models
-    └── [feature]_schemas.py # Feature schemas
-```
-
-### 2. **Import Organization**
-
-```python
-# Standard library imports
-import asyncio
-import json
-from datetime import datetime
-from typing import List, Optional, Dict, Any
-
-# Third-party imports
-import dspy
-import pandas as pd
-from fastapi import APIRouter, Depends, HTTPException
-from pydantic import BaseModel
-from sqlalchemy.orm import Session
-
-# Local imports
-from src.db.init_db import session_factory
-from src.db.schemas.models import User, Chat
-from src.utils.logger import Logger
-from src.managers.chat_manager import ChatManager
-```
-
-## 🛠️ Development Patterns
-
-### 1. **Agent Development Pattern**
-
-```python
-# 1. Define DSPy Signature
-class new_analysis_agent(dspy.Signature):
-    """
-    Comprehensive docstring explaining:
-    - Agent purpose and capabilities
-    - Input requirements and formats
-    - Expected output format
-    - Usage examples
-    """
-    goal = dspy.InputField(desc="Clear description of analysis objective")
-    dataset = dspy.InputField(desc="Dataset structure and content description")
-    plan_instructions = dspy.InputField(desc="Execution plan from planner")
-    
-    summary = dspy.OutputField(desc="Natural language summary of analysis")
-    code = dspy.OutputField(desc="Executable Python code for analysis")
-
-# 2. Add to Agent Configuration
-# In agents_config.json:
-{
-  "template_name": "new_analysis_agent",
-  "description": "Performs specialized analysis on datasets",
-  "variant_type": "both",  # individual, planner, or both
-  "is_premium": false, # Will be active by default
-  "usage_count": 0,
-  "icon_url": "analysis.svg"
-}
-
-# 3. Register in Agent System
-# In agents.py, add to the appropriate loading functions
-```
-
-### 2. **Route Development Pattern**
-
-```python
-# 1. Create route file: src/routes/feature_routes.py
-from fastapi import APIRouter, Depends, HTTPException, Query
-from pydantic import BaseModel
-from typing import List, Optional
-from src.db.init_db import session_factory
-from src.db.schemas.models import FeatureModel
-from src.utils.logger import Logger
-
-logger = Logger("feature_routes", see_time=True, console_log=False)
-router = APIRouter(prefix="/feature", tags=["feature"])
-
-# 2. Define Pydantic schemas
-class FeatureCreate(BaseModel):
-    name: str
-    description: Optional[str] = None
-    
-class FeatureResponse(BaseModel):
-    id: int
-    name: str
-    description: Optional[str]
-    created_at: datetime
-
-# 3. Implement endpoints with proper error handling
-@router.post("/", response_model=FeatureResponse)
-async def create_feature(feature: FeatureCreate):
-    try:
-        session = session_factory()
-        try:
-            new_feature = FeatureModel(
-                name=feature.name,
-                description=feature.description
-            )
-            session.add(new_feature)
-            session.commit()
-            session.refresh(new_feature)
-            
-            return FeatureResponse(
-                id=new_feature.id,
-                name=new_feature.name,
-                description=new_feature.description,
-                created_at=new_feature.created_at
-            )
-            
-        except Exception as e:
-            session.rollback()
-            logger.log_message(f"Error creating feature: {str(e)}", level=logging.ERROR)
-            raise HTTPException(status_code=500, detail=f"Failed to create feature: {str(e)}")
-        finally:
-            session.close()
-            
-    except Exception as e:
-        logger.log_message(f"Error in create_feature: {str(e)}", level=logging.ERROR)
-        raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
-
-# 4. Register in app.py
-from src.routes.feature_routes import router as feature_router
-app.include_router(feature_router)
-```
-
-### 3. **Database Model Pattern**
-
-```python
-# In src/db/schemas/models.py
-from sqlalchemy import Column, Integer, String, DateTime, Boolean, Text, ForeignKey
-from sqlalchemy.orm import relationship
-from sqlalchemy.ext.declarative import declarative_base
-from datetime import datetime, timezone
-
-Base = declarative_base()
-
-class NewModel(Base):
-    __tablename__ = "new_models"
-    
-    # Primary key
-    id = Column(Integer, primary_key=True, autoincrement=True)
-    
-    # Required fields
-    name = Column(String(255), nullable=False, unique=True)
-    
-    # Optional fields
-    description = Column(Text, nullable=True)
-    is_active = Column(Boolean, default=True, nullable=False)
-    
-    # Timestamps
-    created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), nullable=False)
-    updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc), nullable=False)
-    
-    # Foreign keys
-    user_id = Column(Integer, ForeignKey("users.user_id"), nullable=True)
-    
-    # Relationships
-    user = relationship("User", back_populates="new_models")
-    
-    def __repr__(self):
-        return f"<NewModel(id={self.id}, name='{self.name}')>"
-
-# Update User model to include back reference
-class User(Base):
-    # ... existing fields ...
-    new_models = relationship("NewModel", back_populates="user")
-```
-
-### 4. **Manager Pattern**
-
-```python
-# In src/managers/feature_manager.py
-from typing import List, Optional, Dict, Any
-from sqlalchemy.orm import Session
-from src.db.schemas.models import FeatureModel
-from src.utils.logger import Logger
-
-logger = Logger("feature_manager", see_time=True, console_log=False)
-
-class FeatureManager:
-    """
-    Manages business logic for feature operations.
-    Separates complex business logic from route handlers.
-    """
-    
-    def __init__(self, session: Session):
-        self.session = session
-    
-    async def create_feature(self, name: str, description: Optional[str] = None) -> FeatureModel:
-        """Create a new feature with validation and business logic."""
-        try:
-            # Validation
-            if not name or len(name.strip()) == 0:
-                raise ValueError("Feature name cannot be empty")
-            
-            # Check for duplicates
-            existing = self.session.query(FeatureModel).filter_by(name=name).first()
-            if existing:
-                raise ValueError(f"Feature with name '{name}' already exists")
-            
-            # Create feature
-            feature = FeatureModel(name=name, description=description)
-            self.session.add(feature)
-            self.session.commit()
-            self.session.refresh(feature)
-            
-            logger.log_message(f"Created feature: {name}", level=logging.INFO)
-            return feature
-            
-        except Exception as e:
-            self.session.rollback()
-            logger.log_message(f"Error creating feature: {str(e)}", level=logging.ERROR)
-            raise
-    
-    async def get_features(self, active_only: bool = True) -> List[FeatureModel]:
-        """Retrieve features with optional filtering."""
-        try:
-            query = self.session.query(FeatureModel)
-            if active_only:
-                query = query.filter(FeatureModel.is_active == True)
-            
-            features = query.order_by(FeatureModel.created_at.desc()).all()
-            return features
-            
-        except Exception as e:
-            logger.log_message(f"Error retrieving features: {str(e)}", level=logging.ERROR)
-            raise
-```
-
-## 📋 Code Quality Standards
-
-### 1. **Type Hints and Documentation**
-
-```python
-from typing import List, Optional, Dict, Any, Union
-from datetime import datetime
-
-async def process_analysis_data(
-    data: pd.DataFrame,
-    analysis_type: str,
-    user_id: Optional[int] = None,
-    options: Dict[str, Any] = None
-) -> Dict[str, Union[str, List[Any], bool]]:
-    """
-    Process analysis data with specified parameters.
-    
-    Args:
-        data: Input DataFrame containing the data to analyze
-        analysis_type: Type of analysis to perform ("statistical", "ml", "viz")
-        user_id: Optional user ID for tracking and personalization
-        options: Additional options for analysis configuration
-        
-    Returns:
-        Dictionary containing:
-        - status: "success" or "error"
-        - result: Analysis results or error message
-        - metadata: Additional information about the analysis
-        
-    Raises:
-        ValueError: If analysis_type is not supported
-        DataError: If data format is invalid
-        
-    Example:
-        >>> data = pd.DataFrame({"x": [1, 2, 3], "y": [4, 5, 6]})
-        >>> result = await process_analysis_data(data, "statistical")
-        >>> print(result["status"])
-        "success"
-    """
-    if options is None:
-        options = {}
-    
-    # Implementation...
-    return {"status": "success", "result": [], "metadata": {}}
-```
-
-### 2. **Error Handling Patterns**
-
-```python
-# Comprehensive error handling with logging and user-friendly messages
-async def safe_operation(data: Any) -> Dict[str, Any]:
-    """
-    Template for safe operations with comprehensive error handling.
-    """
-    try:
-        # Validation
-        if not data:
-            raise ValueError("Data cannot be empty")
-        
-        # Main operation
-        result = await perform_operation(data)
-        
-        # Success logging
-        logger.log_message("Operation completed successfully", level=logging.INFO)
-        return {"success": True, "data": result}
-        
-    except ValueError as e:
-        # Input validation errors
-        logger.log_message(f"Validation error: {str(e)}", level=logging.WARNING)
-        return {"success": False, "error": "Invalid input", "details": str(e)}
-        
-    except ConnectionError as e:
-        # External service errors
-        logger.log_message(f"Connection error: {str(e)}", level=logging.ERROR)
-        return {"success": False, "error": "Service unavailable", "details": "Please try again later"}
-        
-    except Exception as e:
-        # Unexpected errors
-        logger.log_message(f"Unexpected error in safe_operation: {str(e)}", level=logging.ERROR)
-        return {"success": False, "error": "Internal error", "details": "Please contact support"}
-```
-
-### 3. **Async/Await Best Practices**
-
-```python
-import asyncio
-from typing import List, Coroutine
-
-# Proper async function definition
-async def async_agent_execution(agents: List[str], query: str) -> List[Dict[str, Any]]:
-    """Execute multiple agents concurrently."""
-    
-    # Create coroutines
-    tasks = [
-        execute_single_agent(agent, query) 
-        for agent in agents
-    ]
-    
-    # Execute concurrently with error handling
-    results = []
-    for task in asyncio.as_completed(tasks):
-        try:
-            result = await task
-            results.append(result)
-        except Exception as e:
-            logger.log_message(f"Agent execution failed: {e}", level=logging.ERROR)
-            results.append({"error": str(e)})
-    
-    return results
-
-# Database operations with proper session management
-async def async_database_operation(session: Session) -> Any:
-    """Template for async database operations."""
-    try:
-        # Use asyncio.to_thread for CPU-bound database operations
-        result = await asyncio.to_thread(
-            lambda: session.query(Model).filter(...).all()
-        )
-        return result
-    except Exception as e:
-        session.rollback()
-        raise
-    finally:
-        session.close()
-```
-
-## 🔧 Development Workflow
-
-### 1. **Feature Development Process**
-
-1. **Plan the Feature**:
-   ```bash
-   # Create feature branch
-   git checkout -b feature/new-analysis-agent
-   
-   # Document requirements
-   echo "## New Analysis Agent" >> docs/feature_plan.md
-   ```
-
-2. **Implement Core Logic**:
-   ```bash
-   # Create agent signature
-   # Add to agents_config.json
-   # Implement business logic in managers/
-   # Create route handlers
-   ```
-
-3. **Add Database Changes**:
-   ```bash
-   # Modify models if needed
-   alembic revision --autogenerate -m "Add new analysis tables"
-   alembic upgrade head
-   ```
-
-### 3. **Release Process**
-
-1. **Pre-release Testing**:
-   ```bash
-   # Run full test suite
-   pytest tests/
-   
-   # Test database migrations
-   alembic upgrade head
-   
-   # Test with sample data
-   python scripts/test_with_sample_data.py
-   ```
-
-2. **Documentation Updates**:
-   ```bash
-   # Update API documentation
-   # Update troubleshooting guide
-   # Update changelog
-   ```
-
-3. **Deployment Preparation**:
-   ```bash
-   # Update requirements.txt
-   pip freeze > requirements.txt
-   
-   # Test container build
-   docker build -t auto-analyst-backend .
-   
-   ```
-
-## 📊 Performance Considerations
-
-### 1. **Database Optimization**
-
-```python
-# Use query optimization
-from sqlalchemy.orm import joinedload
-
-# Bad: N+1 query problem
-users = session.query(User).all()
-for user in users:
-    print(user.chats)  # Separate query for each user
-
-# Good: Eager loading
-users = session.query(User).options(joinedload(User.chats)).all()
-for user in users:
-    print(user.chats)  # No additional queries
-
-# Use pagination for large datasets
-def get_paginated_results(session, model, page=1, per_page=20):
-    offset = (page - 1) * per_page
-    return session.query(model).offset(offset).limit(per_page).all()
-```
-
-
-### 2. **Async Optimization**
-
-```python
-# Use connection pooling
-from sqlalchemy.pool import QueuePool
-
-engine = create_engine(
-    DATABASE_URL,
-    poolclass=QueuePool,
-    pool_size=20,
-    max_overflow=30
-)
-
-# Batch operations
-async def batch_process_agents(agents: List[str], queries: List[str]):
-    semaphore = asyncio.Semaphore(5)  # Limit concurrent operations
-    
-    async def process_with_limit(agent, query):
-        async with semaphore:
-            return await process_agent(agent, query)
-    
-    tasks = [
-        process_with_limit(agent, query) 
-        for agent, query in zip(agents, queries)
-    ]
-    
-    return await asyncio.gather(*tasks, return_exceptions=True)
-```
-
-This development workflow guide provides a comprehensive framework for maintaining code quality, consistency, and performance in the Auto-Analyst backend system. Following these patterns ensures that new features integrate seamlessly with the existing architecture while maintaining the high standards of the codebase. 

docs/{api/README.md → endpoints.md}

@@ -5,19 +5,7 @@ This document is a guide to the backend API endpoints utilized within the Auto-A
 The Auto-Analyst application is designed to facilitate seamless interactions and efficient data management, making it essential for users to understand the available endpoints and their functionalities. Each section of this documentation is crafted to provide clarity and insight into how the API operates, ensuring that developers and users alike can effectively leverage its capabilities.
 
 For more specific details regarding the various functionalities offered by the API, please refer to the following sections, which delve deeper into their respective areas:
-
-## 📚 Core Documentation
-
-- **[Getting Started Guide](./getting_started.md)**: Quick start guide for new developers and LLMs to understand the system architecture and get up to speed quickly
-- **[System Architecture](./architecture.md)**: Comprehensive overview of the backend system design, components, and data flow patterns
-- **[Troubleshooting Guide](./troubleshooting.md)**: Common issues, debugging tools, and solutions for development and deployment problems
-
-## 🛠️ API Reference
-
-- **[Core Endpoints](./routes/session.md)**: Review the core endpoints that handle fundamental operations within the application, including data uploads, AI analysis, model settings, and session management.
-- **[Analytics Endpoints](./routes/analytics.md)**: Explore the endpoints dedicated to analytics, providing insights into usage statistics, performance metrics, cost analysis, and real-time monitoring.
-- **[Chat Endpoints](./routes/chats.md)**: Discover the endpoints that manage chat interactions, enabling users to create, retrieve, and manage chat sessions effectively.
-- **[Code Endpoints](./routes/code.md)**: Learn about the endpoints for code execution, editing, fixing, and cleaning operations with advanced AI assistance.
-- **[Deep Analysis Endpoints](./routes/deep_analysis.md)**: Comprehensive documentation for the multi-agent deep analysis system, including streaming progress, report management, template integration, and how user's active agents are leveraged for advanced analytical insights.
-- **[Feedback Endpoints](./routes/feedback.md)**: Understand the endpoints for managing user feedback on AI-generated messages, including rating systems and model performance tracking.
-- **[Templates Endpoints](./routes/templates.md)**: Comprehensive guide to the template system, agent loading, user preferences, and how personalized AI agent configurations work for different users.
+- [Analytics Endpoints](./analytics.md): Explore the endpoints dedicated to analytics, providing insights into usage statistics and performance metrics.
+- [Chat Endpoints](./chats.md): Discover the endpoints that manage chat interactions, enabling users to create, retrieve, and manage chat sessions effectively.
+- [Core Endpoints](./core.md): Review the core endpoints that handle fundamental operations within the application, including data uploads and session management.
+- [Code Endpoints](./code.md): 

docs/getting_started.md

@@ -1,273 +0,0 @@
-# Auto-Analyst Backend - Getting Started Guide
-
-## 🎯 Overview
-
-This guide will help you set up and understand the Auto-Analyst backend system. Auto-Analyst is a multi-agent AI platform that orchestrates specialized agents for comprehensive data analysis.
-
-## 🏗️ Core Concepts
-
-### 1. **Multi-Agent System**
-The platform uses specialized AI agents:
-- **Preprocessing Agent**: Data cleaning and preparation
-- **Statistical Analytics Agent**: Statistical analysis and insights
-- **Machine Learning Agent**: Scikit-learn based modeling
-- **Data Visualization Agent**: Chart and plot generation
-
-### 2. **Template System**
-- **Individual Agents**: Single-purpose agents for specific tasks
-- **Planner Agents**: Multi-agent coordination for complex workflows
-- **User Templates**: Customizable agent preferences
-- **Default vs Premium**: Core agents available to all users
-
-### 3. **Session Management**
-- Session-based user tracking
-- Shared DataFrame context between agents
-- Conversation history and code execution tracking
-
-### 4. **Deep Analysis System**
-- Multi-step analysis workflow (questions → planning → execution → synthesis)
-- Streaming progress updates
-- HTML report generation
-
-## 🚀 Quick Start
-
-### 1. Installation
-
-```bash
-# Clone and navigate to backend
-cd Auto-Analyst-CS/auto-analyst-backend
-
-# Create virtual environment
-python -m venv venv
-source venv/bin/activate  # Linux/Mac
-# or
-venv\Scripts\activate     # Windows
-
-# Install dependencies
-pip install -r requirements.txt
-```
-
-### 2. Environment Variables
-
-Create `.env` file with:
-
-```env
-# Database
-DATABASE_URL=sqlite:///./auto_analyst.db  # For development
-# DATABASE_URL=postgresql://user:pass@host:port/db  # For production
-
-# AI Models
-ANTHROPIC_API_KEY=your_anthropic_key_here
-OPENAI_API_KEY=your_openai_key_here
-
-# Authentication (optional)
-ADMIN_API_KEY=your_admin_key_here
-```
-
-### 3. Database Initialization
-
-```bash
-# Initialize database and default agents
-python -c "
-from src.db.init_db import init_db
-init_db()
-print('✅ Database initialized successfully')
-"
-```
-
-### 4. Start the Server
-
-```bash
-# Development server
-python app.py
-
-# Or with uvicorn
-uvicorn app:app --reload --host 0.0.0.0 --port 8000
-```
-
-### 5. Verify Setup
-
-Visit: `http://localhost:8000/docs` for interactive API documentation
-
-## 📚 Key Files to Understand
-
-### Core Application Files
-
-1. **`app.py`** - Main FastAPI application and core endpoints
-2. **`src/agents/agents.py`** - Agent definitions and orchestration
-3. **`src/agents/deep_agents.py`** - Deep analysis system
-4. **`src/db/schemas/models.py`** - Database models
-5. **`src/managers/chat_manager.py`** - Chat and session management
-
-### Route Files (API Endpoints)
-
-- **`src/routes/session_routes.py`** - File uploads, sessions, authentication
-- **`src/routes/chat_routes.py`** - Chat and messaging
-- **`src/routes/code_routes.py`** - Code execution and processing
-- **`src/routes/templates_routes.py`** - Agent template management
-- **`src/routes/deep_analysis_routes.py`** - Deep analysis reports
-- **`src/routes/analytics_routes.py`** - Usage analytics and monitoring
-
-### Configuration Files
-
-- **`agents_config.json`** - Agent and template definitions
-- **`requirements.txt`** - Python dependencies
-- **`alembic.ini`** - Database migration configuration
-
-## 🔧 Development Workflow
-
-### 1. Adding New Agents
-
-```python
-# 1. Define agent signature in src/agents/agents.py
-class new_agent(dspy.Signature):
-    """Agent description"""
-    goal = dspy.InputField(desc="Analysis goal")
-    dataset = dspy.InputField(desc="Dataset info")
-    result = dspy.OutputField(desc="Analysis result")
-
-# 2. Add to agents_config.json
-{
-  "template_name": "new_agent",
-  "description": "Agent description",
-  "variant_type": "both",
-  "is_premium": false,
-  "usage_count": 0
-}
-
-# 3. Register in agent loading system
-```
-
-### 2. Adding New Endpoints
-
-```python
-# 1. Create route in src/routes/feature_routes.py
-from fastapi import APIRouter
-router = APIRouter(prefix="/feature", tags=["feature"])
-
-@router.get("/endpoint")
-async def new_endpoint():
-    return {"message": "Hello"}
-
-# 2. Register in app.py
-from src.routes.feature_routes import router as feature_router
-app.include_router(feature_router)
-```
-
-### 3. Database Changes
-
-```bash
-# 1. Modify models in src/db/schemas/models.py
-# 2. Create migration
-alembic revision --autogenerate -m "description"
-# 3. Apply migration
-alembic upgrade head
-```
-
-## 🧪 Testing Your Changes
-
-### 1. Test API Endpoints
-
-```bash
-# Use the interactive docs
-open http://localhost:8000/docs
-
-# Or use curl
-curl -X GET "http://localhost:8000/health"
-```
-
-### 2. Test Agent System
-
-```python
-# Test individual agent
-python -c "
-from src.agents.agents import preprocessing_agent
-import dspy
-dspy.LM('anthropic/claude-sonnet-4-20250514')
-agent = dspy.ChainOfThought(preprocessing_agent)
-result = agent(goal='clean data', dataset='test data')
-print(result)
-"
-```
-
-### 3. Test Database Operations
-
-```python
-# Test database
-python -c "
-from src.db.init_db import session_factory
-from src.db.schemas.models import AgentTemplate
-session = session_factory()
-templates = session.query(AgentTemplate).all()
-print(f'Found {len(templates)} templates')
-session.close()
-"
-```
-
-## 🔍 Common Development Tasks
-
-### Adding a New Feature
-
-1. **Plan the Feature**: Define requirements and API design
-2. **Database Changes**: Add new models if needed
-3. **Create Routes**: Add API endpoints in `src/routes/`
-4. **Business Logic**: Add managers in `src/managers/` if complex
-5. **Documentation**: Update relevant `.md` files
-6. **Testing**: Test endpoints and integration
-
-### Debugging Issues
-
-1. **Check Logs**: Application logs show detailed error information
-2. **Database State**: Verify data with database queries
-3. **API Testing**: Use `/docs` interface for endpoint testing
-4. **Agent Behavior**: Test individual agents separately
-
-### Performance Optimization
-
-1. **Database Queries**: Use SQLAlchemy query optimization
-2. **Agent Execution**: Implement async patterns for agent orchestration
-3. **Resource Management**: Monitor memory usage for large datasets
-
-## 📊 System Architecture Overview
-
-```mermaid
-graph TD
-    A[Frontend Request] --> B[FastAPI Router]
-    B --> C[Route Handler]
-    C --> D[Manager Layer]
-    D --> E[Database Layer]
-    D --> F[Agent System]
-    F --> G[AI Models]
-    G --> H[Code Generation]
-    H --> I[Execution Environment]
-    I --> J[Results Processing]
-    J --> K[Response]
-    
-    subgraph "Agent Orchestration"
-        F1[Individual Agents]
-        F2[Planner Module]
-        F3[Deep Analysis]
-        F1 --> F2
-        F2 --> F3
-    end
-    
-    F --> F1
-```
-
-## 📈 Template Integration
-
-The system uses **active user templates** for agent selection:
-
-### Default Agents (Always Available)
-- `preprocessing_agent` (individual & planner variants)
-- `statistical_analytics_agent` (individual & planner variants)  
-- `sk_learn_agent` (individual & planner variants)
-- `data_viz_agent` (individual & planner variants)
-
-### Template Loading Logic
-1. **Individual Agent Execution** (`@agent_name`): Loads ALL available templates
-2. **Planner Execution**: Loads user's enabled templates (max 10 for performance)
-3. **Deep Analysis**: Uses user's active template preferences
-4. **Fallback**: Uses 4 core agents if no user preferences found
-
-This architecture ensures users can leverage their preferred agents while maintaining system performance and reliability. 

docs/routes/analytics.md

@@ -0,0 +1,200 @@
+
+## Analytics Routes Overview
+
+These routes provide comprehensive analytics functionality for the Auto-Analyst application, including dashboard summaries, user and model analytics, and cost breakdowns.
+
+### Authentication
+
+All analytics endpoints require admin authentication via an API key:
+
+```python
+ADMIN_API_KEY = os.getenv("ADMIN_API_KEY", "default-admin-key-change-me")
+```
+
+The API key can be provided via:
+- **Header:** `X-Admin-API-Key`
+- **Query parameter:** `admin_api_key`
+
+---
+
+### Dashboard Endpoints
+
+#### **GET /analytics/dashboard**
+Returns a comprehensive summary of usage data for the dashboard.
+
+**Query Parameters:**
+- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
+
+**Response:**
+```json
+{
+  "total_tokens": 123456,
+  "total_cost": 25.50,
+  "total_requests": 1000,
+  "total_users": 50,
+  "daily_usage": [
+    {
+      "date": "2023-05-01",
+      "tokens": 5000,
+      "cost": 1.25,
+      "requests": 100
+    }
+  ],
+  "model_usage": [
+    {
+      "model_name": "gpt-4",
+      "tokens": 10000,
+      "cost": 10.00,
+      "requests": 200
+    }
+  ],
+  "top_users": [
+    {
+      "user_id": "123",
+      "tokens": 5000,
+      "cost": 5.00,
+      "requests": 50
+    }
+  ],
+  "start_date": "2023-04-01",
+  "end_date": "2023-05-01"
+}
+```
+
+**Process Flow:**
+1. Parse date range from the `period` parameter.
+2. Query total statistics (tokens, cost, requests, users).
+3. Query daily usage broken down by date.
+4. Query model usage statistics.
+5. Query top users by token usage.
+6. Return combined data.
+
+---
+
+### WebSocket **/analytics/dashboard/realtime**
+
+WebSocket endpoint for real-time updates to dashboard data.
+
+**Data Flow:**
+1. Client connects to WebSocket.
+2. Server adds the connection to the active connections set.
+3. Server broadcasts updates to all connected clients when new data arrives.
+4. Connection is removed when the client disconnects.
+
+---
+
+## User Analytics Endpoints
+
+### **GET /analytics/users**
+Returns a list of users with their usage statistics.
+
+**Query Parameters:**
+- `limit` (optional): Maximum number of users to return (default: `100`)
+- `offset` (optional): Offset for pagination (default: `0`)
+
+**Response:**
+```json
+{
+  "users": [
+    {
+      "user_id": "123",
+      "tokens": 5000,
+      "cost": 5.00,
+      "requests": 50,
+      "first_seen": "2023-04-01T12:00:00Z",
+      "last_seen": "2023-05-01T12:00:00Z"
+    }
+  ],
+  "total": 200,
+  "limit": 100,
+  "offset": 0
+}
+```
+
+**Process Flow:**
+1. Query user data with aggregated metrics.
+2. Calculate total users for pagination.
+3. Format and return data.
+
+---
+
+### **GET /analytics/users/activity**
+Returns user activity metrics over time.
+
+**Query Parameters:**
+- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
+
+**Response:**
+```json
+{
+  "user_activity": [
+    {
+      "date": "2023-05-01",
+      "activeUsers": 20,
+      "newUsers": 5,
+      "sessions": 30
+    }
+  ]
+}
+```
+
+**Process Flow:**
+1. Parse date range from `period` parameter.
+2. Get first date each user was seen (for new users count).
+3. Get daily activity metrics.
+4. Fill in any missing dates with zeros.
+5. Return formatted data.
+
+---
+
+## Model Analytics Endpoints
+
+### **GET /analytics/usage/models**
+Returns model usage breakdown.
+
+**Query Parameters:**
+- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
+
+**Response:**
+```json
+{
+  "model_usage": [
+    {
+      "model_name": "gpt-4",
+      "tokens": 10000,
+      "cost": 10.00,
+      "requests": 200,
+      "avg_response_time": 1.5
+    }
+  ]
+}
+```
+
+**Process Flow:**
+1. Parse date range from `period` parameter.
+2. Query model usage with aggregated metrics.
+3. Format and return data.
+
+---
+
+## Cost Analytics Endpoints
+
+### **GET /analytics/costs/summary**
+Returns a summary of costs.
+
+**Query Parameters:**
+- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
+
+**Response:**
+```json
+{
+  "totalCost": 25.50,
+  "totalTokens": 100000,
+  "totalRequests": 1000,
+  "avgDailyCost": 0.85,
+  "costPerThousandTokens": 0.255,
+  "daysInPeriod": 30,
+  "startDate": "2023-04-01",
+  "endDate": "2023-05-01"
+}
+```

docs/{api/routes → routes}/chats.md

@@ -7,23 +7,27 @@ These routes handle chat interactions, message processing, user management, and
 ### **Chat Management**
 
 #### **1. Create a New Chat**
-**Endpoint:** `POST /chats/`  
+**Endpoint:** `POST /chats`  
 **Description:** Creates a new chat session.  
 **Request Body:**  
 ```json
 {
-  "user_id": 123
+  "user_id": 123,
+  "is_admin": false
 }
 ```
 **Response:**  
 ```json
 {
   "chat_id": 456,
-  "user_id": 123,
   "title": "New Chat",
-  "created_at": "2023-05-01T12:00:00Z"
+  "created_at": "2023-05-01T12:00:00Z",
+  "user_id": 123
 }
 ```
+**Process Flow:**  
+1. Create a new chat for the given user.  
+2. Return the chat details.  
 
 ---
 
@@ -50,33 +54,42 @@ These routes handle chat interactions, message processing, user management, and
   ]
 }
 ```
+**Process Flow:**  
+1. Retrieve chat details.  
+2. Validate user access if `user_id` is provided.  
+3. Fetch all messages in the chat.  
+4. Return the chat with its messages.  
 
 ---
 
 #### **3. List Recent Chats**
-**Endpoint:** `GET /chats/`  
+**Endpoint:** `GET /chats`  
 **Description:** Retrieves a list of recent chats, optionally filtered by user ID.  
 **Query Parameters:**  
 - `user_id` (Optional for filtering by user)  
-- `limit` (Maximum number of chats, default: 10, max: 100)  
+- `limit` (Maximum number of chats, default: 10)  
 - `offset` (For pagination, default: 0)  
 **Response:**  
 ```json
 [
   {
     "chat_id": 456,
-    "user_id": 123,
     "title": "New Chat",
-    "created_at": "2023-05-01T12:00:00Z"
+    "created_at": "2023-05-01T12:00:00Z",
+    "user_id": 123
   }
 ]
 ```
+**Process Flow:**  
+1. Fetch recent chats.  
+2. Apply filters and pagination.  
+3. Return the list of chats.  
 
 ---
 
 #### **4. Update a Chat**
 **Endpoint:** `PUT /chats/{chat_id}`  
-**Description:** Updates a chat's title or user ID.  
+**Description:** Updates a chat’s title or user ID.  
 **Path Parameter:** `chat_id` (ID of the chat to update)  
 **Request Body:**  
 ```json
@@ -94,25 +107,57 @@ These routes handle chat interactions, message processing, user management, and
   "user_id": 123
 }
 ```
+**Process Flow:**  
+1. Update the chat’s title or user ID.  
+2. Return the updated details.  
 
 ---
 
 #### **5. Delete a Chat**
 **Endpoint:** `DELETE /chats/{chat_id}`  
-**Description:** Deletes a chat and all its messages while preserving model usage records.  
+**Description:** Deletes a chat and all its messages.  
 **Path Parameter:** `chat_id` (ID of the chat to delete)  
 **Query Parameter:** `user_id` (Optional for access control)  
 **Response:**  
 ```json
 {
-  "message": "Chat 456 deleted successfully",
-  "preserved_model_usage": true
+  "message": "Chat 456 deleted successfully"
 }
 ```
+**Process Flow:**  
+1. Validate if the chat exists and if the user has access.  
+2. Delete the chat and associated messages.  
+3. Return a success message.  
+
+---
+
+#### **6. Search Chats**
+**Endpoint:** `GET /chats/search/`  
+**Description:** Searches chats based on a query string.  
+**Query Parameters:**  
+- `query` (Search term)  
+- `user_id` (Optional filter)  
+- `limit` (Max results)  
+**Response:**  
+```json
+[
+  {
+    "chat_id": 456,
+    "title": "Chat about machine learning",
+    "created_at": "2023-05-01T12:00:00Z",
+    "user_id": 123
+  }
+]
+```
+**Process Flow:**  
+1. Search chat titles and messages for the query.  
+2. Filter by `user_id` if provided.  
+3. Apply a result limit.  
+4. Return the matching chats.  
 
 ---
 
-#### **6. Cleanup Empty Chats**
+#### **7. Cleanup Empty Chats**
 **Endpoint:** `POST /chats/cleanup-empty`  
 **Description:** Deletes empty chats for a user.  
 **Request Body:**  
@@ -128,33 +173,30 @@ These routes handle chat interactions, message processing, user management, and
   "message": "Deleted 5 empty chats"
 }
 ```
+**Process Flow:**  
+1. Identify chats with no messages.  
+2. Delete those chats.  
+3. Return the count of deleted chats.  
 
 ---
 
 ### **Message Management**
 
-#### **1. Add Message to Chat**
-**Endpoint:** `POST /chats/{chat_id}/messages`  
-**Description:** Adds a message to an existing chat.  
+
+
+#### **1. Send a Message and Get AI Response**
+**Endpoint:** `POST /chats/{chat_id}/message`  
+**Description:** Sends a message and receives an AI-generated response.  
 **Path Parameter:** `chat_id` (ID of the chat)  
-**Query Parameter:** `user_id` (Optional for access control)  
 **Request Body:**  
-```json
-{
-  "content": "Hello, I need help with data analysis",
-  "sender": "user"
-}
-```
-**Response:**  
-```json
-{
-  "message_id": 789,
-  "chat_id": 456,
-  "content": "Hello, I need help with data analysis",
-  "sender": "user",
-  "timestamp": "2023-05-01T12:01:00Z"
-}
-```
+**Process Flow:**  
+1. Verify chat existence and user access.  
+2. Store the user message.  
+3. Generate an AI response via `ai_manager`.  
+4. Track model usage metrics.  
+5. Store the AI response.  
+6. Update the chat title if it's the first or second message.  
+7. Return both messages.  
 
 ---
 
@@ -179,3 +221,37 @@ These routes handle chat interactions, message processing, user management, and
   "created_at": "2023-05-01T12:00:00Z"
 }
 ```
+**Process Flow:**  
+1. Check if a user with the email exists.  
+2. Create a new user if not found.  
+3. Return the user details.  
+
+---
+
+### **Debugging**
+
+#### **1. Test Model Usage Tracking**
+**Endpoint:** `POST /chats/debug/test-model-usage`  
+**Query Parameters:**  
+- `model_name`: Model to test  
+- `user_id`: Optional  
+**Response:**  
+```json
+{
+  "success": true,
+  "message": "Model usage tracking test completed",
+  "response": "This is a test response",
+  "usage_recorded": {
+    "usage_id": 123,
+    "model_name": "gpt-4",
+    "tokens": 50,
+    "cost": 0.0005,
+    "timestamp": "2023-05-01T12:00:00Z"
+  }
+}
+```
+**Process Flow:**  
+1. Generate a test prompt.  
+2. Call AI manager.  
+3. Fetch usage data.  
+4. Return test results.

docs/{api/routes → routes}/code.md

@@ -16,17 +16,15 @@ Executes Python code against the current session's dataframe.
 **Request Body:**
 ```json
 {
-    "code": "string",              // Python code to execute
-    "session_id": "string",        // Optional session ID
-    "message_id": 123              // Optional message ID for tracking
+    "code": "string"  // Python code to execute
 }
 ```
 
 **Response:**
 ```json
 {
-    "output": "string",            // Execution output
-    "plotly_outputs": [            // Optional array of plotly outputs
+    "output": "string",  // Execution output
+    "plotly_outputs": [  // Optional array of plotly outputs
         "string"
     ]
 }
@@ -44,15 +42,15 @@ Uses AI to edit code based on user instructions.
 **Request Body:**
 ```json
 {
-    "original_code": "string",     // Code to be edited
-    "user_prompt": "string"        // Instructions for editing
+    "original_code": "string",  // Code to be edited
+    "user_prompt": "string"     // Instructions for editing
 }
 ```
 
 **Response:**
 ```json
 {
-    "edited_code": "string"        // The edited code
+    "edited_code": "string"  // The edited code
 }
 ```
 
@@ -61,22 +59,22 @@ Uses AI to edit code based on user instructions.
 - `500 Internal Server Error`: Editing error
 
 ### Fix Code
-Uses AI to fix code with errors, employing a block-by-block approach with DSPy refinement.
+Uses AI to fix code with errors, employing a block-by-block approach.
 
 **Endpoint:** `POST /code/fix`
 
 **Request Body:**
 ```json
 {
-    "code": "string",              // Code containing errors
-    "error": "string"              // Error message to fix
+    "code": "string",    // Code containing errors
+    "error": "string"    // Error message to fix
 }
 ```
 
 **Response:**
 ```json
 {
-    "fixed_code": "string"         // The fixed code
+    "fixed_code": "string"  // The fixed code
 }
 ```
 
@@ -92,14 +90,14 @@ Cleans and formats code by organizing imports and ensuring proper code block for
 **Request Body:**
 ```json
 {
-    "code": "string"               // Code to clean
+    "code": "string"  // Code to clean
 }
 ```
 
 **Response:**
 ```json
 {
-    "cleaned_code": "string"       // The cleaned code
+    "cleaned_code": "string"  // The cleaned code
 }
 ```
 
@@ -107,30 +105,6 @@ Cleans and formats code by organizing imports and ensuring proper code block for
 - `400 Bad Request`: No code provided
 - `500 Internal Server Error`: Cleaning error
 
-### Get Latest Code
-Retrieves the latest code from a specific message.
-
-**Endpoint:** `POST /code/get-latest-code`
-
-**Request Body:**
-```json
-{
-    "message_id": 123              // Message ID to retrieve code from
-}
-```
-
-**Response:**
-```json
-{
-    "code": "string"               // The retrieved code
-}
-```
-
-**Error Responses:**
-- `400 Bad Request`: Missing message ID
-- `404 Not Found`: Message not found
-- `500 Internal Server Error`: Retrieval error
-
 ## Code Processing Features
 
 ### Import Organization
@@ -144,12 +118,10 @@ The system supports code blocks marked with special comments:
 - Start marker: `# agent_name code start`
 - End marker: `# agent_name code end`
 
-### Error Handling with DSPy Refinement
-When fixing code, the system uses DSPy's refinement mechanism:
+### Error Handling
+When fixing code, the system:
 - Identifies specific code blocks with errors
 - Processes error messages to extract relevant information
-- Uses a scoring function to validate fixes
-- Employs iterative refinement with up to 3 attempts
 - Fixes each block individually while maintaining the overall structure
 - Preserves code block markers and relationships
 
@@ -160,23 +132,14 @@ When editing or fixing code, the system provides context about the current datas
 - Null value counts
 - Sample values for each column
 
-### Code Execution Safety
-The execution system includes safety measures:
-- Removes blocking calls like `plt.show()`
-- Handles `__main__` block extraction
-- Cleans up print statements with unwanted newlines
-- Executes code in isolated namespaces
-
 ## Session Management
 All endpoints require a valid session ID, which is used to:
 - Access the current dataset
 - Maintain state between requests
 - Track code execution history
-- Store execution results for analysis
 
 ## Error Handling
 The system provides detailed error messages while maintaining security by:
 - Logging errors for debugging
 - Returning user-friendly error messages
 - Preserving original code in case of processing failures
-- Using code scoring to validate fixes before returning results

docs/routes/core.md

@@ -0,0 +1,242 @@
+
+# **Auto-Analyst API Documentation**
+
+The core application routes are designed to manage the data and AI analysis capabilities of the Auto-Analyst application.
+
+## **1. Core Application Routes**
+### **Data Management**
+- **POST /upload_dataframe**  
+  **Uploads a dataset for analysis.**  
+  **Request:**  
+  - `file`: CSV file  
+  - `name`: Dataset name  
+  - `description`: Dataset description  
+  - `session_id`: Session ID  
+  **Response:**  
+  ```json
+  { "message": "Dataframe uploaded successfully", "session_id": "abc123" }
+  ```
+  **Process Flow:**  
+  - Read CSV file  
+  - Create dataset description  
+  - Update session with dataset  
+  - Return success message  
+
+- **GET /api/default-dataset**  
+  **Gets the default dataset.**  
+  **Query Parameters:** `session_id`  
+  **Response:**  
+  ```json
+  {
+    "headers": ["column1", "column2", ...],
+    "rows": [[val1, val2, ...], ...],
+    "name": "Housing Dataset",
+    "description": "A comprehensive dataset containing housing information..."
+  }
+  ```
+  **Process Flow:**  
+  - Reset session to use default dataset  
+  - Format dataset preview  
+  - Return formatted data  
+
+- **POST /reset-session**  
+  **Resets session to default dataset.**  
+  **Query Parameters:** `session_id`, `name` (optional), `description` (optional)  
+  **Response:**  
+  ```json
+  {
+    "message": "Session reset to default dataset",
+    "session_id": "abc123",
+    "dataset": "Housing.csv"
+  }
+  ```
+  **Process Flow:**  
+  - Reset session  
+  - Update dataset description (if provided)  
+  - Return success message  
+
+---
+
+### **2. AI Analysis**
+- **POST /chat/{agent_name}**  
+  **Processes a query using a specific AI agent.**  
+  **Path Parameters:** `agent_name`  
+  **Request Body:**  
+  ```json
+  { "query": "Analyze the relationship between price and size" }
+  ```
+  **Query Parameters:** `session_id`, `user_id` (optional), `chat_id` (optional)  
+  **Response:**  
+  ```json
+  {
+    "agent_name": "data_viz_agent",
+    "query": "Analyze the relationship between price and size",
+    "response": "# Analysis\n\nThere appears to be a strong positive correlation...",
+    "session_id": "abc123"
+  }
+  ```
+  **Process Flow:**  
+  - Get session state  
+  - Validate dataset and agent  
+  - Execute agent query  
+  - Format and return response  
+
+- **POST /chat**  
+  **Processes a query using multiple AI agents.**  
+  **Request Body:**  
+  ```json
+  { "query": "Analyze the housing data" }
+  ```
+  **Response:** *Streaming JSON objects:*  
+  ```json
+  {"agent": "data_viz_agent", "content": "# Visualization\n\n...", "status": "success"}
+  {"agent": "statistical_analytics_agent", "content": "# Statistical Analysis\n\n...", "status": "success"}
+  ```
+  **Process Flow:**  
+  - Get session state  
+  - Validate dataset  
+  - Generate AI analysis plan  
+  - Execute with multiple agents and stream responses  
+
+- **POST /execute_code**  
+  **Executes Python code and returns results.**  
+  **Request Body:**  
+  ```json
+  { "code": "import pandas as pd\nimport matplotlib.pyplot as plt\n..." }
+  ```
+  **Response:**  
+  ```json
+  {
+    "output": "Execution successful",
+    "plotly_outputs": ["```plotly\n{\"data\": [...], \"layout\": {...}}\n```"]
+  }
+  ```
+  **Process Flow:**  
+  - Validate dataset  
+  - Execute code  
+  - Capture output and plots  
+  - Return results  
+
+---
+
+### **3. Model Settings**
+- **GET /api/model-settings**  
+  **Fetches current model settings.**  
+  **Response:**  
+  ```json
+  {
+    "provider": "openai",
+    "model": "gpt-4o-mini",
+    "hasCustomKey": true,
+    "temperature": 1.0,
+    "maxTokens": 6000
+  }
+  ```
+- **POST /settings/model**  
+  **Updates model settings.**  
+  **Request Body:**  
+  ```json
+  {
+    "provider": "openai",
+    "model": "gpt-4",
+    "api_key": "sk-...",
+    "temperature": 0.7,
+    "max_tokens": 8000
+  }
+  ```
+  **Response:**  
+  ```json
+  { "message": "Model settings updated successfully" }
+  ```
+  **Process Flow:**  
+  - Update model settings  
+  - Test configuration  
+  - Return success or error  
+
+- **GET /agents**  
+  **Lists available AI agents.**  
+  **Response:**  
+  ```json
+  {
+    "available_agents": ["data_viz_agent", "sk_learn_agent", "statistical_analytics_agent", "preprocessing_agent"],
+    "description": "List of available specialized agents that can be called using @agent_name"
+  }
+  ```
+
+---
+
+### **4. Authentication & Session Management**
+- **Session ID Sources:**  
+  - Query parameter: `session_id`  
+  - Header: `X-Session-ID`  
+  - Auto-generated if not provided  
+- **Session State Includes:**  
+  - Current dataset  
+  - AI system instance  
+  - Model configuration  
+
+#### **Admin Authentication**
+- **API Key Sources:**  
+  - Header: `X-Admin-API-Key`  
+  - Query Parameter: `admin_api_key`  
+- **Validation:**  
+  - Checked against `ADMIN_API_KEY` environment variable  
+
+---
+
+### **5. AI Agents Integration**
+- **Available Agents:**  
+  - `data_viz_agent`: Creates data visualizations (Plotly)  
+  - `sk_learn_agent`: ML analysis (Scikit-learn)  
+  - `statistical_analytics_agent`: Statistical analysis (StatsModels)  
+  - `preprocessing_agent`: Data transformation  
+
+**Integration Flow:**  
+- Agents are registered in `AVAILABLE_AGENTS`  
+- Queries are dispatched based on content  
+- Responses are streamed in Markdown  
+
+---
+
+### **6. Real-time Updates via WebSockets**
+- **Endpoints:**  
+  - `/analytics/dashboard/realtime`  
+  - `/analytics/realtime`  
+- **Connections stored in:**  
+  - `active_dashboard_connections`  
+  - `active_user_connections`  
+- **Broadcast Function:**
+  ```python
+  async def broadcast_dashboard_update(update_data: Dict[str, Any]):
+      for connection in active_dashboard_connections.copy():
+          try:
+              await connection.send_text(json.dumps(update_data))
+          except Exception:
+              active_dashboard_connections.remove(connection)
+  ```
+
+---
+
+### **7. Error Handling**
+- **Try-Except Blocks:**  
+  ```python
+  try:
+      return result
+  except ValueError as e:
+      raise HTTPException(status_code=404, detail=str(e))
+  except Exception as e:
+      logger.error(f"Error: {str(e)}")
+      raise HTTPException(status_code=500, detail=f"Failed: {str(e)}")
+  ```
+- **HTTP Exception Types:**  
+  - `400`: Bad Request  
+  - `401`: Unauthorized  
+  - `403`: Forbidden  
+  - `404`: Not Found  
+  - `500`: Internal Server Error  
+
+- **Logging Setup:**  
+  ```python
+  logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+  logger = logging.getLogger(__name__)
+  ```

docs/system/database-schema.md

@@ -1,289 +0,0 @@
-# Auto-Analyst Database Schema Documentation
-
-## 📋 Overview
-
-The Auto-Analyst backend uses a relational database schema designed for scalability and data integrity. The schema supports both **SQLite** (development) and **PostgreSQL** (production) databases through SQLAlchemy ORM.
-
-### **Database Features**
-- **User Management** - Authentication and user data
-- **Chat System** - Conversation sessions and message history  
-- **AI Model Tracking** - Usage analytics and cost monitoring
-- **Code Execution** - Code generation and execution tracking
-- **Agent Templates** - Customizable AI agent configurations
-- **Deep Analysis** - Multi-step analysis reports and results
-- **User Feedback** - Rating and feedback system
-
----
-
-## 🗄️ Database Tables
-
-### **1. Users Table (`users`)**
-
-**Purpose**: Core user authentication and profile management
-
-| Column | Type | Constraints | Description |
-|--------|------|-------------|-------------|
-| `user_id` | `INTEGER` | PRIMARY KEY, AUTO INCREMENT | Unique user identifier |
-| `username` | `STRING` | UNIQUE, NOT NULL | User's display name |
-| `email` | `STRING` | UNIQUE, NOT NULL | User's email address |
-| `created_at` | `DATETIME` | DEFAULT: UTC NOW | Account creation timestamp |
-
-**Relationships:**
-- **One-to-Many**: `chats` (User → Chat sessions)
-- **One-to-Many**: `usage_records` (User → Model usage tracking)
-- **One-to-Many**: `deep_analysis_reports` (User → Analysis reports)
-- **One-to-Many**: `template_preferences` (User → Agent preferences)
-
----
-
-### **2. Chats Table (`chats`)**
-
-**Purpose**: Conversation sessions and chat organization
-
-| Column | Type | Constraints | Description |
-|--------|------|-------------|-------------|
-| `chat_id` | `INTEGER` | PRIMARY KEY, AUTO INCREMENT | Unique chat session identifier |
-| `user_id` | `INTEGER` | FOREIGN KEY → `users.user_id`, CASCADE DELETE | Chat owner (nullable for anonymous) |
-| `title` | `STRING` | DEFAULT: 'New Chat' | Human-readable chat title |
-| `created_at` | `DATETIME` | DEFAULT: UTC NOW | Chat creation timestamp |
-
-**Relationships:**
-- **Many-to-One**: `user` (Chat → User)
-- **One-to-Many**: `messages` (Chat → Messages)
-- **One-to-Many**: `usage_records` (Chat → Model usage)
-
----
-
-### **3. Messages Table (`messages`)**
-
-**Purpose**: Individual messages within chat conversations
-
-| Column | Type | Constraints | Description |
-|--------|------|-------------|-------------|
-| `message_id` | `INTEGER` | PRIMARY KEY, AUTO INCREMENT | Unique message identifier |
-| `chat_id` | `INTEGER` | FOREIGN KEY → `chats.chat_id`, CASCADE DELETE | Parent chat session |
-| `sender` | `STRING` | NOT NULL | Message sender: 'user' or 'ai' |
-| `content` | `TEXT` | NOT NULL | Message content (text/markdown) |
-| `timestamp` | `DATETIME` | DEFAULT: UTC NOW | Message creation time |
-
-**Relationships:**
-- **Many-to-One**: `chat` (Message → Chat)
-- **One-to-One**: `feedback` (Message → Feedback)
-
----
-
-### **4. Model Usage Table (`model_usage`)**
-
-**Purpose**: AI model usage tracking for analytics and billing
-
-| Column | Type | Constraints | Description |
-|--------|------|-------------|-------------|
-| `usage_id` | `INTEGER` | PRIMARY KEY | Unique usage record identifier |
-| `user_id` | `INTEGER` | FOREIGN KEY → `users.user_id`, SET NULL | User who triggered the usage |
-| `chat_id` | `INTEGER` | FOREIGN KEY → `chats.chat_id`, SET NULL | Associated chat session |
-| `model_name` | `STRING(100)` | NOT NULL | AI model used (e.g., 'gpt-4o-mini') |
-| `provider` | `STRING(50)` | NOT NULL | Model provider ('openai', 'anthropic', etc.) |
-| `prompt_tokens` | `INTEGER` | DEFAULT: 0 | Input tokens consumed |
-| `completion_tokens` | `INTEGER` | DEFAULT: 0 | Output tokens generated |
-| `total_tokens` | `INTEGER` | DEFAULT: 0 | Total tokens (input + output) |
-| `query_size` | `INTEGER` | DEFAULT: 0 | Query size in characters |
-| `response_size` | `INTEGER` | DEFAULT: 0 | Response size in characters |
-| `cost` | `FLOAT` | DEFAULT: 0.0 | Cost in USD for this usage |
-| `timestamp` | `DATETIME` | DEFAULT: UTC NOW | Usage timestamp |
-| `is_streaming` | `BOOLEAN` | DEFAULT: FALSE | Whether response was streamed |
-| `request_time_ms` | `INTEGER` | DEFAULT: 0 | Request processing time (milliseconds) |
-
-**Relationships:**
-- **Many-to-One**: `user` (Usage → User)
-- **Many-to-One**: `chat` (Usage → Chat)
-
----
-
-### **5. Code Executions Table (`code_executions`)**
-
-**Purpose**: Track code generation and execution attempts
-
-| Column | Type | Constraints | Description |
-|--------|------|-------------|-------------|
-| `execution_id` | `INTEGER` | PRIMARY KEY, AUTO INCREMENT | Unique execution identifier |
-| `message_id` | `INTEGER` | FOREIGN KEY → `messages.message_id`, CASCADE DELETE | Associated message |
-| `chat_id` | `INTEGER` | FOREIGN KEY → `chats.chat_id`, CASCADE DELETE | Parent chat session |
-| `user_id` | `INTEGER` | FOREIGN KEY → `users.user_id`, SET NULL | User who triggered execution |
-| `initial_code` | `TEXT` | NULLABLE | First version of generated code |
-| `latest_code` | `TEXT` | NULLABLE | Most recent code version |
-| `is_successful` | `BOOLEAN` | DEFAULT: FALSE | Whether execution succeeded |
-| `output` | `TEXT` | NULLABLE | Execution output (including errors) |
-| `model_provider` | `STRING(50)` | NULLABLE | AI model provider used |
-| `model_name` | `STRING(100)` | NULLABLE | AI model name used |
-| `failed_agents` | `TEXT` | NULLABLE | JSON list of failed agent names |
-| `error_messages` | `TEXT` | NULLABLE | JSON map of error messages by agent |
-| `created_at` | `DATETIME` | DEFAULT: UTC NOW | Execution creation time |
-| `updated_at` | `DATETIME` | DEFAULT: UTC NOW, ON UPDATE | Last update timestamp |
-
----
-
-### **6. Message Feedback Table (`message_feedback`)**
-
-**Purpose**: User feedback and model settings for messages
-
-| Column | Type | Constraints | Description |
-|--------|------|-------------|-------------|
-| `feedback_id` | `INTEGER` | PRIMARY KEY, AUTO INCREMENT | Unique feedback identifier |
-| `message_id` | `INTEGER` | FOREIGN KEY → `messages.message_id`, CASCADE DELETE | Associated message |
-| `rating` | `INTEGER` | NULLABLE | Star rating (1-5 scale) |
-| `model_name` | `STRING(100)` | NULLABLE | Model used for this message |
-| `model_provider` | `STRING(50)` | NULLABLE | Model provider used |
-| `temperature` | `FLOAT` | NULLABLE | Temperature setting used |
-| `max_tokens` | `INTEGER` | NULLABLE | Max tokens setting used |
-| `created_at` | `DATETIME` | DEFAULT: UTC NOW | Feedback creation time |
-| `updated_at` | `DATETIME` | DEFAULT: UTC NOW, ON UPDATE | Last update timestamp |
-
-**Relationships:**
-- **One-to-One**: `message` (Feedback ↔ Message)
-
----
-
-### **7. Deep Analysis Reports Table (`deep_analysis_reports`)**
-
-**Purpose**: Store comprehensive multi-agent analysis reports
-
-| Column | Type | Constraints | Description |
-|--------|------|-------------|-------------|
-| `report_id` | `INTEGER` | PRIMARY KEY, AUTO INCREMENT | Unique report identifier |
-| `report_uuid` | `STRING(100)` | UNIQUE, NOT NULL | Frontend-generated UUID |
-| `user_id` | `INTEGER` | FOREIGN KEY → `users.user_id`, CASCADE DELETE | Report owner |
-| `goal` | `TEXT` | NOT NULL | Analysis objective/question |
-| `status` | `STRING(20)` | NOT NULL, DEFAULT: 'pending' | Status: 'pending', 'running', 'completed', 'failed' |
-| `start_time` | `DATETIME` | DEFAULT: UTC NOW | Analysis start time |
-| `end_time` | `DATETIME` | NULLABLE | Analysis completion time |
-| `duration_seconds` | `INTEGER` | NULLABLE | Total analysis duration |
-| `deep_questions` | `TEXT` | NULLABLE | Generated analytical questions |
-| `deep_plan` | `TEXT` | NULLABLE | Analysis execution plan |
-| `summaries` | `JSON` | NULLABLE | Array of analysis summaries |
-| `analysis_code` | `TEXT` | NULLABLE | Generated Python code |
-| `plotly_figures` | `JSON` | NULLABLE | Array of Plotly figure data |
-| `synthesis` | `JSON` | NULLABLE | Array of synthesis insights |
-| `final_conclusion` | `TEXT` | NULLABLE | Final analysis conclusion |
-| `html_report` | `TEXT` | NULLABLE | Complete HTML report |
-| `progress_percentage` | `INTEGER` | DEFAULT: 0 | Progress percentage (0-100) |
-| `total_tokens_used` | `INTEGER` | DEFAULT: 0 | Total tokens consumed |
-| `estimated_cost` | `FLOAT` | DEFAULT: 0.0 | Estimated cost in USD |
-| `credits_consumed` | `INTEGER` | DEFAULT: 0 | Credits deducted for analysis |
-| `created_at` | `DATETIME` | DEFAULT: UTC NOW | Report creation time |
-| `updated_at` | `DATETIME` | DEFAULT: UTC NOW, ON UPDATE | Last update timestamp |
-
-**Relationships:**
-- **Many-to-One**: `user` (Report → User)
-
----
-
-### **8. Agent Templates Table (`agent_templates`)**
-
-**Purpose**: Store predefined AI agent configurations
-
-| Column | Type | Constraints | Description |
-|--------|------|-------------|-------------|
-| `template_id` | `INTEGER` | PRIMARY KEY, AUTO INCREMENT | Unique template identifier |
-| `template_name` | `STRING(100)` | UNIQUE, NOT NULL | Internal template name |
-| `display_name` | `STRING(200)` | NULLABLE | User-friendly display name |
-| `description` | `TEXT` | NOT NULL | Template description |
-| `prompt_template` | `TEXT` | NOT NULL | Agent behavior instructions |
-| `icon_url` | `STRING(500)` | NULLABLE | Template icon URL |
-| `category` | `STRING(50)` | NULLABLE | Template category |
-| `is_premium_only` | `BOOLEAN` | DEFAULT: FALSE | Requires premium subscription |
-| `variant_type` | `STRING(20)` | DEFAULT: 'individual' | 'planner', 'individual', or 'both' |
-| `is_active` | `BOOLEAN` | DEFAULT: TRUE | Template is active/available |
-| `created_at` | `DATETIME` | DEFAULT: UTC NOW | Template creation time |
-
-**Relationships:**
-- **One-to-Many**: `user_preferences` (Template → User preferences)
-
----
-
-### **9. User Template Preferences Table (`user_template_preferences`)**
-
-**Purpose**: Track user preferences and usage for agent templates
-
-| Column | Type | Constraints | Description |
-|--------|------|-------------|-------------|
-| `preference_id` | `INTEGER` | PRIMARY KEY, AUTO INCREMENT | Unique preference identifier |
-| `user_id` | `INTEGER` | FOREIGN KEY → `users.user_id`, CASCADE DELETE | User who owns preference |
-| `template_id` | `INTEGER` | FOREIGN KEY → `agent_templates.template_id`, CASCADE DELETE | Associated template |
-| `is_enabled` | `BOOLEAN` | DEFAULT: TRUE | Whether user has template enabled |
-| `usage_count` | `INTEGER` | DEFAULT: 0 | Number of times user used template |
-| `last_used_at` | `DATETIME` | NULLABLE | Last time user used template |
-| `created_at` | `DATETIME` | DEFAULT: UTC NOW | Preference creation time |
-
-**Relationships:**
-- **Many-to-One**: `user` (Preference → User)
-- **Many-to-One**: `template` (Preference → Template)
-
-**Constraints:**
-- **Unique**: `(user_id, template_id)` - One preference per user per template
-
----
-
-## 🔗 Entity Relationship Diagram
-
-```
-Users (1) ──────────── (Many) Chats
-  │                           │
-  │                           ├── (Many) Messages
-  │                           │      │
-  │                           │      └── (1) MessageFeedback
-  │                           │
-  │                           └── (Many) CodeExecutions
-  │
-  ├── (Many) ModelUsage
-  │
-  ├── (Many) DeepAnalysisReports
-  │
-  └── (Many) UserTemplatePreferences
-               │
-               └── (Many) AgentTemplates
-```
-
----
-
-## 📊 Database Performance
-
-### **Optimized Indexes**
-
-```sql
--- High-performance queries
-CREATE INDEX idx_messages_chat_timestamp ON messages(chat_id, timestamp DESC);
-CREATE INDEX idx_model_usage_user_time ON model_usage(user_id, timestamp DESC);
-CREATE INDEX idx_model_usage_model_time ON model_usage(model_name, timestamp DESC);
-CREATE INDEX idx_reports_user_time ON deep_analysis_reports(user_id, created_at DESC);
-```
-
-### **Cascade Deletion Rules**
-
-| Parent → Child | Rule | Description |
-|----------------|------|-------------|
-| `users` → `chats` | CASCADE | Delete all user chats when user deleted |
-| `chats` → `messages` | CASCADE | Delete all chat messages when chat deleted |
-| `messages` → `feedback` | CASCADE | Delete feedback when message deleted |
-| `users` → `model_usage` | SET NULL | Keep usage records for analytics |
-
----
-
-## 🛡️ Security & Maintenance
-
-### **Data Protection**
-- User data isolated by `user_id`
-- Sensitive fields require encryption in production
-- Automatic cleanup of anonymous data after 90 days
-
-### **Regular Maintenance**
-```sql
--- Clean old anonymous chats
-DELETE FROM chats WHERE user_id IS NULL AND created_at < DATE_SUB(NOW(), INTERVAL 90 DAY);
-
--- Update statistics for query optimization
-ANALYZE users, chats, messages, model_usage;
-```
-
----
-
-This schema supports the full Auto-Analyst application with optimized performance, data integrity, and scalability for both development and production environments. 

docs/troubleshooting/troubleshooting.md

@@ -1,537 +0,0 @@
-# Auto-Analyst Backend Troubleshooting Guide
-
-## 🚨 Common Startup Issues
-
-### 1. **Database Connection Problems**
-
-#### Problem: Database connection failed
-```
-❌ sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) no such table: users
-```
-
-**Solutions:**
-1. **Initialize Database**:
-   ```bash
-   python -c "
-   from src.db.init_db import init_db
-   init_db()
-   print('✅ Database initialized')
-   "
-   ```
-
-2. **Check Database File Permissions**:
-   ```bash
-   # For SQLite
-   ls -la auto_analyst.db
-   chmod 666 auto_analyst.db  # If needed
-   ```
-
-3. **Verify DATABASE_URL**:
-   ```bash
-   # Check .env file
-   cat .env | grep DATABASE_URL
-   
-   # For PostgreSQL (production)
-   DATABASE_URL=postgresql://user:password@host:port/database
-   
-   # For SQLite (development)  
-   DATABASE_URL=sqlite:///./auto_analyst.db
-   ```
-
-#### Problem: PostgreSQL connection issues
-```
-❌ psycopg2.OperationalError: FATAL: database "auto_analyst" does not exist
-```
-
-**Solutions:**
-1. **Create Database**:
-   ```sql
-   -- Connect to PostgreSQL
-   psql -h localhost -U postgres
-   CREATE DATABASE auto_analyst;
-   \q
-   ```
-
-2. **Update Connection String**:
-   ```env
-   DATABASE_URL=postgresql://username:password@localhost:5432/auto_analyst
-   ```
-
-### 2. **Agent Template Loading Issues**
-
-#### Problem: No agents found
-```
-❌ RuntimeError: No agents loaded for user. Cannot proceed with analysis.
-```
-
-**Solutions:**
-1. **Initialize Default Agents**:
-   ```python
-   python -m scripts.populate_agent_templates
-   print('✅ Default agents initialized')
-   "
-   ```
-
-2. **Check Agent Templates in Database**:
-   ```python
-   python -c "
-   from src.db.init_db import session_factory
-   from src.db.schemas.models import AgentTemplate
-   session = session_factory()
-   templates = session.query(AgentTemplate).all()
-   print(f'Found {len(templates)} templates:')
-   for t in templates:
-       print(f'  - {t.template_name}: {t.is_active}')
-   session.close()
-   "
-   ```
-
-3. **Populate Templates from Config**:
-   ```bash
-   python scripts/populate_agent_templates.py
-   ```
-
-### 3. **API Key Issues**
-
-#### Problem: Missing API keys
-```
-❌ AuthenticationError: Invalid API key provided
-```
-
-**Solutions:**
-1. **Check Environment Variables**:
-   ```bash
-   # Verify API keys are set
-   echo $ANTHROPIC_API_KEY
-   echo $OPENAI_API_KEY
-   
-   # Or check .env file
-   cat .env | grep API_KEY
-   ```
-
-2. **Add Missing Keys**:
-   ```env
-   # Add to .env file
-   ANTHROPIC_API_KEY=sk-ant-api03-...
-   OPENAI_API_KEY=sk-...
-   ADMIN_API_KEY=your_admin_key_here
-   ```
-
-3. **Test API Key Validity**:
-   ```python
-   python -c "
-   import os
-   from anthropic import Anthropic
-   client = Anthropic(api_key=os.getenv('ANTHROPIC_API_KEY'))
-   try:
-       # Test call
-       response = client.messages.create(
-           model='claude-3-sonnet-20241022',
-           max_tokens=10,
-           messages=[{'role': 'user', 'content': 'Hello'}]
-       )
-       print('✅ Anthropic API key valid')
-   except Exception as e:
-       print(f'❌ Anthropic API key invalid: {e}')
-   "
-   ```
-
-## 🤖 Agent System Issues
-
-### 1. **Agent Not Found Errors**
-
-#### Problem: Specific agent not available
-```
-❌ KeyError: 'custom_agent' not found in loaded agents
-```
-
-**Solutions:**
-1. **Check Available Agents**:
-   ```python
-   python -c "
-   from src.agents.agents import load_user_enabled_templates_from_db
-   from src.db.init_db import session_factory
-   session = session_factory()
-   agents = load_user_enabled_templates_from_db('test_user', session)
-   print('Available agents:', list(agents.keys()))
-   session.close()
-   "
-   ```
-
-2. **Verify Agent Template Exists**:
-   ```python
-   python -c "
-   from src.db.init_db import session_factory
-   from src.db.schemas.models import AgentTemplate
-   session = session_factory()
-   agent = session.query(AgentTemplate).filter_by(template_name='custom_agent').first()
-   if agent:
-       print(f'Agent found: {agent.display_name}, Active: {agent.is_active}')
-   else:
-       print('Agent not found in database')
-   session.close()
-   "
-   ```
-
-3. **Add Missing Agent Template**:
-   ```python
-   # Add to agents_config.json or use database insertion
-   python scripts/populate_agent_templates.py
-   ```
-
-### 2. **Deep Analysis Failures**
-
-#### Problem: Deep analysis stops unexpectedly
-```
-❌ DeepAnalysisError: Agent execution failed at step 3
-```
-
-**Solutions:**
-1. **Check Agent Configuration**:
-   ```python
-   # Verify user has required agents enabled
-   python -c "
-   from src.agents.deep_agents import get_user_enabled_agent_names
-   from src.db.init_db import session_factory
-   session = session_factory()
-   agents = get_user_enabled_agent_names('test_user', session)
-   required = ['preprocessing_agent', 'statistical_analytics_agent', 'sk_learn_agent', 'data_viz_agent']
-   print('Required agents:', required)
-   print('Available agents:', agents)
-   print('Missing:', [a for a in required if a not in agents])
-   session.close()
-   "
-   ```
-
-2. **Increase Timeout Settings**:
-   ```python
-   # In deep_agents.py, increase timeout values
-   timeout = 300  # Increase from default
-   ```
-
-3. **Check Dataset Size**:
-   ```python
-   # Reduce dataset size for complex analysis
-   df_sample = df.sample(n=1000)  # Use sample for testing
-   ```
-
-## ⚡ Code Execution Problems
-
-### 1. **Code Execution Timeouts**
-
-#### Problem: Code execution takes too long
-```
-❌ TimeoutError: Code execution exceeded 120 seconds
-```
-
-**Solutions:**
-1. **Optimize Generated Code**:
-   - Use data sampling for large datasets
-   - Simplify analysis requirements
-   - Use sampling for large datasets
-
-2. **Check Resource Usage**:
-   ```python
-   import psutil
-   print(f"Memory usage: {psutil.virtual_memory().percent}%")
-   print(f"CPU usage: {psutil.cpu_percent()}%")
-   ```
-
-3. **Increase Timeout Settings**:
-   ```python
-   # In clean_and_store_code function
-   future.result(timeout=600)  # Increase timeout to 10 minutes
-   ```
-
-#### Problem: Import Errors in Generated Code
-```
-❌ ModuleNotFoundError: No module named 'some_library'
-```
-
-**Solutions:**
-1. **Check Available Libraries**:
-   ```python
-   # Available in execution environment:
-   import pandas as pd
-   import numpy as np
-   import plotly.express as px
-   import plotly.graph_objects as go
-   import sklearn
-   import statsmodels.api as sm
-   ```
-
-2. **Add Missing Dependencies**:
-   ```bash
-   pip install missing_library
-   ```
-
-3. **Update Execution Environment**:
-   ```python
-   # In clean_and_store_code function
-   exec_globals.update({
-       'new_library': __import__('new_library')
-   })
-   ```
-
-### 4. **Database Issues**
-
-#### Problem: Migration Errors
-```
-❌ alembic.util.exc.CommandError: Can't locate revision identified by 'xyz'
-```
-
-**Solutions:**
-1. **Reset Migration History**:
-   ```bash
-   # Delete migration files (except __init__.py)
-   rm migrations/versions/*.py
-   
-   # Create new initial migration
-   alembic revision --autogenerate -m "initial migration"
-   alembic upgrade head
-   ```
-
-2. **Force Migration**:
-   ```bash
-   # Mark current state as up-to-date
-   alembic stamp head
-   ```
-
-3. **Recreate Database**:
-   ```bash
-   # For SQLite (development)
-   rm auto_analyst.db
-   python -c "from src.db.init_db import init_db; init_db()"
-   ```
-
-#### Problem: Constraint Violations
-```
-❌ IntegrityError: UNIQUE constraint failed
-```
-
-**Solutions:**
-1. **Check Existing Records**:
-   ```python
-   from src.db.init_db import session_factory
-   from src.db.schemas.models import AgentTemplate
-   
-   session = session_factory()
-   templates = session.query(AgentTemplate).all()
-   for t in templates:
-       print(f"{t.template_name}: {t.template_id}")
-   session.close()
-   ```
-
-2. **Clean Duplicate Data**:
-   ```bash
-   python -c "
-   from src.db.init_db import session_factory
-   from src.db.schemas.models import AgentTemplate
-   session = session_factory()
-   # Remove duplicates based on template_name
-   seen = set()
-   for template in session.query(AgentTemplate).all():
-       if template.template_name in seen:
-           session.delete(template)
-       else:
-           seen.add(template.template_name)
-   session.commit()
-   session.close()
-   "
-   ```
-
-### 5. **Authentication and Authorization Issues**
-
-#### Problem: Unauthorized Access
-```
-❌ 401 Unauthorized: Invalid session
-```
-
-**Solutions:**
-1. **Check Session ID**:
-   ```python
-   # Ensure session_id is provided in request
-   headers = {"X-Session-ID": "your_session_id"}
-   # Or as query parameter: ?session_id=your_session_id
-   ```
-
-2. **Create Valid Session**:
-   ```bash
-   curl -X POST "http://localhost:8000/session_info" \
-        -H "Content-Type: application/json"
-   ```
-
-3. **Verify Admin API Key**:
-   ```bash
-   curl -X GET "http://localhost:8000/analytics/usage" \
-        -H "X-API-Key: your_admin_key"
-   ```
-
-### 6. **Performance Issues**
-
-#### Problem: Slow Response Times
-```
-⚠️ Request taking longer than expected
-```
-
-**Solutions:**
-1. **Enable Database Connection Pooling**:
-   ```python
-   # In init_db.py
-   engine = create_engine(
-       DATABASE_URL,
-       poolclass=QueuePool,
-       pool_size=10,
-       max_overflow=20
-   )
-   ```
-
-2. **Optimize Database Queries**:
-   ```python
-   # Use eager loading for relationships
-   session.query(User).options(joinedload(User.chats)).all()
-   ```
-
-3. **Add Response Caching**:
-   ```python
-   # Use local caching for expensive operations
-   @lru_cache(maxsize=100)
-   def expensive_operation(data):
-       return result
-   ```
-
-#### Problem: Memory Usage High
-```
-⚠️ Memory usage above 80%
-```
-
-**Solutions:**
-1. **Optimize DataFrame Operations**:
-   ```python
-   # Use chunking for large datasets
-   for chunk in pd.read_csv('file.csv', chunksize=1000):
-       process_chunk(chunk)
-   ```
-
-2. **Clear Unused Variables**:
-   ```python
-   # In code execution
-   del large_dataframe
-   import gc
-   gc.collect()
-   ```
-
-3. **Monitor Memory Usage**:
-   ```python
-   import psutil
-   import logging
-   
-   memory_percent = psutil.virtual_memory().percent
-   if memory_percent > 80:
-       logging.warning(f"High memory usage: {memory_percent}%")
-   ```
-
-## 🔧 Debugging Tools and Commands
-
-### Health Check Commands
-
-```bash
-# Test basic connectivity
-curl http://localhost:8000/health
-
-# Check database status
-python -c "
-from src.db.init_db import session_factory
-try:
-    session = session_factory()
-    session.execute('SELECT 1')
-    print('✅ Database connection OK')
-    session.close()
-except Exception as e:
-    print(f'❌ Database error: {e}')
-"
-
-# Verify agent templates
-python -c "
-from src.db.init_db import session_factory
-from src.db.schemas.models import AgentTemplate
-session = session_factory()
-count = session.query(AgentTemplate).count()
-print(f'Agent templates in database: {count}')
-session.close()
-"
-```
-
-### Performance Monitoring
-
-```python
-# Memory and CPU monitoring
-import psutil
-import time
-
-def monitor_system():
-    while True:
-        cpu = psutil.cpu_percent(interval=1)
-        memory = psutil.virtual_memory()
-        print(f"CPU: {cpu}% | Memory: {memory.percent}% | Available: {memory.available // 1024 // 1024}MB")
-        time.sleep(5)
-
-# Run monitoring
-monitor_system()
-```
-
-### Database Inspection
-
-```python
-# Inspect database tables
-from src.db.init_db import session_factory
-from src.db.schemas.models import *
-
-session = session_factory()
-
-# Count records in each table
-tables = [User, Chat, Message, AgentTemplate, UserTemplatePreference, DeepAnalysisReport]
-for table in tables:
-    count = session.query(table).count()
-    print(f"{table.__name__}: {count} records")
-
-session.close()
-```
-
-### Log Analysis
-
-```bash
-# View recent logs
-tail -f logs/app.log
-
-# Search for errors
-grep "ERROR" logs/app.log | tail -20
-
-# Search for specific issues
-grep -i "agent" logs/app.log | grep -i "error"
-```
-
-## 🚀 Performance Optimization Tips
-
-### Database Optimization
-
-1. **Use Indexes**: Ensure frequently queried columns have indexes
-2. **Query Optimization**: Use `joinedload` for relationships
-3. **Connection Pooling**: Configure appropriate pool sizes
-4. **Batch Operations**: Use bulk operations for multiple records
-
-### Agent Performance
-
-1. **Async Execution**: Use async patterns for concurrent operations
-2. **Result Caching**: Cache expensive computations
-3. **Memory Management**: Clean up large objects after use
-4. **Code Optimization**: Simplify generated code for better performance
-
-### System Monitoring
-
-1. **Resource Tracking**: Monitor CPU, memory, and disk usage
-2. **Error Monitoring**: Set up alerting for critical errors
-3. **Performance Metrics**: Track response times and throughput
-4. **Usage Analytics**: Monitor feature usage and optimization opportunities
-
-This troubleshooting guide covers the most common issues you'll encounter with the Auto-Analyst backend. For additional help, check the system logs and use the debugging tools provided. 

entrypoint_local.sh

@@ -1,175 +0,0 @@
-#!/bin/bash
-
-# Entrypoint script for Auto-Analyst backend
-# This script safely initializes the database and starts the application
-# SAFE for PostgreSQL/RDS - only modifies SQLite databases
-
-set -e  # Exit on any error
-
-echo "🚀 Starting Auto-Analyst Backend..."
-
-# Function to run safe database initialization
-init_production_database() {
-    echo "🔧 Running SAFE database initialization..."
-    
-    # Run the safe initialization script
-    python scripts/init_production_db.py
-    
-    # Don't fail if database initialization has issues - let app try to start
-    if [ $? -eq 0 ]; then
-        echo "✅ Database initialization completed successfully"
-    else
-        echo "⚠️  Database initialization had issues, but continuing..."
-        echo "📋 App will start but some features may not work properly"
-    fi
-}
-
-# Function to verify basic app imports work
-verify_app_imports() {
-    echo "🔍 Verifying application imports..."
-    python -c "
-try:
-    from app import app
-    print('✅ Main application imports successful')
-except Exception as e:
-    print(f'❌ Application import failed: {e}')
-    exit(1)
-" || {
-    echo "❌ Critical application import failure - cannot start"
-    exit 1
-}
-}
-
-# Function to verify database connectivity (non-failing)
-verify_database_connectivity() {
-    echo "🔗 Testing database connectivity..."
-    python -c "
-try:
-    from src.db.init_db import get_session, is_postgres_db
-    from src.db.schemas.models import AgentTemplate
-    
-    db_type = 'PostgreSQL/RDS' if is_postgres_db() else 'SQLite'
-    print(f'🗄️  Database type: {db_type}')
-    
-    session = get_session()
-    
-    # Try to query templates if table exists
-    try:
-        template_count = session.query(AgentTemplate).count()
-        print(f'✅ Database connected. Found {template_count} templates.')
-    except Exception as table_error:
-        print(f'⚠️  Database connected but template table issue: {table_error}')
-        print('📋 Template functionality may not work')
-    finally:
-        session.close()
-        
-except Exception as e:
-    print(f'⚠️  Database connectivity issue: {e}')
-    print('📋 App will start but database features may not work')
-"
-    # Don't exit on database connectivity issues - let app try to start
-}
-
-# Function to populate agents and templates for development (SQLite only)
-# Uses agents_config.json if available, falls back to legacy method
-populate_agents_templates() {
-    echo "🔧 Checking if agents/templates need to be populated..."
-    python -c "
-try:
-    from src.db.init_db import DATABASE_URL
-    from src.db.schemas.models import AgentTemplate
-    from src.db.init_db import session_factory
-    
-    # Check database type
-    if DATABASE_URL.startswith('sqlite'):
-        print('🔍 SQLite database detected - checking template population')
-        
-        session = session_factory()
-        try:
-            template_count = session.query(AgentTemplate).count()
-            
-            if template_count == 0:
-                print('📋 No templates found - populating agents and templates...')
-                session.close()
-                exit(1)  # Signal that population is needed
-            else:
-                print(f'✅ Found {template_count} templates - population not needed')
-                session.close()
-                exit(0)  # Signal that population is not needed
-        except Exception as e:
-            print(f'⚠️  Error checking templates: {e}')
-            print('📋 Will attempt to populate anyway')
-            session.close()
-            exit(1)  # Signal that population is needed
-    else:
-        print('🔍 PostgreSQL/RDS detected - skipping auto-population')
-        exit(0)  # Signal that population is not needed
-        
-except Exception as e:
-    print(f'❌ Error during template check: {e}')
-    exit(0)  # Don't fail startup, just skip population
-"
-    
-    # Check if population is needed (exit code 1 means yes)
-    if [ $? -eq 1 ]; then
-        echo "🚀 Running agent/template population for SQLite..."
-        
-        # Check if agents_config.json exists (try multiple locations)
-        if [ -f "agents_config.json" ] || [ -f "/app/agents_config.json" ] || [ -f "../agents_config.json" ]; then
-            echo "📖 Found agents_config.json - validating configuration..."
-            
-            # Validate configuration first
-            python scripts/populate_agent_templates.py validate
-            validation_result=$?
-            
-            if [ $validation_result -eq 0 ]; then
-                echo "✅ Configuration valid - proceeding with sync"
-                python scripts/populate_agent_templates.py sync
-            else
-                echo "⚠️  Configuration validation failed - attempting sync anyway"
-                python scripts/populate_agent_templates.py sync
-            fi
-        else
-            echo "⚠️  agents_config.json not found - trying legacy method"
-            python scripts/populate_agent_templates.py
-        fi
-        
-        if [ $? -eq 0 ]; then
-            echo "✅ Agent/template population completed successfully"
-        else
-            echo "⚠️  Agent/template population had issues, but continuing..."
-            echo "📋 You may need to populate templates manually"
-            echo "💡 Tip: Ensure agents_config.json exists in the backend directory"
-        fi
-    fi
-}
-
-# Check if we need to find agents_config.json from space root
-if [ ! -f "/app/agents_config.json" ]; then
-    echo "⚠️  agents_config.json not found in container - checking build issues"
-    echo "📁 Files in /app directory:"
-    ls -la /app/ | head -10
-else
-    echo "✅ agents_config.json found in container"
-fi
-
-# Main startup sequence
-echo "🔧 Initializing production environment..."
-
-# Verify critical imports first
-verify_app_imports
-
-# Initialize database safely (won't modify RDS)
-init_production_database
-
-# Test database connectivity (non-failing)
-verify_database_connectivity
-
-# Populate agents and templates for development (SQLite only)
-populate_agents_templates
-
-echo "🎯 Starting FastAPI application..."
-echo "🌐 Application will be available on port 7860"
-
-# Start the FastAPI application
-exec uvicorn app:app --host 0.0.0.0 --port 7860 

requirements.txt

@@ -1,31 +1,32 @@
-aiofiles==24.1.0
-beautifulsoup4==4.13.4
-chardet==5.2.0
-dspy==3.2.1
-litellm==1.82.3
+groq==0.18.0
+dspy==2.5.43
+dspy-ai==2.5.43
 email_validator==2.2.0
-fastapi==0.135.1
-fastapi-cli==0.0.7
+fastapi==0.111.1
 FastAPI-SQLAlchemy==0.2.1
-fastapi-sso==0.16.0
-groq==0.18.0
-gunicorn==23.0.0
-huggingface-hub==0.30.2
+gunicorn==22.0.0
 joblib==1.4.2
-Markdown==3.7
+litellm==1.53.7
+llama-index==0.12.14
+llama-index-agent-openai==0.4.2
+llama-index-embeddings-openai==0.3.1
+llama-index-indices-managed-llama-cloud==0.6.4
+llama-index-llms-openai==0.3.14
+llama-index-multi-modal-llms-openai==0.4.2
+llama-index-program-openai==0.3.1
+llama-index-question-gen-openai==0.3.0
 matplotlib==3.10.0
-matplotlib-inline==0.1.7
+multiprocess==0.70.16
 numpy==2.2.2
 openpyxl==3.1.2
 xlrd==2.0.1
-openai==2.28.0
+openai==1.60.1
 pandas==2.2.3
-polars==1.31.0
 pillow==11.1.0
 plotly==5.24.1
-psycopg2==2.9.10
-python-dateutil==2.9.0.post0
+pypdf==5.2.0
 python-dotenv==1.0.1
+regex==2024.11.6
 requests==2.32.3
 scikit-learn==1.6.1
 scipy==1.15.1
@@ -37,16 +38,10 @@ tabulate==0.9.0
 threadpoolctl==3.5.0
 tiktoken==0.8.0
 tokenizers==0.21.0
-tqdm==4.67.1
-urllib3==2.4.0
-uvicorn==0.34.0
-websockets>=13.1.0
+uvicorn==0.22.0
+websockets==14.2
+Werkzeug==3.1.3
 wheel==0.45.1
-xgboost-cpu==3.0.2
-bokeh==3.7.3
-pymc==5.23.0
-lightgbm==4.6.0
-arviz==0.21.0
-optuna==4.3.0
-litellm[proxy]
-duckdb==1.3.2
+gunicorn
+psycopg2
+chardet

scripts/create_test_user.py

@@ -0,0 +1,45 @@
+import sys
+import os
+
+# Add parent directory to path so we can import modules
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from init_db import User, session_factory
+
+def create_test_users():
+    """Create test users for development"""
+    session = session_factory()
+    
+    try:
+        # Create a few test users
+        test_users = [
+            {"username": "testuser1", "email": "user1@example.com"},
+            {"username": "testuser2", "email": "user2@example.com"},
+            {"username": "admin", "email": "admin@example.com"},
+            {"username": "demo", "email": "demo@example.com"},
+            {"username": "guest", "email": "guest@example.com"},
+        ]
+        
+        for user_data in test_users:
+            # Check if user already exists
+            existing = session.query(User).filter(User.email == user_data["email"]).first()
+            if not existing:
+                user = User(**user_data)
+                session.add(user)
+                print(f"Created user: {user_data['username']}")
+            else:
+                print(f"User {user_data['username']} already exists")
+        
+        session.commit()
+        print("Test users created successfully")
+        
+    except Exception as e:
+        session.rollback()
+        print(f"Error creating test users: {e}")
+    
+    finally:
+        session.close()
+
+if __name__ == "__main__":
+    create_test_users()
+    print("Done!") 

scripts/format_response.py

@@ -8,7 +8,7 @@ import logging
 from src.utils.logger import Logger
 import textwrap
 
-logger = Logger(__name__, level=logging.INFO, see_time=False, console_log=False)
+logger = Logger(__name__, level="INFO", see_time=False, console_log=False)
 
 @contextlib.contextmanager
 def stdoutIO(stdout=None):
@@ -44,33 +44,20 @@ API_KEY_PATTERNS = [
 ]
 
 # Network request patterns
-NETWORK_REQUEST_PATTERNS = re.compile(r"(requests\.|urllib\.|http\.client|httpx\.|socket\.connect\()")
+NETWORK_REQUEST_PATTERNS = re.compile(r"(requests\.|urllib\.|http\.|\.post\(|\.get\(|\.connect\()")
 
-# DataFrame creation with hardcoded data - block only this specific pattern
-
-def check_security_concerns(code_str, dataset_names):
+def check_security_concerns(code_str):
     """Check code for security concerns and return info about what was found"""
     security_concerns = {
         "has_concern": False,
+        "messages": [],
         "blocked_imports": False,
         "blocked_dynamic_imports": False,
         "blocked_env_access": False,
         "blocked_file_access": False,
         "blocked_api_keys": False,
-        "blocked_network": False,
-        "blocked_dataframe_invention": False,  # Add this new field
-        "messages": []
+        "blocked_network": False
     }
-
-    dataset_names_pattern = "|".join(re.escape(name) for name in dataset_names)
-    DATAFRAME_INVENTION_PATTERN = re.compile(
-                rf"({dataset_names_pattern})\s*=\s*pd\.DataFrame\s*\(\s*\{{\s*[^}}]*\}}", 
-                re.MULTILINE
-            )
-    if DATAFRAME_INVENTION_PATTERN.search(code_str):
-        security_concerns["has_concern"] = True
-        security_concerns["blocked_dataframe_invention"] = True
-        security_concerns["messages"].append(f"DataFrame creation blocked for dataset variables: {', '.join(dataset_names)}")
     
     # Check for sensitive imports
     if IMPORT_PATTERN.search(code_str) or FROM_IMPORT_PATTERN.search(code_str):
@@ -110,19 +97,11 @@ def check_security_concerns(code_str, dataset_names):
         security_concerns["blocked_network"] = True
         security_concerns["messages"].append("Network requests blocked")
     
-    
-    
     return security_concerns
 
-def clean_code_for_security(code_str, security_concerns, dataset_names):
+def clean_code_for_security(code_str, security_concerns):
     """Apply security modifications to the code based on detected concerns"""
-
     modified_code = code_str
-    dataset_names_pattern = "|".join(re.escape(name) for name in dataset_names)
-    DATAFRAME_INVENTION_PATTERN = re.compile(
-                rf"({dataset_names_pattern})\s*=\s*pd\.DataFrame\s*\(\s*\{{\s*[^}}]*\}}", 
-                re.MULTILINE
-            )
     
     # Block sensitive imports if needed
     if security_concerns["blocked_imports"]:
@@ -150,13 +129,6 @@ def clean_code_for_security(code_str, security_concerns, dataset_names):
     if security_concerns["blocked_network"]:
         modified_code = NETWORK_REQUEST_PATTERNS.sub(r'"BLOCKED_NETWORK_REQUEST"', modified_code)
     
-    # Block DataFrame creation with hardcoded data if detected
-    if security_concerns["blocked_dataframe_invention"]:
-        modified_code = DATAFRAME_INVENTION_PATTERN.sub(
-            r"# BLOCKED_DATAFRAME_INVENTION: \g<0>", 
-            modified_code
-        )
-    
     # Add warning banner if needed
     if security_concerns["has_concern"]:
         security_message = "⚠️ SECURITY WARNING: " + ". ".join(security_concerns["messages"]) + "."
@@ -164,104 +136,6 @@ def clean_code_for_security(code_str, security_concerns, dataset_names):
     
     return modified_code
     
-def format_correlation_output(text):
-    """Format correlation matrix output for better readability"""
-    lines = text.split('\n')
-    formatted_lines = []
-    
-    for line in lines:
-        # Skip empty lines at the beginning
-        if not line.strip() and not formatted_lines:
-            continue
-        
-        if not line.strip():
-            formatted_lines.append(line)
-            continue
-        
-        # Check if this line contains correlation values or variable names
-        stripped_line = line.strip()
-        parts = stripped_line.split()
-        
-        if len(parts) > 1:
-            # Check if this is a header line with variable names
-            if all(part.replace('_', '').replace('-', '').isalpha() for part in parts):
-                # This is a header row with variable names
-                formatted_header = f"{'':12}"  # Empty first column for row labels
-                for part in parts:
-                    formatted_header += f"{part:>12}"
-                formatted_lines.append(formatted_header)
-            elif any(char.isdigit() for char in stripped_line) and ('.' in stripped_line or '-' in stripped_line):
-                # This looks like a correlation line with numbers
-                row_name = parts[0] if parts else ""
-                values = parts[1:] if len(parts) > 1 else []
-                
-                formatted_row = f"{row_name:<12}"
-                for value in values:
-                    try:
-                        val = float(value)
-                        formatted_row += f"{val:>12.3f}"
-                    except ValueError:
-                        formatted_row += f"{value:>12}"
-                
-                formatted_lines.append(formatted_row)
-            else:
-                # Other lines (like titles)
-                formatted_lines.append(line)
-        else:
-            formatted_lines.append(line)
-    
-    return '\n'.join(formatted_lines)
-
-def format_summary_stats(text):
-    """Format summary statistics for better readability"""
-    lines = text.split('\n')
-    formatted_lines = []
-    
-    for line in lines:
-        if not line.strip():
-            formatted_lines.append(line)
-            continue
-        
-        # Check if this is a header line with statistical terms only (missing first column)
-        stripped_line = line.strip()
-        if any(stat in stripped_line.lower() for stat in ['count', 'mean', 'median', 'std', 'min', 'max', '25%', '50%', '75%']):
-            parts = stripped_line.split()
-            # Check if this is a header row (starts with statistical terms)
-            if parts and parts[0].lower() in ['count', 'mean', 'median', 'std', 'min', 'max', '25%', '50%', '75%']:
-                # This is a header row - add proper spacing
-                formatted_header = f"{'':12}"  # Empty first column for row labels
-                for part in parts:
-                    formatted_header += f"{part:>15}"
-                formatted_lines.append(formatted_header)
-            else:
-                # This is a data row - format normally
-                row_name = parts[0] if parts else ""
-                values = parts[1:] if len(parts) > 1 else []
-                
-                formatted_row = f"{row_name:<12}"
-                for value in values:
-                    try:
-                        if '.' in value or 'e' in value.lower():
-                            val = float(value)
-                            if abs(val) >= 1000000:
-                                formatted_row += f"{val:>15.2e}"
-                            elif abs(val) >= 1:
-                                formatted_row += f"{val:>15.2f}"
-                            else:
-                                formatted_row += f"{val:>15.6f}"
-                        else:
-                            val = int(value)
-                            formatted_row += f"{val:>15}"
-                    except ValueError:
-                        formatted_row += f"{value:>15}"
-                
-                formatted_lines.append(formatted_row)
-        else:
-            # Other lines (titles, etc.) - keep as is
-            formatted_lines.append(line)
-    
-    return '\n'.join(formatted_lines)
-    
 def clean_print_statements(code_block):
     """
     This function cleans up any `print()` statements that might contain unwanted `\n` characters.
@@ -304,14 +178,6 @@ def format_code_block(code_str):
     return f'\n{code_clean}\n'
 
 def format_code_backticked_block(code_str):
-    # Add None check at the beginning
-    if code_str is None:
-        return "```python\n# No code available\n```"  # Return empty code block instead of None
-    
-    # Add type check to ensure it's a string
-    if not isinstance(code_str, str):
-        return f"```python\n# Invalid code type: {type(code_str)}\n```"
-    
     code_clean = re.sub(r'^```python\n?', '', code_str, flags=re.MULTILINE)
     code_clean = re.sub(r'\n```$', '', code_clean)
     # Only match assignments at top level (not indented)
@@ -320,7 +186,6 @@ def format_code_backticked_block(code_str):
   
     # Remove reading the csv file if it's already in the context
     modified_code = re.sub(r"df\s*=\s*pd\.read_csv\([\"\'].*?[\"\']\).*?(\n|$)", '', code_clean)
-    modified_code = re.sub(r'^(\s*)(df\s*=.*)$', r'\1# \2', code_clean, flags=re.MULTILINE)
     
     # Only match assignments at top level (not indented)
     # 1. Remove 'df = pd.DataFrame()' if it's at the top level
@@ -344,7 +209,8 @@ def format_code_backticked_block(code_str):
     return f'```python\n{code_clean}\n```'
 
     
-def execute_code_from_markdown(code_str, datasets=None):
+# In format_response.py, modify the execute_code function:
+def execute_code_from_markdown(code_str, dataframe=None):
     import pandas as pd
     import plotly.express as px
     import plotly
@@ -355,169 +221,13 @@ def execute_code_from_markdown(code_str, datasets=None):
     import re
     import traceback
     import sys
-    from io import StringIO, BytesIO
-    import base64
+    from io import StringIO
 
-    context_names = list(datasets.keys())
     # Check for security concerns in the code
-    security_concerns = check_security_concerns(code_str, context_names)
+    security_concerns = check_security_concerns(code_str)
     
     # Apply security modifications to the code
-    modified_code = clean_code_for_security(code_str, security_concerns, context_names)
-    
-    # Enhanced print function that detects and formats tabular data
-    captured_outputs = []
-    original_print = print
-    
-    # Set pandas display options for full table display
-    pd.set_option('display.max_columns', None)
-    pd.set_option('display.max_rows', 20)  # Limit to 20 rows instead of unlimited
-    pd.set_option('display.width', None)
-    pd.set_option('display.max_colwidth', 50)
-    pd.set_option('display.expand_frame_repr', False)
-    
-
-    
-    def enhanced_print(*args, **kwargs):
-        # Convert all args to strings
-        str_args = [str(arg) for arg in args]
-        output_text = kwargs.get('sep', ' ').join(str_args)
-        
-        # Special case for DataFrames - use pipe delimiter and clean format
-        if isinstance(args[0], pd.DataFrame) and len(args) == 1:
-            # Format DataFrame with pipe delimiter using to_csv for reliable column separation
-            df = args[0]
-            
-            # Use StringIO to capture CSV output with pipe delimiter
-            from io import StringIO
-            csv_buffer = StringIO()
-            
-            # Export to CSV with pipe delimiter, preserving index
-            df.to_csv(csv_buffer, sep='|', index=True, float_format='%.6g')
-            csv_output = csv_buffer.getvalue()
-            
-            # Clean up the CSV output - remove quotes and extra formatting
-            lines = csv_output.strip().split('\n')
-            cleaned_lines = []
-            
-            for line in lines:
-                # Remove any quotes that might have been added by to_csv
-                clean_line = line.replace('"', '')
-                # Split by pipe, strip whitespace from each part, then rejoin
-                parts = [part.strip() for part in clean_line.split('|')]
-                cleaned_lines.append(' | '.join(parts))
-            
-            output_text = '\n'.join(cleaned_lines)
-            captured_outputs.append(f"<TABLE_START>\n{output_text}\n<TABLE_END>")
-            original_print(output_text)
-            return
-        
-        # Detect if this looks like tabular data (generic approach)
-        is_table = False
-        
-        # Check for table patterns:
-        # 1. Multiple lines with consistent spacing
-        lines = output_text.split('\n')
-        if len(lines) > 2:
-            # Count lines that look like they have multiple columns (2+ spaces between words)
-            multi_column_lines = sum(1 for line in lines if len(line.split()) > 1 and '  ' in line)
-            if multi_column_lines >= 2:  # At least 2 lines with multiple columns
-                is_table = True
-            
-            # Check for pandas DataFrame patterns like index with column names
-            if any(re.search(r'^\s*\d+\s+', line) for line in lines):
-                # Look for lines starting with an index number followed by spaces
-                is_table = True
-                
-            # Look for table-like structured output with multiple rows of similar format
-            if len(lines) >= 3:
-                # Sample a few lines to check for consistent structure
-                sample_lines = [lines[i] for i in range(min(len(lines), 5)) if i < len(lines) and lines[i].strip()]
-                
-                # Check for consistent whitespace patterns
-                if len(sample_lines) >= 2:
-                    # Get positions of whitespace groups in first line
-                    whitespace_positions = []
-                    for i, line in enumerate(sample_lines):
-                        if not line.strip():
-                            continue
-                        positions = [m.start() for m in re.finditer(r'\s{2,}', line)]
-                        if i == 0:
-                            whitespace_positions = positions
-                        elif len(positions) == len(whitespace_positions):
-                            # Check if whitespace positions are roughly the same
-                            is_similar = all(abs(pos - whitespace_positions[j]) <= 3 
-                                            for j, pos in enumerate(positions) 
-                                            if j < len(whitespace_positions))
-                            if is_similar:
-                                is_table = True
-        
-        # 2. Contains common table indicators
-        if any(indicator in output_text.lower() for indicator in [
-            'count', 'mean', 'std', 'min', 'max', '25%', '50%', '75%',  # Summary stats
-            'correlation', 'corr',  # Correlation tables
-            'coefficient', 'r-squared', 'p-value',  # Regression tables
-        ]):
-            is_table = True
-        
-        # 3. Has many decimal numbers (likely a data table)
-        if output_text.count('.') > 5 and len(lines) > 2:
-            is_table = True
-            
-        # If we have detected a table, convert space-delimited to pipe-delimited format
-        if is_table:
-            # Convert the table to pipe-delimited format for better parsing in frontend
-            formatted_lines = []
-            for line in lines:
-                if not line.strip():
-                    formatted_lines.append(line)  # Keep empty lines
-                    continue
-                
-                # Split by multiple spaces and join with pipe delimiter
-                parts = re.split(r'\s{2,}', line.strip())
-                if parts:
-                    formatted_lines.append(" | ".join(parts))
-                else:
-                    formatted_lines.append(line)
-            
-            # Use the pipe-delimited format
-            output_text = "\n".join(formatted_lines)
-            
-            # Format and mark the output for table processing in UI
-            captured_outputs.append(f"<TABLE_START>\n{output_text}\n<TABLE_END>")
-        else:
-            captured_outputs.append(output_text)
-        
-        # Also use original print for stdout capture
-        original_print(*args, **kwargs)
-
-    # Custom matplotlib capture function
-    def capture_matplotlib_chart():
-        """Capture current matplotlib figure as base64 encoded image"""
-        try:
-            fig = plt.gcf()  # Get current figure
-            if fig.get_axes():  # Check if figure has any plots
-                buffer = BytesIO()
-                fig.savefig(buffer, format='png', dpi=150, bbox_inches='tight', 
-                           facecolor='white', edgecolor='none')
-                buffer.seek(0)
-                img_base64 = base64.b64encode(buffer.getvalue()).decode('utf-8')
-                buffer.close()
-                plt.close(fig)  # Close the figure to free memory
-                return img_base64
-            return None
-        except Exception:
-            return None
-
-    # Store original plt.show function
-    original_plt_show = plt.show
-    
-    def custom_plt_show(*args, **kwargs):
-        """Custom plt.show that captures the chart instead of displaying it"""
-        img_base64 = capture_matplotlib_chart()
-        if img_base64:
-            matplotlib_outputs.append(img_base64)
-        # Don't call original show to prevent display
+    modified_code = clean_code_for_security(code_str, security_concerns)
     
     context = {
         'pd': pd,
@@ -529,18 +239,8 @@ def execute_code_from_markdown(code_str, datasets=None):
         '__import__': __import__,
         'sns': sns,
         'np': np,
-        'json_outputs': [],  # List to store multiple Plotly JSON outputs
-        'matplotlib_outputs': [],  # List to store matplotlib chart images as base64
-        'print': enhanced_print  # Replace print with our enhanced version
+        'json_outputs': []  # List to store multiple Plotly JSON outputs
     }
-    
-    # Add matplotlib_outputs to local scope for the custom show function
-    matplotlib_outputs = context['matplotlib_outputs']
-    
-    # Replace plt.show with our custom function
-    plt.show = custom_plt_show
-    
-    
 
     # Modify code to store multiple JSON outputs
     modified_code = re.sub(
@@ -553,7 +253,8 @@ def execute_code_from_markdown(code_str, datasets=None):
         r'(\w*_?)fig(\w*)\.to_html\(.*?\)',
         r'json_outputs.append(plotly.io.to_json(\1fig\2, pretty=True))',
         modified_code
-    )    
+    )
+    
     # Remove reading the csv file if it's already in the context
     modified_code = re.sub(r"df\s*=\s*pd\.read_csv\([\"\'].*?[\"\']\).*?(\n|$)", '', modified_code)
     
@@ -565,70 +266,27 @@ def execute_code_from_markdown(code_str, datasets=None):
         modified_code,
         flags=re.MULTILINE
     )
-    
-
-    # Custom display function for DataFrames to show head + tail for large datasets
-    original_repr = pd.DataFrame.__repr__
-    
-    def custom_df_repr(self):
-        if len(self) > 15:
-            # For large DataFrames, show first 10 and last 5 rows
-            head_part = self.head(10)
-            tail_part = self.tail(5)
-            
-            head_str = head_part.__repr__()
-            tail_str = tail_part.__repr__()
-            
-            # Extract just the data rows (skip the header from tail)
-            tail_lines = tail_str.split('\n')
-            tail_data = '\n'.join(tail_lines[1:])  # Skip header line
-            
-            return f"{head_str}\n...\n{tail_data}"
-        else:
-            return original_repr(self)
-    
-    # Apply custom representation temporarily
-    pd.DataFrame.__repr__ = custom_df_repr
 
     # If a dataframe is provided, add it to the context
-    for dataset_name, dataset_df in datasets.items():
-        if dataset_df is not None:
-            context[dataset_name] = dataset_df
-            logger.log_message(f"Added dataset '{dataset_name}' to execution context", level=logging.DEBUG)
-
+    if dataframe is not None:
+        context['df'] = dataframe
 
     # remove pd.read_csv() if it's already in the context
     modified_code = re.sub(r"pd\.read_csv\(\s*[\"\'].*?[\"\']\s*\)", '', modified_code)
 
     # Remove sample dataframe lines with multiple array values
     modified_code = re.sub(r"^# Sample DataFrames?.*?(\n|$)", '', modified_code, flags=re.MULTILINE | re.IGNORECASE)
-    
-    # Replace plt.savefig() calls with plt.show() to ensure plots are displayed
-    modified_code = re.sub(r'plt\.savefig\([^)]*\)', 'plt.show()', modified_code)
-    
-    # Instead of removing plt.show(), keep them - they'll be handled by our custom function
-    # Also handle seaborn plots that might not have explicit plt.show()
-    # Add plt.show() after seaborn plot functions if not already present
-    seaborn_plot_functions = [
-        'sns.scatterplot', 'sns.lineplot', 'sns.barplot', 'sns.boxplot', 'sns.violinplot',
-        'sns.stripplot', 'sns.swarmplot', 'sns.pointplot', 'sns.catplot', 'sns.relplot',
-        'sns.displot', 'sns.histplot', 'sns.kdeplot', 'sns.ecdfplot', 'sns.rugplot',
-        'sns.distplot', 'sns.jointplot', 'sns.pairplot', 'sns.FacetGrid', 'sns.PairGrid',
-        'sns.heatmap', 'sns.clustermap', 'sns.regplot', 'sns.lmplot', 'sns.residplot'
-    ]
-    
-    # Add automatic plt.show() after seaborn plots if not already present
-    for func in seaborn_plot_functions:
-        pattern = rf'({re.escape(func)}\([^)]*\)(?:\.[^(]*\([^)]*\))*)'
-        def add_show(match):
-            plot_call = match.group(1)
-            # Check if the next non-empty line already has plt.show()
-            return f'{plot_call}\nplt.show()'
         
-        modified_code = re.sub(pattern, add_show, modified_code)
+    # Remove plt.show() statements
+    modified_code = re.sub(r"plt\.show\(\).*?(\n|$)", '', modified_code)
     
     # Only add df = pd.read_csv() if no dataframe was provided and the code contains pd.read_csv
-
+    if dataframe is None and 'pd.read_csv' not in modified_code:
+        modified_code = re.sub(
+            r'import pandas as pd',
+            r'import pandas as pd\n\n# Read Housing.csv\ndf = pd.read_csv("Housing.csv")',
+            modified_code
+        )
 
     # Identify code blocks by comments
     code_blocks = []
@@ -656,42 +314,11 @@ def execute_code_from_markdown(code_str, datasets=None):
     all_outputs = []
     for block_name, block_code in code_blocks:
         try:
-            # Clear captured outputs for each block
-            captured_outputs.clear()
-            
-            # Fix indentation issues before execution
-            try:
-                block_code = textwrap.dedent(block_code)
-            except Exception as dedent_error:
-                logger.log_message(f"Failed to dedent code block '{block_name}': {str(dedent_error)}", level=logging.WARNING)
-            
             with stdoutIO() as s:
                 exec(block_code, context)  # Execute the block
-            
-            # Get both stdout and our enhanced captured outputs
-            stdout_output = s.getvalue()
-            
-            # Combine outputs, preferring our enhanced format when available
-            if captured_outputs:
-                combined_output = '\n'.join(captured_outputs)
-            else:
-                combined_output = stdout_output
-            
-            all_outputs.append((block_name, combined_output, None))  # None means no error
+            output = s.getvalue()
+            all_outputs.append((block_name, output, None))  # None means no error
         except Exception as e:
-            # Reset pandas options in case of error
-            pd.reset_option('display.max_columns')
-            pd.reset_option('display.max_rows')
-            pd.reset_option('display.width')
-            pd.reset_option('display.max_colwidth')
-            pd.reset_option('display.expand_frame_repr')
-            
-            # Restore original DataFrame representation in case of error
-            pd.DataFrame.__repr__ = original_repr
-            
-            # Restore original plt.show
-            plt.show = original_plt_show
-            
             error_traceback = traceback.format_exc()
             
             # Extract error message and error type
@@ -838,23 +465,9 @@ def execute_code_from_markdown(code_str, datasets=None):
             
             all_outputs.append((block_name, None, formatted_error))
     
-    # Reset pandas options after execution
-    pd.reset_option('display.max_columns')
-    pd.reset_option('display.max_rows')
-    pd.reset_option('display.width')
-    pd.reset_option('display.max_colwidth')
-    pd.reset_option('display.expand_frame_repr')
-    
-    # Restore original DataFrame representation
-    pd.DataFrame.__repr__ = original_repr
-    
-    # Restore original plt.show
-    plt.show = original_plt_show
-    
     # Compile all outputs and errors
     output_text = ""
     json_outputs = context.get('json_outputs', [])
-    matplotlib_outputs = context.get('matplotlib_outputs', [])
     error_found = False
     
     for block_name, output, error in all_outputs:
@@ -865,9 +478,9 @@ def execute_code_from_markdown(code_str, datasets=None):
             output_text += f"\n\n=== OUTPUT FROM {block_name.upper()}_AGENT ===\n{output}\n"
     
     if error_found:
-        return output_text, [], []
+        return output_text, []
     else:
-        return output_text, json_outputs, matplotlib_outputs
+        return output_text, json_outputs
     
     
 def format_plan_instructions(plan_instructions):
@@ -875,127 +488,54 @@ def format_plan_instructions(plan_instructions):
     Format any plan instructions (JSON string or dict) into markdown sections per agent.
     """
     # Parse input into a dict
-
-    if "basic_qa_agent" in str(plan_instructions):
-        return "**Non-Data Request**: Please ask a data related query, don't waste credits!"
-
-
     try:
         if isinstance(plan_instructions, str):
-            try:
-                instructions = json.loads(plan_instructions)
-            except json.JSONDecodeError as e:
-                # Try to clean the string if it's not valid JSON
-                cleaned_str = plan_instructions.strip()
-                if cleaned_str.startswith("'") and cleaned_str.endswith("'"):
-                    cleaned_str = cleaned_str[1:-1]
-                try:
-                    instructions = json.loads(cleaned_str)
-                except json.JSONDecodeError:
-                    raise ValueError(f"Invalid JSON format in plan instructions: {str(e)}")
+            instructions = json.loads(plan_instructions)
         elif isinstance(plan_instructions, dict):
             instructions = plan_instructions
         else:
-            raise TypeError(f"Unsupported plan instructions type: {type(plan_instructions)}")
-    except Exception as e:
-        raise ValueError(f"Error processing plan instructions: {str(e)} + {dspy.settings.lm} ")
+            return f"Unsupported plan instructions type: {type(plan_instructions)}"
+    except json.JSONDecodeError:
+        return f"Invalid plan instructions: {plan_instructions}"
     # logger.log_message(f"Plan instructions: {instructions}", level=logging.INFO)
 
-
-
     markdown_lines = []
     for agent, content in instructions.items():
-        if agent != 'basic_qa_agent':
-            agent_title = agent.replace('_', ' ').title()
-            markdown_lines.append(f"#### {agent_title}")
-            if isinstance(content, dict):
-                # Handle 'create' key
-                create_vals = content.get('create', [])
-                if create_vals:
-                    markdown_lines.append(f"- **Create**:")
-                    for item in create_vals:
-                        markdown_lines.append(f"  - {item}")
-                else:
-                    markdown_lines.append(f"- **Create**: None")
-    
-                # Handle 'use' key
-                use_vals = content.get('use', [])
-                if use_vals:
-                    markdown_lines.append(f"- **Use**:")
-                    for item in use_vals:
-                        markdown_lines.append(f"  - {item}")
-                else:
-                    markdown_lines.append(f"- **Use**: None")
-    
-                # Handle 'instruction' key
-                instr = content.get('instruction')
-                if isinstance(instr, str) and instr:
-                    markdown_lines.append(f"- **Instruction**: {instr}")
-                else:
-                    markdown_lines.append(f"- **Instruction**: None")
+        agent_title = agent.replace('_', ' ').title()
+        markdown_lines.append(f"#### {agent_title}")
+        if isinstance(content, dict):
+            # Handle 'create' key
+            create_vals = content.get('create', [])
+            if create_vals:
+                markdown_lines.append(f"- **Create**:")
+                for item in create_vals:
+                    markdown_lines.append(f"  - {item}")
             else:
-                # Fallback for non-dict content
-                markdown_lines.append(f"- {content}")
-            markdown_lines.append("")  # blank line between agents
-        else:
-            markdown_lines.append(f"**Non-Data Request**: {content.get('instruction')}")
-
-    return "\n".join(markdown_lines).strip()
-    
-    # Return empty string if no complexity found
-    return ""
+                markdown_lines.append(f"- **Create**: None")
+
+            # Handle 'use' key
+            use_vals = content.get('use', [])
+            if use_vals:
+                markdown_lines.append(f"- **Use**:")
+                for item in use_vals:
+                    markdown_lines.append(f"  - {item}")
+            else:
+                markdown_lines.append(f"- **Use**: None")
 
-def format_complexity(instructions):
-    markdown_lines = []
-    complexity = None  # Initialize complexity to avoid UnboundLocalError
-    
-    # Extract complexity from various possible locations in the structure
-    if isinstance(instructions, dict):
-        # Case 1: Direct complexity field
-        if 'complexity' in instructions:
-            complexity = instructions['complexity']
-        # Case 2: Complexity in 'plan' object
-        elif 'plan' in instructions and isinstance(instructions['plan'], dict):
-            if 'complexity' in instructions['plan']:
-                complexity = instructions['plan']['complexity']
+            # Handle 'instruction' key
+            instr = content.get('instruction')
+            if isinstance(instr, str) and instr:
+                markdown_lines.append(f"- **Instruction**: {instr}")
+            else:
+                markdown_lines.append(f"- **Instruction**: None")
         else:
-            complexity = "unrelated"
-            
-        # Override for basic_qa_agent cases
-        if 'plan' in instructions and isinstance(instructions['plan'], str) and "basic_qa_agent" in instructions['plan']:
-            complexity = "unrelated"
-    else:
-        # If instructions is not a dict, default to unrelated
-        complexity = "unrelated"
-    
-    if complexity:
-        # Pink color scheme variations
-        color_map = {
-            "unrelated": "#FFB6B6",  # Light pink
-            "basic": "#FF9E9E",      # Medium pink
-            "intermediate": "#FF7F7F", # Main pink
-            "advanced": "#FF5F5F"    # Dark pink
-        }
-        
-        indicator_map = {
-            "unrelated": "○",
-            "basic": "●",
-            "intermediate": "●●",
-            "advanced": "●●●"
-        }
-        
-        color = color_map.get(complexity.lower(), "#FFB6B6")  # Default to light pink
-        indicator = indicator_map.get(complexity.lower(), "○")
-        
-        # Slightly larger display with pink styling
-        markdown_lines.append(f"<div style='color: {color}; border: 2px solid {color}; padding: 2px 8px; border-radius: 12px; display: inline-block; font-size: 14.4px;'>{indicator} {complexity}</div>\n")
+            # Fallback for non-dict content
+            markdown_lines.append(f"- {content}")
+        markdown_lines.append("")  # blank line between agents
 
-        return "\n".join(markdown_lines).strip()
+    return "\n".join(markdown_lines).strip()
     
-    # Return empty string if no complexity found
-    return ""
-
-def format_response_to_markdown(api_response, agent_name = None, datasets=None):
+def format_response_to_markdown(api_response, agent_name = None, dataframe=None):
     try:
         markdown = []
         # logger.log_message(f"API response for {agent_name} at {time.strftime('%Y-%m-%d %H:%M:%S')}: {api_response}", level=logging.INFO)
@@ -1020,81 +560,64 @@ def format_response_to_markdown(api_response, agent_name = None, datasets=None):
             if "memory" in agent or not content:
                 continue
                 
-            if "complexity" in content:
-                complexity_result = format_complexity(content)
-                if complexity_result:  # Only append if not empty
-                    markdown.append(f"{complexity_result}")
-                
             markdown.append(f"\n## {agent.replace('_', ' ').title()}\n")
             
             if agent == "analytical_planner":
-                logger.log_message(f"Analytical planner content: {content}", level=logging.INFO)
-                if 'plan_desc' in content and content['plan_desc']:
-                    markdown.append(f"### Reasoning {content['plan_desc']}")
+                # logger.log_message(f"Analytical planner content: {content}", level=logging.INFO)
+                if 'plan_desc' in content:
+                    markdown.append(f"### Reasoning\n{content['plan_desc']}\n")
                 if 'plan_instructions' in content:
-                    plan_result = format_plan_instructions(content['plan_instructions'])
-                    if plan_result:  # Only append if not empty/None
-                        markdown.append(f"{plan_result}")
+                    markdown.append(f"### Plan Instructions\n{format_plan_instructions(content['plan_instructions'])}\n")
                 else:
-                    if content.get('rationale'):
-                        markdown.append(f"### Reasoning {content['rationale']}")
+                    markdown.append(f"### Reasoning\n{content['rationale']}\n")
             else:  
                 if "rationale" in content:
-                    if content.get('rationale'):
-                        markdown.append(f"### Reasoning{content['rationale']}")
+                    markdown.append(f"### Reasoning\n{content['rationale']}\n")
+
+            if 'code' in content:
+                markdown.append(f"### Code Implementation\n{format_code_backticked_block(content['code'])}\n")
+                # if agent_name is not None:
+                #     # execute the code
+                #     clean_code = format_code_block(content['code'])
+                #     output, json_outputs = execute_code_from_markdown(clean_code, dataframe)
+                #     if output:
+                #         markdown.append("### Execution Output\n")
+                #         markdown.append(f"```output\n{output}\n```\n")
+
+                #     if json_outputs:
+                #         markdown.append("### Plotly JSON Outputs\n")
+                #         for idx, json_output in enumerate(json_outputs):
+                #             if len(json_output) > 1000000:  # If JSON is larger than 1MB
+                #                 logger.log_message(f"Large JSON output detected: {len(json_output)} bytes", level=logging.WARNING)
+                #             markdown.append(f"```plotly\n{json_output}\n```\n")
 
-            if 'code' in content and content['code'] is not None:
-                formatted_code = format_code_backticked_block(content['code'])
-                if formatted_code:  # Check if formatted_code is not None
-                    markdown.append(f"### Code Implementation\n{formatted_code}\n")
-            if 'answer' in content and content['answer']:
-                markdown.append(f"### Answer{content['answer']} Please ask a query about the data")
             if 'summary' in content:
-                import re
-                summary_text = content['summary']
-                summary_text = re.sub(r'```python\n(.*?)\n```', '', summary_text, flags=re.DOTALL)
-
+                # make the summary a bullet-point list
+                summary_lines = remove_code_block_from_summary(content['summary'])
+                summary_lines = content['summary'].split('\n')
+                # remove code block from summary
                 markdown.append("### Summary\n")
-
-                # Extract pre-list intro, bullet points, and post-list text
-                intro_match = re.split(r'\(\d+\)', summary_text, maxsplit=1)
-                if len(intro_match) > 1:
-                    intro_text = intro_match[0].strip()
-                    rest_text = "(1)" + intro_match[1]  # reattach for bullet parsing
-                else:
-                    intro_text = summary_text.strip()
-                    rest_text = ""
-
-                if intro_text:
-                    markdown.append(f"{intro_text}\n")
-
-                # Split bullets at numbered items like (1)...(8)
-                bullets = re.split(r'\(\d+\)', rest_text)
-                bullets = [b.strip(" ,.\n") for b in bullets if b.strip()]
-
-                # Check for post-list content (anything after the last number)
-                for i, bullet in enumerate(bullets):
-                    markdown.append(f"* {bullet}\n")
-
-
-
+                for line in summary_lines:
+                    if line != "":
+                        if line.strip().startswith('•') or line.strip().startswith('-') or line.strip().startswith('*'):
+                            line = line.strip().replace('•', '').replace('-', '').replace('*', '')
+                            markdown.append(f"* {line.strip()}\n")
+                        else:
+                            markdown.append(f"{line.strip()}\n")
 
             if 'refined_complete_code' in content and 'summary' in content:
                 try:
                     if content['refined_complete_code'] is not None and content['refined_complete_code'] != "":
                         clean_code = format_code_block(content['refined_complete_code']) 
                         markdown_code = format_code_backticked_block(content['refined_complete_code'])
-                        output, json_outputs, matplotlib_outputs = execute_code_from_markdown(clean_code, datasets)
+                        output, json_outputs = execute_code_from_markdown(clean_code, dataframe)
                     elif "```python" in content['summary']:
                         clean_code = format_code_block(content['summary'])
                         markdown_code = format_code_backticked_block(content['summary'])
-                        output, json_outputs, matplotlib_outputs = execute_code_from_markdown(clean_code, datasets)
+                        output, json_outputs = execute_code_from_markdown(clean_code, dataframe)
                 except Exception as e:
                     logger.log_message(f"Error in execute_code_from_markdown: {str(e)}", level=logging.ERROR)
                     markdown_code = f"**Error**: {str(e)}"
-                    output = None
-                    json_outputs = []
-                    matplotlib_outputs = []
                     # continue
                 
                 if markdown_code is not None:
@@ -1108,30 +631,45 @@ def format_response_to_markdown(api_response, agent_name = None, datasets=None):
                     markdown.append("### Plotly JSON Outputs\n")
                     for idx, json_output in enumerate(json_outputs):
                         markdown.append(f"```plotly\n{json_output}\n```\n")
-                        
-                if matplotlib_outputs:
-                    markdown.append("### Matplotlib/Seaborn Charts\n")
-                    for idx, img_base64 in enumerate(matplotlib_outputs):
-                        markdown.append(f"```matplotlib\n{img_base64}\n```\n")
             # if agent_name is not None:  
             #     if f"memory_{agent_name}" in api_response:
             #         markdown.append(f"### Memory\n{api_response[f'memory_{agent_name}']}\n")
 
     except Exception as e:
         logger.log_message(f"Error in format_response_to_markdown: {str(e)}", level=logging.ERROR)
-        return f"error formating markdown {str(e)}"
+        return f"{str(e)}"
         
     # logger.log_message(f"Generated markdown content for agent '{agent_name}' at {time.strftime('%Y-%m-%d %H:%M:%S')}: {markdown}, length: {len(markdown)}", level=logging.INFO)
     
     if not markdown or len(markdown) <= 1:
-        logger.log_message(
-            f"Invalid markdown content for agent '{agent_name}' at {time.strftime('%Y-%m-%d %H:%M:%S')}: "
-            f"Content: '{markdown}', Type: {type(markdown)}, Length: {len(markdown) if markdown else 0}, "
-            f"API Response: {api_response}",
-            level=logging.ERROR
-        )
-        return ""
+        logger.log_message(f"Generated markdown (ERROR) content for agent '{agent_name}' at {time.strftime('%Y-%m-%d %H:%M:%S')}: {markdown}, length: {len(markdown)}, api_response: {api_response}", level=logging.INFO)
+        return "Please provide a valid query..."
         
     return '\n'.join(markdown)
 
+# Example usage with dummy data
+if __name__ == "__main__":
+    sample_response = {
+        "code_combiner_agent": {
+            "reasoning": "Sample reasoning for multiple charts.",
+            "refined_complete_code": """
+```python
+import plotly.express as px
+import pandas as pd
+
+# Sample Data
+df = pd.DataFrame({'Category': ['A', 'B', 'C'], 'Values': [10, 20, 30]})
+
+# First Chart
+fig = px.bar(df, x='Category', y='Values', title='Bar Chart')
+fig.show()
+
+# Second Chart
+fig2 = px.pie(df, values='Values', names='Category', title='Pie Chart')
+fig2.show()
+```
+"""
+        }
+    }
 
+    formatted_md = format_response_to_markdown(sample_response)

scripts/generate_test_data.py

@@ -0,0 +1,90 @@
+import sys
+import os
+import random
+from datetime import datetime, timedelta
+import sqlite3
+
+# Add parent directory to path so we can import modules
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from init_db import ModelUsage, session_factory
+
+# Models and providers to use in test data
+MODELS = {
+    "gpt-3.5-turbo": {"provider": "OpenAI", "cost_per_1k": 0.0015},
+    "gpt-4": {"provider": "OpenAI", "cost_per_1k": 0.03},
+    "gpt-4o": {"provider": "OpenAI", "cost_per_1k": 0.01},
+    "gpt-4o-mini": {"provider": "OpenAI", "cost_per_1k": 0.0015},
+    "o1-mini": {"provider": "OpenAI", "cost_per_1k": 0.00015},
+    "claude-3-opus": {"provider": "Anthropic", "cost_per_1k": 0.015},
+    "claude-3-sonnet": {"provider": "Anthropic", "cost_per_1k": 0.008},
+    "claude-3-haiku": {"provider": "Anthropic", "cost_per_1k": 0.003},
+    "llama-3-8b": {"provider": "Groq", "cost_per_1k": 0.0005},
+    "llama-3-70b": {"provider": "Groq", "cost_per_1k": 0.002},
+}
+
+# User IDs to use (can be random if you don't have specific users)
+USER_IDS = [1, 2, 3, 4, 5]
+
+def generate_test_data(num_records=100):
+    """Generate test model usage data"""
+    session = session_factory()
+    
+    try:
+        # Generate records for the past 30 days
+        end_date = datetime.utcnow()
+        start_date = end_date - timedelta(days=30)
+        
+        for _ in range(num_records):
+            # Random timestamp within the date range
+            random_days = random.randint(0, 30)
+            timestamp = end_date - timedelta(days=random_days, 
+                                           hours=random.randint(0, 23),
+                                           minutes=random.randint(0, 59))
+            
+            # Select random model and user
+            model_name = random.choice(list(MODELS.keys()))
+            model_info = MODELS[model_name]
+            user_id = random.choice(USER_IDS)
+            
+            # Generate random token counts
+            prompt_tokens = random.randint(100, 1000)
+            completion_tokens = random.randint(50, 500)
+            total_tokens = prompt_tokens + completion_tokens
+            
+            # Calculate cost
+            cost = (total_tokens / 1000) * model_info["cost_per_1k"]
+            
+            # Create model usage record
+            usage = ModelUsage(
+                user_id=user_id,
+                chat_id=random.randint(1, 50),  # Random chat ID
+                model_name=model_name,
+                provider=model_info["provider"],
+                prompt_tokens=prompt_tokens,
+                completion_tokens=completion_tokens,
+                total_tokens=total_tokens,
+                query_size=prompt_tokens * 4,  # Approximate characters
+                response_size=completion_tokens * 4,  # Approximate characters
+                cost=cost,
+                timestamp=timestamp,
+                is_streaming=random.choice([True, False]),
+                request_time_ms=random.randint(500, 5000)  # Between 0.5 and 5 seconds
+            )
+            session.add(usage)
+        
+        session.commit()
+        print(f"Successfully generated {num_records} test records")
+    
+    except Exception as e:
+        session.rollback()
+        print(f"Error generating test data: {e}")
+    
+    finally:
+        session.close()
+
+if __name__ == "__main__":
+    # Default to 100 records, but allow command line override
+    num_records = int(sys.argv[1]) if len(sys.argv) > 1 else 100
+    generate_test_data(num_records)
+    print("Done! The model_usage table has been populated with test data.") 

scripts/init_production_db.py

@@ -1,191 +0,0 @@
-#!/usr/bin/env python3
-"""
-Production database initialization script.
-This ensures templates are populated properly and verifies database health.
-SAFE for PostgreSQL/RDS - only creates tables on SQLite databases.
-"""
-
-import sys
-import os
-import logging
-from datetime import datetime, UTC
-
-# Add the project root to the Python path
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from src.db.init_db import init_db, session_factory, engine, is_postgres_db
-from src.db.schemas.models import Base, AgentTemplate, UserTemplatePreference
-from scripts.populate_agent_templates import populate_templates
-from sqlalchemy import inspect, text
-from src.utils.logger import Logger
-
-logger = Logger("init_production_db", see_time=True, console_log=True)
-
-def get_database_type():
-    """Get the database type (sqlite or postgresql)."""
-    try:
-        if is_postgres_db():
-            return "postgresql"
-        else:
-            return "sqlite"
-    except Exception as e:
-        logger.log_message(f"Error determining database type: {e}", logging.ERROR)
-        return "unknown"
-
-def check_table_exists(table_name: str) -> bool:
-    """Check if a table exists in the database."""
-    try:
-        inspector = inspect(engine)
-        tables = inspector.get_table_names()
-        return table_name in tables
-    except Exception as e:
-        logger.log_message(f"Error checking table existence: {e}", logging.ERROR)
-        return False
-
-def verify_database_schema():
-    """Verify that all required tables exist. Only create tables on SQLite."""
-    db_type = get_database_type()
-    logger.log_message(f"🔍 Verifying database schema for {db_type.upper()} database...", logging.INFO)
-    
-    required_tables = [
-        'users', 'chats', 'messages', 'model_usage', 'code_executions',
-        'message_feedback', 'deep_analysis_reports', 'agent_templates',
-        'user_template_preferences'
-    ]
-    
-    missing_tables = []
-    existing_tables = []
-    
-    for table in required_tables:
-        if not check_table_exists(table):
-            missing_tables.append(table)
-            logger.log_message(f"❌ Missing table: {table}", logging.WARNING)
-        else:
-            existing_tables.append(table)
-            logger.log_message(f"✅ Table exists: {table}", logging.INFO)
-    
-    if missing_tables:
-        if db_type == "sqlite":
-            logger.log_message(f"🔧 Creating missing tables on SQLite: {missing_tables}", logging.INFO)
-            try:
-                # Safe to create tables on SQLite
-                Base.metadata.create_all(engine)
-                logger.log_message("✅ All tables created successfully on SQLite", logging.INFO)
-            except Exception as e:
-                logger.log_message(f"❌ Failed to create tables: {e}", logging.ERROR)
-                raise
-        else:
-            # PostgreSQL/RDS - DO NOT create tables automatically
-            logger.log_message(f"⚠️  WARNING: Missing tables detected in {db_type.upper()} database: {missing_tables}", logging.WARNING)
-            logger.log_message("🛡️  SAFETY: Not creating tables automatically on PostgreSQL/RDS", logging.INFO)
-            logger.log_message("📋 Please ensure these tables exist in your RDS database:", logging.INFO)
-            for table in missing_tables:
-                logger.log_message(f"   - {table}", logging.INFO)
-            
-            # Continue without failing - the app might still work with existing tables
-            if 'agent_templates' in missing_tables or 'user_template_preferences' in missing_tables:
-                logger.log_message("⚠️  Template functionality may not work without agent_templates and user_template_preferences tables", logging.WARNING)
-    else:
-        logger.log_message(f"✅ All required tables exist in {db_type.upper()} database", logging.INFO)
-
-def verify_template_data():
-    """Verify that agent templates are populated. Safe for all database types."""
-    logger.log_message("📋 Verifying template data...", logging.INFO)
-    
-    session = session_factory()
-    try:
-        # Check if agent_templates table exists before querying
-        if not check_table_exists('agent_templates'):
-            logger.log_message("⚠️  agent_templates table does not exist, skipping template verification", logging.WARNING)
-            return
-        
-        template_count = session.query(AgentTemplate).filter(AgentTemplate.is_active == True).count()
-        logger.log_message(f"📊 Found {template_count} active templates", logging.INFO)
-        
-        if template_count == 0:
-            logger.log_message("🔧 No templates found, populating...", logging.INFO)
-            try:
-                populate_templates()
-                
-                # Verify population worked
-                new_count = session.query(AgentTemplate).filter(AgentTemplate.is_active == True).count()
-                logger.log_message(f"✅ Templates populated. Total active templates: {new_count}", logging.INFO)
-            except Exception as e:
-                logger.log_message(f"❌ Template population failed: {e}", logging.ERROR)
-                logger.log_message("⚠️  App will continue but template functionality may not work", logging.WARNING)
-        else:
-            logger.log_message("✅ Templates already populated", logging.INFO)
-            
-    except Exception as e:
-        logger.log_message(f"❌ Error verifying templates: {e}", logging.ERROR)
-        logger.log_message("⚠️  Template verification failed, but app will continue", logging.WARNING)
-    finally:
-        session.close()
-
-def test_template_api_functionality():
-    """Test that template-related database operations work. Safe for all database types."""
-    logger.log_message("🧪 Testing template API functionality...", logging.INFO)
-    
-    session = session_factory()
-    try:
-        # Check if agent_templates table exists before testing
-        if not check_table_exists('agent_templates'):
-            logger.log_message("⚠️  agent_templates table does not exist, skipping API test", logging.WARNING)
-            return
-        
-        # Test basic template query
-        templates = session.query(AgentTemplate).filter(AgentTemplate.is_active == True).limit(5).all()
-        logger.log_message(f"✅ Successfully queried {len(templates)} templates", logging.INFO)
-        
-        if templates:
-            sample_template = templates[0]
-            logger.log_message(f"📄 Sample template: {sample_template.template_name} - {sample_template.display_name}", logging.INFO)
-        else:
-            logger.log_message("📭 No templates found in database", logging.INFO)
-        
-    except Exception as e:
-        logger.log_message(f"❌ Template API test failed: {e}", logging.ERROR)
-        logger.log_message("⚠️  Template API may not work properly", logging.WARNING)
-    finally:
-        session.close()
-
-def run_safe_initialization():
-    """Run safe database initialization that respects production databases."""
-    db_type = get_database_type()
-    logger.log_message(f"🚀 Starting SAFE database initialization for {db_type.upper()}...", logging.INFO)
-    
-    if db_type == "postgresql":
-        logger.log_message("🛡️  PostgreSQL/RDS detected - running in SAFE mode", logging.INFO)
-        logger.log_message("📋 Will only verify schema and populate templates", logging.INFO)
-    elif db_type == "sqlite":
-        logger.log_message("💽 SQLite detected - full initialization mode", logging.INFO)
-    
-    try:
-        # Step 1: Initialize database (safe for all types)
-        logger.log_message("Step 1: Basic database initialization", logging.INFO)
-        if db_type == "sqlite":
-            init_db()  # Only run full init on SQLite
-        else:
-            logger.log_message("Skipping init_db() for PostgreSQL (safety)", logging.INFO)
-        
-        # Step 2: Verify schema (safe - only creates tables on SQLite)
-        logger.log_message("Step 2: Schema verification", logging.INFO)
-        verify_database_schema()
-        
-        # Step 3: Verify template data (safe for all types)
-        logger.log_message("Step 3: Template data verification", logging.INFO)
-        verify_template_data()
-        
-        # Step 4: Test functionality (safe for all types)
-        logger.log_message("Step 4: Functionality testing", logging.INFO)
-        test_template_api_functionality()
-        
-        logger.log_message(f"🎉 Safe database initialization completed for {db_type.upper()}!", logging.INFO)
-        
-    except Exception as e:
-        logger.log_message(f"💥 Database initialization failed: {e}", logging.ERROR)
-        logger.log_message("⚠️  App may still start but some features might not work", logging.WARNING)
-        # Don't raise - let the app try to start anyway
-
-if __name__ == "__main__":
-    run_safe_initialization() 

scripts/populate_agent_templates.py

@@ -1,508 +0,0 @@
-#!/usr/bin/env python3
-"""
-SQLite Agent Template Management Script
-Similar to manage_templates.py but optimized for local SQLite development.
-Reads agents from agents_config.json and manages SQLite database.
-"""
-
-import sys
-import os
-import json
-import requests
-from datetime import datetime, UTC
-from pathlib import Path
-
-# Add the project root to the Python path
-script_dir = os.path.dirname(os.path.abspath(__file__))
-backend_dir = os.path.dirname(script_dir)
-project_root = os.path.dirname(os.path.dirname(backend_dir))
-
-# Change to backend directory to ensure proper path resolution
-os.chdir(backend_dir)
-sys.path.append(backend_dir)
-
-from src.db.init_db import session_factory, DATABASE_URL
-from src.db.schemas.models import AgentTemplate
-from sqlalchemy.exc import IntegrityError
-
-def get_database_type():
-    """Detect database type from DATABASE_URL"""
-    if DATABASE_URL.startswith('postgresql'):
-        return "postgresql"
-    elif DATABASE_URL.startswith('sqlite'):
-        return "sqlite"
-    else:
-        return "unknown"
-
-def load_agents_config():
-    """Load agents configuration from agents_config.json"""
-    # Try multiple possible locations for agents_config.json
-    possible_paths = [
-        os.path.join(backend_dir, 'agents_config.json'),  # Backend directory (copied file)
-        os.path.join(project_root, 'agents_config.json'),  # Project root
-        '/app/agents_config.json',  # Container root (HF Spaces)
-        'agents_config.json'  # Current directory
-    ]
-    
-    config_path = None
-    for path in possible_paths:
-        if os.path.exists(path):
-            config_path = path
-            print(f"📖 Found agents_config.json at: {config_path}")
-            break
-    
-    if not config_path:
-        paths_str = '\n  '.join(possible_paths)
-        raise FileNotFoundError(f"agents_config.json not found in any of these locations:\n  {paths_str}")
-    
-    with open(config_path, 'r', encoding='utf-8') as f:
-        config = json.load(f)
-    
-    return config.get('templates', [])
-
-def download_icon(icon_url, template_name):
-    """Download icon from URL and save to frontend directory"""
-    if not icon_url or not icon_url.startswith('http'):
-        print(f"⏭️  Skipping icon download for {template_name} (not a URL: {icon_url})")
-        return icon_url
-    
-    try:
-        # Determine frontend directory
-        frontend_dir = os.path.join(project_root, 'Auto-Analyst-CS', 'auto-analyst-frontend')
-        public_dir = os.path.join(frontend_dir, 'public')
-        
-        if not os.path.exists(public_dir):
-            print(f"⚠️  Frontend public directory not found: {public_dir}")
-            return icon_url
-        
-        # Parse the path from icon_url
-        if '/icons/templates/' in icon_url:
-            relative_path = icon_url.split('/icons/templates/')[-1]
-            icon_dir = os.path.join(public_dir, 'icons', 'templates')
-        else:
-            # Fallback: use filename from URL
-            filename = icon_url.split('/')[-1]
-            if not filename.endswith(('.svg', '.png', '.jpg', '.jpeg')):
-                filename += '.svg'
-            relative_path = filename
-            icon_dir = os.path.join(public_dir, 'icons', 'templates')
-        
-        # Create icon directory if it doesn't exist
-        os.makedirs(icon_dir, exist_ok=True)
-        
-        # Download and save icon
-        icon_path = os.path.join(icon_dir, relative_path)
-        
-        # Skip if already exists
-        if os.path.exists(icon_path):
-            print(f"📁 Icon already exists: {relative_path}")
-            return f"/icons/templates/{relative_path}"
-        
-        response = requests.get(icon_url, timeout=10)
-        response.raise_for_status()
-        
-        with open(icon_path, 'wb') as f:
-            f.write(response.content)
-        
-        print(f"📥 Downloaded icon: {relative_path}")
-        return f"/icons/templates/{relative_path}"
-        
-    except Exception as e:
-        print(f"❌ Failed to download icon for {template_name}: {str(e)}")
-        return icon_url
-
-def sync_agents_from_config():
-    """Synchronize agents from agents_config.json to SQLite database"""
-    session = session_factory()
-    db_type = get_database_type()
-    
-    # if db_type != "sqlite":
-    #     print(f"⚠️  This script is designed for SQLite, but detected {db_type}")
-    #     print("Consider using manage_templates.py for PostgreSQL")
-    #     return
-    
-    try:
-        # Load configuration
-        print(f"📖 Loading agents from agents_config.json...")
-        templates_config = load_agents_config()
-        
-        if not templates_config:
-            print("❌ No templates found in agents_config.json")
-            return
-        
-        # Track statistics
-        created_count = 0
-        updated_count = 0
-        skipped_count = 0
-        
-        print(f"🔍 Processing {len(templates_config)} templates for SQLite database")
-        print(f"📋 Database URL: {DATABASE_URL}")
-        
-        # Group templates by category for display
-        categories = {}
-        for template_data in templates_config:
-            category = template_data.get('category', 'Uncategorized')
-            if category not in categories:
-                categories[category] = []
-            categories[category].append(template_data)
-        
-        # Process templates by category
-        for category, templates in categories.items():
-            print(f"\n📁 {category}:")
-            
-            for template_data in templates:
-                template_name = template_data["template_name"]
-                
-                # Check if template already exists
-                existing = session.query(AgentTemplate).filter(
-                    AgentTemplate.template_name == template_name
-                ).first()
-                
-                # Download icon if it's a URL
-                icon_url = template_data.get("icon_url", "")
-                if icon_url.startswith('http'):
-                    icon_url = download_icon(icon_url, template_name)
-                
-                if existing:
-                    # Update existing template
-                    existing.display_name = template_data["display_name"]
-                    existing.description = template_data["description"]
-                    existing.icon_url = icon_url
-                    existing.prompt_template = template_data["prompt_template"]
-                    existing.category = template_data.get("category", "Uncategorized")
-                    existing.is_premium_only = template_data.get("is_premium_only", False)
-                    existing.is_active = template_data.get("is_active", True)
-                    existing.variant_type = template_data.get("variant_type", "individual")
-                    existing.base_agent = template_data.get("base_agent", template_name)
-                    existing.updated_at = datetime.now(UTC)
-                    
-                    variant_icon = "🤖" if template_data.get("variant_type") == "planner" else "👤"
-                    premium_icon = "🔒" if template_data.get("is_premium_only") else "🆓"
-                    print(f"🔄 Updated: {template_name} {variant_icon} {premium_icon}")
-                    updated_count += 1
-                else:
-                    # Create new template
-                    template = AgentTemplate(
-                        template_name=template_name,
-                        display_name=template_data["display_name"],
-                        description=template_data["description"],
-                        icon_url=icon_url,
-                        prompt_template=template_data["prompt_template"],
-                        category=template_data.get("category", "Uncategorized"),
-                        is_premium_only=template_data.get("is_premium_only", False),
-                        is_active=template_data.get("is_active", True),
-                        variant_type=template_data.get("variant_type", "individual"),
-                        base_agent=template_data.get("base_agent", template_name),
-                        created_at=datetime.now(UTC),
-                        updated_at=datetime.now(UTC)
-                    )
-                    
-                    session.add(template)
-                    variant_icon = "🤖" if template_data.get("variant_type") == "planner" else "👤"
-                    premium_icon = "🔒" if template_data.get("is_premium_only") else "🆓"
-                    print(f"✅ Created: {template_name} {variant_icon} {premium_icon}")
-                    created_count += 1
-        
-        # Handle removals if specified in config
-        remove_list = []
-        # Re-load the full config to check for removals
-        try:
-            full_config_path = None
-            possible_paths = [
-                os.path.join(backend_dir, 'agents_config.json'),
-                os.path.join(project_root, 'agents_config.json'),
-                '/app/agents_config.json',
-                'agents_config.json'
-            ]
-            
-            for path in possible_paths:
-                if os.path.exists(path):
-                    full_config_path = path
-                    break
-            
-            if full_config_path:
-                with open(full_config_path, 'r', encoding='utf-8') as f:
-                    full_config = json.load(f)
-                    if 'remove' in full_config:
-                        remove_list = full_config['remove']
-        except Exception as e:
-            print(f"⚠️ Could not load removal list: {e}")
-        
-        # Remove templates marked for removal
-        if remove_list:
-            print(f"\n🗑️ --- Processing Removals ---")
-            for template_name in remove_list:
-                existing = session.query(AgentTemplate).filter(
-                    AgentTemplate.template_name == template_name
-                ).first()
-                
-                if existing:
-                    session.delete(existing)
-                    print(f"🗑️ Removed: {template_name}")
-                else:
-                    print(f"⏭️ Skipping removal: {template_name} (not found)")
-        
-        # Commit all changes
-        session.commit()
-        
-        print(f"\n📊 --- Summary ---")
-        print(f"✅ Templates created: {created_count}")
-        print(f"🔄 Templates updated: {updated_count}")
-        print(f"⏭️ Templates skipped: {skipped_count}")
-        
-        # Show total count in database
-        total_count = session.query(AgentTemplate).count()
-        free_count = session.query(AgentTemplate).filter(AgentTemplate.is_premium_only == False).count()
-        premium_count = session.query(AgentTemplate).filter(AgentTemplate.is_premium_only == True).count()
-        individual_count = session.query(AgentTemplate).filter(AgentTemplate.variant_type == 'individual').count()
-        planner_count = session.query(AgentTemplate).filter(AgentTemplate.variant_type == 'planner').count()
-        
-        print(f"🗄️ Total templates in database: {total_count}")
-        print(f"🆓 Free templates: {free_count}")
-        print(f"🔒 Premium templates: {premium_count}")
-        print(f"👤 Individual variants: {individual_count}")
-        print(f"🤖 Planner variants: {planner_count}")
-        
-    except Exception as e:
-        session.rollback()
-        print(f"❌ Error syncing templates: {str(e)}")
-        raise
-    finally:
-        session.close()
-
-def list_templates():
-    """List all existing templates in the database"""
-    session = session_factory()
-    
-    try:
-        templates = session.query(AgentTemplate).order_by(
-            AgentTemplate.category, 
-            AgentTemplate.is_premium_only, 
-            AgentTemplate.template_name
-        ).all()
-        
-        if not templates:
-            print("No templates found in database.")
-            return
-        
-        print(f"\n--- Existing Templates ({len(templates)} total) ---")
-        
-        current_category = None
-        for template in templates:
-            if template.category != current_category:
-                current_category = template.category
-                print(f"\n📁 {current_category}:")
-            
-            status = "🔒 Premium" if template.is_premium_only else "🆓 Free"
-            active = "✅ Active" if template.is_active else "❌ Inactive"
-            variant = getattr(template, 'variant_type', 'individual')
-            variant_icon = "🤖" if variant == "planner" else "👤"
-            
-            print(f"  • {template.template_name} ({template.display_name})")
-            print(f"    {status} - {active} - {variant_icon} {variant}")
-            print(f"    📝 {template.description}")
-    
-    except Exception as e:
-        print(f"❌ Error listing templates: {str(e)}")
-    finally:
-        session.close()
-
-def remove_all_templates():
-    """Remove all templates from database (for testing)"""
-    session = session_factory()
-    
-    try:
-        deleted_count = session.query(AgentTemplate).delete()
-        session.commit()
-        print(f"🗑️ Removed {deleted_count} templates from database")
-    
-    except Exception as e:
-        session.rollback()
-        print(f"❌ Error removing templates: {str(e)}")
-    finally:
-        session.close()
-
-def validate_config():
-    """Validate the agents_config.json structure"""
-    try:
-        templates_config = load_agents_config()
-        
-        print(f"📋 Validating agents_config.json...")
-        print(f"✅ Found {len(templates_config)} templates")
-        
-        # Check required fields
-        required_fields = ['template_name', 'display_name', 'description', 'prompt_template']
-        issues = []
-        
-        for i, template in enumerate(templates_config):
-            for field in required_fields:
-                if field not in template:
-                    issues.append(f"Template {i}: Missing required field '{field}'")
-        
-        if issues:
-            print(f"❌ Validation issues found:")
-            for issue in issues:
-                print(f"  • {issue}")
-        else:
-            print(f"✅ Configuration is valid")
-            
-        # Show summary by category
-        categories = {}
-        for template in templates_config:
-            category = template.get('category', 'Uncategorized')
-            if category not in categories:
-                categories[category] = {'free': 0, 'premium': 0, 'individual': 0, 'planner': 0}
-            
-            if template.get('is_premium_only', False):
-                categories[category]['premium'] += 1
-            else:
-                categories[category]['free'] += 1
-                
-            if template.get('variant_type', 'individual') == 'planner':
-                categories[category]['planner'] += 1
-            else:
-                categories[category]['individual'] += 1
-        
-        print(f"\n📊 Summary by category:")
-        for category, counts in categories.items():
-            total = counts['free'] + counts['premium']
-            print(f"  📁 {category}: {total} templates")
-            print(f"    🆓 Free: {counts['free']} | 🔒 Premium: {counts['premium']}")
-            print(f"    👤 Individual: {counts['individual']} | 🤖 Planner: {counts['planner']}")
-    
-    except Exception as e:
-        print(f"❌ Error validating config: {str(e)}")
-
-def create_minimal_templates():
-    """Create a minimal set of essential templates for container environments"""
-    session = session_factory()
-    
-    try:
-        print("🔧 Creating minimal template set...")
-        
-        # Define minimal essential templates
-        minimal_templates = [
-            {
-                "template_name": "preprocessing_agent",
-                "display_name": "Data Preprocessing Agent",
-                "description": "Cleans and prepares DataFrame using Pandas and NumPy",
-                "icon_url": "/icons/templates/preprocessing_agent.svg",
-                "category": "Data Manipulation",
-                "is_premium_only": False,
-                "variant_type": "individual",
-                "base_agent": "preprocessing_agent",
-                "is_active": True,
-                "prompt_template": "You are a preprocessing agent that cleans and prepares data using Pandas and NumPy. Handle missing values, detect column types, and convert date strings to datetime. Generate clean Python code for data preprocessing based on the user's analysis goals."
-            },
-            {
-                "template_name": "data_viz_agent",
-                "display_name": "Data Visualization Agent",
-                "description": "Creates interactive visualizations using Plotly",
-                "icon_url": "/icons/templates/data_viz_agent.svg",
-                "category": "Data Visualization",
-                "is_premium_only": False,
-                "variant_type": "individual",
-                "base_agent": "data_viz_agent",
-                "is_active": True,
-                "prompt_template": "You are a data visualization agent. Create interactive visualizations using Plotly based on user requirements. Generate appropriate chart types, apply styling, and ensure visualizations effectively communicate insights."
-            },
-            {
-                "template_name": "sk_learn_agent",
-                "display_name": "Machine Learning Agent",
-                "description": "Trains ML models using scikit-learn",
-                "icon_url": "/icons/templates/sk_learn_agent.svg",
-                "category": "Data Modelling",
-                "is_premium_only": False,
-                "variant_type": "individual",
-                "base_agent": "sk_learn_agent",
-                "is_active": True,
-                "prompt_template": "You are a machine learning agent. Use scikit-learn to train and evaluate ML models including classification, regression, and clustering. Provide feature importance insights and model performance metrics."
-            }
-        ]
-        
-        created_count = 0
-        
-        for template_data in minimal_templates:
-            template_name = template_data["template_name"]
-            
-            # Check if template already exists
-            existing = session.query(AgentTemplate).filter(
-                AgentTemplate.template_name == template_name
-            ).first()
-            
-            if not existing:
-                template = AgentTemplate(
-                    template_name=template_name,
-                    display_name=template_data["display_name"],
-                    description=template_data["description"],
-                    icon_url=template_data["icon_url"],
-                    prompt_template=template_data["prompt_template"],
-                    category=template_data["category"],
-                    is_premium_only=template_data["is_premium_only"],
-                    is_active=template_data["is_active"],
-                    variant_type=template_data["variant_type"],
-                    base_agent=template_data["base_agent"],
-                    created_at=datetime.now(UTC),
-                    updated_at=datetime.now(UTC)
-                )
-                
-                session.add(template)
-                print(f"✅ Created minimal template: {template_name}")
-                created_count += 1
-            else:
-                print(f"⏭️ Template already exists: {template_name}")
-        
-        session.commit()
-        print(f"📊 Created {created_count} minimal templates")
-        
-    except Exception as e:
-        session.rollback()
-        print(f"❌ Error creating minimal templates: {str(e)}")
-        raise
-    finally:
-        session.close()
-
-def populate_templates():
-    """Legacy compatibility function for backward compatibility"""
-    print("⚠️  Legacy populate_templates() called - checking for agents_config.json...")
-    
-    # Check if agents_config.json exists anywhere
-    possible_paths = [
-        os.path.join(backend_dir, 'agents_config.json'),
-        os.path.join(project_root, 'agents_config.json'),
-        '/app/agents_config.json',
-        'agents_config.json'
-    ]
-    
-    config_exists = any(os.path.exists(path) for path in possible_paths)
-    
-    if config_exists:
-        print("📖 Found agents_config.json - using sync_agents_from_config()")
-        sync_agents_from_config()
-    else:
-        print("⚠️  agents_config.json not found - using fallback minimal templates")
-        print("💡 Creating essential templates for container environment")
-        create_minimal_templates()
-
-if __name__ == "__main__":
-    import argparse
-    
-    parser = argparse.ArgumentParser(description="SQLite Agent Template Management")
-    parser.add_argument("action", choices=["sync", "list", "remove-all", "validate"], 
-                       help="Action to perform")
-    
-    args = parser.parse_args()
-    
-    if args.action == "sync":
-        print("🚀 Synchronizing agents from agents_config.json to SQLite...")
-        sync_agents_from_config()
-    elif args.action == "list":
-        list_templates()
-    elif args.action == "validate":
-        validate_config()
-    elif args.action == "remove-all":
-        confirm = input("⚠️ Are you sure you want to remove ALL templates? (yes/no): ")
-        if confirm.lower() == "yes":
-            remove_all_templates()
-        else:
-            print("Operation cancelled.") 

scripts/setup_analytics_data.py

@@ -0,0 +1,114 @@
+import sys
+import os
+from datetime import datetime, timedelta
+import sqlite3
+
+# Add parent directory to path so we can import modules
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from src.db.schemas.models import ModelUsage, User
+from src.db.init_db import session_factory, init_db
+from scripts.generate_test_data import generate_test_data
+from scripts.create_test_user import create_test_users
+
+def check_database():
+    """Check if database exists and has the required tables"""
+    db_path = os.path.join(
+        os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 
+        "chat_database.db"
+    )
+    
+    if not os.path.exists(db_path):
+        print(f"Database file not found at {db_path}")
+        print("Creating database...")
+        init_db()
+        return False
+        
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+    
+    # Check if model_usage table exists
+    cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='model_usage'")
+    if not cursor.fetchone():
+        print("model_usage table does not exist.")
+        print("Creating tables...")
+        init_db()
+        return False
+        
+    # Count records in model_usage table
+    cursor.execute("SELECT COUNT(*) FROM model_usage")
+    count = cursor.fetchone()[0]
+    print(f"Found {count} records in model_usage table")
+    
+    conn.close()
+    return count > 0
+
+def setup_analytics():
+    """Set up analytics data for testing"""
+    # Check database status
+    has_data = check_database()
+    
+    # Create test users
+    print("\nCreating test users...")
+    create_test_users()
+    
+    # Generate model usage data if needed
+    if not has_data:
+        print("\nGenerating model usage test data...")
+        generate_test_data(100)
+    else:
+        print("\nDatabase already has model usage data. Skipping generation.")
+        choice = input("Generate additional data anyway? (y/n): ")
+        if choice.lower() == 'y':
+            records = int(input("How many records to generate? [default: 100]: ") or "100")
+            generate_test_data(records)
+    
+    # Verify data was created
+    session = session_factory()
+    try:
+        # Count model usage records
+        usage_count = session.query(ModelUsage).count()
+        print(f"\nTotal model usage records: {usage_count}")
+        
+        # Count users
+        user_count = session.query(User).count()
+        print(f"Total users: {user_count}")
+        
+        # Check recent data
+        yesterday = datetime.utcnow() - timedelta(days=1)
+        recent_count = session.query(ModelUsage).filter(ModelUsage.timestamp >= yesterday).count()
+        print(f"Records from the last 24 hours: {recent_count}")
+        
+        # Get model breakdown
+        models = {}
+        providers = {}
+        
+        for usage in session.query(ModelUsage).all():
+            if usage.model_name not in models:
+                models[usage.model_name] = 0
+            models[usage.model_name] += 1
+            
+            if usage.provider not in providers:
+                providers[usage.provider] = 0
+            providers[usage.provider] += 1
+            
+        print("\nModel breakdown:")
+        for model, count in models.items():
+            print(f"  {model}: {count} records")
+            
+        print("\nProvider breakdown:")
+        for provider, count in providers.items():
+            print(f"  {provider}: {count} records")
+            
+    finally:
+        session.close()
+    
+    print("\nSetup complete!")
+    print("To access the analytics dashboard:")
+    print("1. Make sure the backend server is running")
+    print("2. In your browser, set localStorage.adminApiKey to 'default-admin-key-change-me'")
+    print("   (or the value in your ADMIN_API_KEY environment variable)")
+    print("3. Go to http://localhost:3000/analytics/dashboard")
+
+if __name__ == "__main__":
+    setup_analytics() 

scripts/test_agent_parsing.py

@@ -0,0 +1,42 @@
+
+def parse_agents(agent_string):
+    """
+    Parse a string containing agent names separated by ->, (, ), or commas
+    and return a list of agent names.
+    """
+    if not agent_string or not agent_string.strip():
+        return []
+    
+    # Replace parentheses with spaces to handle cases with parentheses
+    import re
+    cleaned_string = re.sub(r'\(.*?\)', '', agent_string)
+    
+    # Split by -> to get individual agent segments
+    agent_segments = cleaned_string.split('->')
+    
+    # Process each segment to extract agent names
+    agents = []
+    for segment in agent_segments:
+        # Split by comma and strip whitespace
+        segment_agents = [agent.strip() for agent in segment.split(',') if agent.strip()]
+        agents.extend(segment_agents)
+    
+    return agents
+
+sample = "preprocessing_agent -> statistical_analytics_agent"
+agents = parse_agents(sample)
+print(agents)
+
+# Test with different formats
+test_samples = [
+    "preprocessing_agent -> data_viz_agent",
+    "preprocessing_agent(dataset, goal)",
+    "preprocessing_agent -> statistical_analytics_agent, data_viz_agent",
+    "preprocessing_agent, statistical_analytics_agent -> data_viz_agent",
+    "",
+    "preprocessing_agent(dataset, goal) -> data_viz_agent(dataset)"
+]
+
+for test in test_samples:
+    print(f"Input: {test}")
+    print(f"Output: {parse_agents(test)}")

scripts/test_model_usage.py

@@ -0,0 +1,114 @@
+import requests
+import time
+import uuid
+import os
+import sys
+
+# Add the parent directory to the path so we can import modules
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from ai_manager import AI_Manager
+from init_db import session_factory, ModelUsage
+
+def test_ai_manager_directly():
+    """Test the AI Manager directly to see if it records usage"""
+    print("Starting direct AI Manager test...")
+    
+    # Create a test AI Manager
+    ai_manager = AI_Manager()
+    
+    # Generate a unique user ID and chat ID for testing
+    test_user_id = int(time.time()) % 10000
+    test_chat_id = int(uuid.uuid4().int % 100000)
+    test_model = "test-direct-model"
+    
+    print(f"Using test user_id: {test_user_id}, chat_id: {test_chat_id}")
+    
+    # Call save_usage_to_db directly
+    ai_manager.save_usage_to_db(
+        user_id=test_user_id,
+        chat_id=test_chat_id,
+        model_name=test_model,
+        provider="Test Provider",
+        prompt_tokens=150,
+        completion_tokens=250,
+        total_tokens=400,
+        query_size=600,
+        response_size=1200,
+        cost=0.002,
+        request_time_ms=200,
+        is_streaming=False
+    )
+    
+    print("Usage data saved directly via AI Manager")
+    
+    # Check if it was actually saved
+    session = session_factory()
+    try:
+        # Count records for our test user
+        count = session.query(ModelUsage).filter(
+            ModelUsage.user_id == test_user_id,
+            ModelUsage.model_name == test_model
+        ).count()
+        
+        if count > 0:
+            print(f"SUCCESS: Found {count} records for test user {test_user_id}")
+            records = session.query(ModelUsage).filter(
+                ModelUsage.user_id == test_user_id
+            ).all()
+            
+            for record in records:
+                print(f"  - {record.usage_id}: {record.model_name}, {record.total_tokens} tokens, ${record.cost}")
+        else:
+            print(f"FAILURE: No records found for test user {test_user_id}")
+            print("Check the database connection and ModelUsage table")
+    finally:
+        session.close()
+    
+def test_api_endpoint():
+    """Test the API endpoint that creates test usage records"""
+    print("\nTesting API endpoint for creating test usage...")
+    
+    admin_key = os.getenv("ADMIN_API_KEY", "admin-api-key-change-me")
+    base_url = "http://localhost:8000" 
+    
+    # Generate a unique user ID for testing
+    test_user_id = int(time.time()) % 10000
+    test_model = "test-api-model"
+    
+    # Call the API endpoint
+    response = requests.post(
+        f"{base_url}/analytics/debug/create_test_usage",
+        params={
+            "user_id": test_user_id,
+            "model_name": test_model
+        },
+        headers={
+            "X-Admin-API-Key": admin_key
+        }
+    )
+    
+    if response.status_code == 200:
+        data = response.json()
+        print(f"SUCCESS: Created test usage via API endpoint")
+        print(f"Response: {data}")
+        
+        # Verify the record was created
+        verification = requests.get(
+            f"{base_url}/analytics/debug/user_usage/{test_user_id}",
+            headers={
+                "X-Admin-API-Key": admin_key
+            }
+        ).json()
+        
+        print(f"User {test_user_id} has {verification.get('total_records', 0)} records")
+        if verification.get('recent_records'):
+            print(f"Recent records: {verification['recent_records']}")
+    else:
+        print(f"FAILURE: API endpoint returned status {response.status_code}")
+        print(f"Response: {response.text}")
+
+if __name__ == "__main__":
+    # Run both tests
+    test_ai_manager_directly()
+    test_api_endpoint() 

scripts/test_token_counting.py

@@ -0,0 +1,176 @@
+import sys
+from pathlib import Path
+sys.path.append(str(Path(__file__).resolve().parent.parent))
+
+from src.managers.ai_manager import AI_Manager
+
+ai_manager = AI_Manager()
+
+input_prompt = "What are the key insights from this data? "
+
+output = """
+## Analytical Planner
+
+### Reasoning
+To extract key insights from the housing dataset, it is essential to first preprocess the data to ensure its quality and suitability for analysis. Following preprocessing, performing statistical analysis will help identify significant patterns and relationships within the data. Finally, visualizing the results will provide a clear and interpretable representation of the insights derived.
+
+
+## Preprocessing Agent
+
+### Reasoning
+To derive key insights from the Housing Dataset, it is essential to start with data preprocessing and introductory analysis. This involves categorizing the data into numeric and categorical features, handling missing values appropriately, and preparing the data for further exploration. By organizing the data effectively and understanding the relationships between different variables, we can uncover meaningful patterns and trends that inform our analysis objectives.
+
+### Code Implementation
+```python
+import numpy as np
+import pandas as pd
+
+# Define a safe datetime conversion function
+def safe_to_datetime(date):
+    try:
+        return pd.to_datetime(date, errors='coerce', cache=False)
+    except (ValueError, TypeError):
+        return pd.NaT
+
+# Create copies of the original dataframe
+df_copy = df.copy()
+
+# Identify categorical and numeric columns
+categorical_columns = df_copy.select_dtypes(include=['object', 'category']).columns.tolist()
+numeric_columns = df_copy.select_dtypes(include=[np.number]).columns.tolist()
+
+# Handle missing values
+# For numeric columns, fill missing values with the median
+df_copy[numeric_columns] = df_copy[numeric_columns].fillna(df_copy[numeric_columns].median())
+
+# For categorical columns, fill missing values with the mode
+for col in categorical_columns:
+    df_copy[col] = df_copy[col].fillna(df_copy[col].mode()[0])
+
+# Convert string dates to datetime format if any date columns exist
+# Example: If there's a column named 'date', uncomment the following lines
+# if 'date' in df_copy.columns:
+#     df_copy['date'] = df_copy['date'].apply(safe_to_datetime)
+
+# Create a correlation matrix for numeric columns
+correlation_matrix = df_copy[numeric_columns].corr()
+
+# Display the first few rows of the processed dataframe and the correlation matrix
+print(df_copy.head())
+print("\nCorrelation Matrix:\n", correlation_matrix)
+```
+
+### Summary
+1. **Import Libraries**: The necessary libraries, NumPy and Pandas, are imported to handle numerical operations and data manipulation.
+
+2. **Datetime Conversion Function**: A `safe_to_datetime` function is defined to gracefully handle the conversion of string dates to datetime objects, ensuring that any errors during conversion result in `NaT` (Not a Time) values without interrupting the workflow.
+
+3. **Dataframe Copy**: A copy of the original dataframe `df` is created to preserve the original data during preprocessing.
+
+4. **Identify Column Types**:
+   - **Categorical Columns**: Identified using `select_dtypes` to include data types `object` and `category`.
+   - **Numeric Columns**: Identified by selecting columns with numeric data types using `np.number`.
+
+5. **Handle Missing Values**:
+   - **Numeric Columns**: Missing values are filled with the median of each numeric column to maintain the central tendency without being skewed by outliers.
+   - **Categorical Columns**: Missing values are filled with the mode (most frequent value) to preserve the most common category.
+
+6. **Datetime Conversion**: Although the current dataset does not contain date columns, the preparation is made to handle such conversions if needed in the future by checking for date-specific columns and applying the `safe_to_datetime` function.
+
+7. **Correlation Matrix**: A correlation matrix is generated for the numeric columns to identify the strength and direction of relationships between different numerical variables, which is crucial for understanding how features interact with each other.
+
+8. **Display Outputs**: The first few rows of the processed dataframe and the correlation matrix are printed to provide an initial view of the data and its interrelationships, laying the groundwork for deeper exploratory data analysis.
+
+
+## Statistical Analytics Agent
+
+### Reasoning
+To extract key insights from the Housing Dataset, we will perform a multiple linear regression analysis. This will allow us to identify which factors significantly influence housing prices. By analyzing both numerical variables (such as area, bedrooms, bathrooms) and categorical variables (like mainroad, furnishingstatus), we can determine the key drivers of price variations. Proper handling of missing values and categorical data will ensure the robustness of our analysis.
+
+### Code Implementation
+```python
+import pandas as pd
+import numpy as np
+import statsmodels.api as sm
+
+# Create a copy of the dataframe
+df_copy = df.copy()
+
+try:
+    # Define the dependent variable
+    y = df_copy['price']
+    
+    # Define the independent variables
+    X = df_copy.drop('price', axis=1)
+    
+    # Check for missing values and drop rows with any missing values
+    X = X.dropna()
+    y = y.loc[X.index]
+    
+    # Convert categorical variables to 'category' dtype
+    categorical_columns = X.select_dtypes(include=['object', 'category']).columns
+    for col in categorical_columns:
+        X[col] = X[col].astype('category')
+    
+    # Add a constant term to the predictors
+    X = sm.add_constant(X)
+    
+    # Fit the OLS regression model
+    model = sm.OLS(y.astype(float), X.astype(float)).fit()
+    
+    # Print the model summary
+    print(model.summary())
+    
+except Exception as e:
+    print(f"An error occurred: {e}")
+```
+
+### Summary
+The code performs a multiple linear regression analysis to identify the key factors affecting housing prices. It begins by creating a copy of the original dataframe to preserve the data. The dependent variable `price` is separated from the independent variables. Rows with any missing values are removed to ensure data quality. Categorical variables are converted to the 'category' data type, which allows statsmodels to handle them appropriately in the regression. A constant term is added to the independent variables to account for the intercept in the model. The Ordinary Least Squares (OLS) regression model is then fitted using the prepared data. Finally, the summary of the regression model is printed, providing detailed statistics on the significance and impact of each predictor on the housing price.
+
+
+## Data Viz Agent
+
+### Reasoning
+To extract key insights from the housing dataset, it's important to analyze the relationship between house area and price, as well as understand how factors like the number of bedrooms and bathrooms influence pricing. A scatter plot of area versus price can reveal trends and correlations, while color-coding by the number of bedrooms can provide additional depth to the analysis.
+
+### Code Implementation
+```python
+import plotly.express as px
+
+# Check if the dataset has more than 50,000 rows and sample if necessary
+if len(df) > 50000:
+    df_sample = df.sample(n=50000)
+else:
+    df_sample = df.copy()
+
+# Create a scatter plot of area vs price, colored by number of bedrooms
+fig = px.scatter(df_sample, 
+                 x='area', 
+                 y='price', 
+                 color='bedrooms', 
+                 title='Housing Price vs Area Colored by Number of Bedrooms', 
+                 labels={'area': 'Area (sq ft)', 'price': 'Price'},
+                 hover_data=['bathrooms', 'stories'])
+
+# Update layout for clarity and styling
+fig.update_layout(
+    template='plotly_white',
+    xaxis_title='Area (sq ft)',
+    yaxis_title='Price',
+    legend_title='Bedrooms'
+)
+
+fig.to_html(full_html=False)
+```
+
+### Summary
+The scatter plot visualizes the relationship between the area of the houses and their prices. By color-coding the points based on the number of bedrooms, it becomes easier to observe how bedroom count correlates with both area and price. This helps in identifying trends such as whether larger houses with more bedrooms tend to be priced higher.
+"""
+
+
+input_tokens = len(ai_manager.tokenizer.encode(input_prompt))
+output_tokens = len(ai_manager.tokenizer.encode(output))
+
+print(f"Input tokens: {input_tokens}")
+print(f"Output tokens: {output_tokens}")

scripts/tier_maker.py

@@ -2,16 +2,8 @@ from src.utils.model_registry import MODEL_COSTS, MODEL_TIERS
 
 # divide models in 3 tiers based on cost per 1k tokens
 # tier 1: < $0.0005
-# tier 2: < $0.001
-# tier 3: > $0.05
-# tier 4: > $0.1
-
-TIERS_COST = {
-    "tier1": 0.0005,
-    "tier2": 0.001,
-    "tier3": 0.05,
-    "tier4": 0.1
-}
+# tier 2: $0.0005 - $0.001
+# tier 3: > $0.001
 
 def get_tier(model_name):
     for provider, models in MODEL_COSTS.items():
@@ -24,7 +16,7 @@ def get_tier_1():
     tier_1 = []
     for provider, models in MODEL_COSTS.items():
         for model, cost in models.items():
-            if cost["input"] + cost["output"] < TIERS_COST["tier1"]:
+            if cost["input"] + cost["output"] < 0.0005:
                 tier_1.append(model)
     return tier_1
 
@@ -32,7 +24,7 @@ def get_tier_2():
     tier_2 = []
     for provider, models in MODEL_COSTS.items():
         for model, cost in models.items():
-            if cost["input"] + cost["output"] >= TIERS_COST["tier1"] and cost["input"] + cost["output"] < TIERS_COST["tier2"]:
+            if cost["input"] + cost["output"] >= 0.0005 and cost["input"] + cost["output"] < 0.001:
                 tier_2.append(model)
     return tier_2
 
@@ -40,17 +32,9 @@ def get_tier_3():
     tier_3 = []
     for provider, models in MODEL_COSTS.items():
         for model, cost in models.items():
-            if cost["input"] + cost["output"] >= TIERS_COST["tier2"] and cost["input"] + cost["output"] < TIERS_COST["tier3"]:
+            if cost["input"] + cost["output"] >= 0.001:
                 tier_3.append(model)
-    return tier_3  
-
-def get_tier_4():
-    tier_4 = []
-    for provider, models in MODEL_COSTS.items():
-        for model, cost in models.items():
-            if cost["input"] + cost["output"] >= TIERS_COST["tier3"]:
-                tier_4.append(model)
-    return tier_4
+    return tier_3   
 
 # Print current tier definitions from registry
 import json
@@ -74,11 +58,6 @@ model_tiers = {
         "name": "Premium",
         "credits": 5,
         "models": get_tier_3()
-    },
-    "tier4": {
-        "name": "Premium Plus",
-        "credits": 10,
-        "models": get_tier_4()
     }
 }
 

scripts/verify_session_state.py

@@ -0,0 +1,76 @@
+import requests
+import uuid
+import time
+
+def test_session_workflow():
+    # Base URL
+    base_url = "http://localhost:8000"
+    
+    # Create a unique session ID
+    session_id = str(uuid.uuid4())
+    print(f"Testing with session ID: {session_id}")
+    
+    # Step 1: Create a user and associate with session
+    print("\n1. Creating user...")
+    username = f"test_user_{session_id[:8]}"
+    email = f"{username}@example.com"
+    login_response = requests.post(
+        f"{base_url}/auth/login",
+        json={"username": username, "email": email, "session_id": session_id}
+    ).json()
+    print(f"Login response: {login_response}")
+    
+    # Step 2: Verify session state
+    print("\n2. Verifying session state...")
+    session_state = requests.get(f"{base_url}/debug/session/{session_id}").json()
+    print(f"Session state: {session_state}")
+    
+    # Step 3: Make a chat request with this session
+    print("\n3. Making a test chat request...")
+    chat_response = requests.post(
+        f"{base_url}/chat/data_explorer",
+        json={"query": "Show me a summary of the dataset"},
+        params={"session_id": session_id}
+    ).json()
+    print(f"Chat response received: {len(str(chat_response))} bytes")
+    
+    # Step 4: Verify session state again
+    print("\n4. Verifying session state after chat...")
+    session_state = requests.get(f"{base_url}/debug/session/{session_id}").json()
+    print(f"Updated session state: {session_state}")
+    
+    # Step 5: Check analytics data
+    print("\n5. Checking analytics data...")
+    time.sleep(1)  # Wait a moment for data to be saved
+    admin_key = "default-admin-key-change-me"  # Adjust to your admin key
+
+    # Check general model usage
+    analytics_response = requests.get(
+        f"{base_url}/analytics/debug/model_usage",
+        headers={"X-Admin-API-Key": admin_key}
+    ).json()
+    print(f"Total records in database: {analytics_response.get('total_records', 0)}")
+
+    if 'sample_records' in analytics_response and analytics_response['sample_records']:
+        print(f"Latest usage records: {analytics_response['sample_records']}")
+    else:
+        print("No sample records found in general model usage")
+
+    # Check user-specific usage
+    user_id = login_response.get('user_id')
+    if user_id:
+        user_usage = requests.get(
+            f"{base_url}/analytics/debug/user_usage/{user_id}",
+            headers={"X-Admin-API-Key": admin_key}
+        ).json()
+        print(f"User {user_id} has {user_usage.get('total_records', 0)} usage records")
+        
+        if user_usage.get('recent_records'):
+            print(f"Recent usage: {user_usage['recent_records']}")
+        else:
+            print(f"No usage records found for user {user_id}")
+
+    print("\nTest completed!")
+
+if __name__ == "__main__":
+    test_session_workflow()

src/agents/deep_agents.py

@@ -1,1110 +0,0 @@
-import asyncio
-import ast
-import json
-import os
-import dspy
-import numpy as np
-import pandas as pd
-from dotenv import load_dotenv
-from src.utils.logger import Logger
-import logging
-import datetime
-import re
-import textwrap
-
-def clean_print_statements(code_block):
-    """
-    This function cleans up any `print()` statements that might contain unwanted `\n` characters.
-    It ensures print statements are properly formatted without unnecessary newlines.
-    """
-    # This regex targets print statements, even if they have newlines inside
-    return re.sub(r'print\((.*?)(\\n.*?)(.*?)\)', r'print(\1\3)', code_block, flags=re.DOTALL)
-
-
-def clean_unicode_chars(text):
-    """
-    Clean Unicode characters that might cause encoding issues.
-    Replaces common Unicode characters with ASCII equivalents.
-    """
-    if not isinstance(text, str):
-        return text
-    
-    # Replace common Unicode characters with ASCII equivalents
-    replacements = {
-        '\u2192': ' -> ',  # Right arrow
-        '\u2190': ' <- ',  # Left arrow
-        '\u2194': ' <-> ', # Left-right arrow
-        '\u2500': '-',     # Box drawing horizontal
-        '\u2502': '|',     # Box drawing vertical
-        '\u2026': '...',   # Ellipsis
-        '\u2013': '-',     # En dash
-        '\u2014': '-',     # Em dash
-        '\u201c': '"',     # Left double quotation mark
-        '\u201d': '"',     # Right double quotation mark
-        '\u2018': "'",     # Left single quotation mark
-        '\u2019': "'",     # Right single quotation mark
-    }
-    
-    for unicode_char, ascii_replacement in replacements.items():
-        text = text.replace(unicode_char, ascii_replacement)
-    
-    # Remove any remaining non-ASCII characters
-    text = text.encode('ascii', 'ignore').decode('ascii')
-    
-    return text
-
-
-def remove_main_block(code):
-    # Match the __main__ block
-    pattern = r'(?m)^if\s+__name__\s*==\s*["\']__main__["\']\s*:\s*\n((?:\s+.*\n?)*)'
-    
-    match = re.search(pattern, code)
-    if match:
-        main_block = match.group(1)
-        
-        # Dedent the code block inside __main__
-        dedented_block = textwrap.dedent(main_block)
-        
-        # Remove \n from any print statements in the block (also handling multiline print cases)
-        dedented_block = clean_print_statements(dedented_block)
-        # Replace the block in the code
-        cleaned_code = re.sub(pattern, dedented_block, code)
-        
-        # Optional: Remove leading newlines if any
-        cleaned_code = cleaned_code.strip()
-        
-        return cleaned_code
-    return code
-
-
-# Configure Plotly to prevent auto-display
-def configure_plotly_no_display():
-    """Configure Plotly to prevent automatic browser display"""
-    try:
-        import plotly.io as pio
-        
-        # Set environment variables to prevent browser opening
-        os.environ['BROWSER'] = ''
-        os.environ['PLOTLY_RENDERER'] = 'json'
-        
-        # Configure Plotly renderers
-        pio.renderers.default = 'json'
-        pio.templates.default = 'plotly_white'
-        
-        # Disable Kaleido auto-display if available
-        try:
-            import plotly.graph_objects as go
-            # Configure figure defaults to not auto-display
-            go.Figure.show = lambda self, *args, **kwargs: None
-        except ImportError:
-            pass
-            
-    except ImportError:
-        print("Warning: Plotly not available for configuration")
-
-# Call the configuration function immediately
-configure_plotly_no_display()
-
-logger = Logger("deep_agents", see_time=True, console_log=True, level=logging.DEBUG)
-load_dotenv()
-
-class deep_questions(dspy.Signature):
-    """
-You are a data analysis assistant.
-Your role is to take a user's high-level analytical goal and generate a set of deep, targeted follow-up questions. These questions should guide an analyst toward a more thorough understanding of the goal by encouraging exploration, segmentation, and causal reasoning.
-Instructions:
-- Generate up to 5 insightful, data-relevant questions.
-- Use the dataset structure to tailor your questions (e.g., look at the available columns, data types, and what kind of information they can reveal).
-- The questions should help the user decompose their analytic goal and explore it from multiple angles (e.g., time trends, customer segments, usage behavior, external factors, feedback).
-- Each question should be specific enough to guide actionable analysis or investigation.
-- Use a clear and concise style, but maintain depth.
-Inputs:
-- goal: The user's analytical goal or main question they want to explore
-- dataset_info: A description of the dataset the user is querying, including:
-    - What the dataset represents
-    - Key columns and their data types
-Output:
-- deep_questions: A list of up to 5 specific, data-driven questions that support the analytic goal
----
-Example:
-Analytical Goal:
-Understand why churn has been rising
-Dataset Info:
-Customer Retention Dataset tracking subscription activity over time.  
-Columns:
-- customer_id (string)
-- join_date (date)
-- churn_date (date, nullable)
-- is_churned (boolean)
-- plan_type (string: 'basic', 'premium', 'enterprise')
-- region (string)
-- last_login_date (date)
-- avg_weekly_logins (float)
-- support_tickets_last_30d (int)
-- satisfaction_score (float, 0–10 scale)
-Decomposed Questions:
-1. How has the churn rate changed month-over-month, and during which periods was the increase most pronounced?
-2. Are specific plan types or regions showing a higher churn rate relative to others?
-3. What is the average satisfaction score and support ticket count among churned users compared to retained users?
-4. Do churned users exhibit different login behavior (e.g., avg_weekly_logins) in the weeks leading up to their churn date?
-5. What is the tenure distribution (time from join_date to churn_date) among churned customers, and are short-tenure users more likely to churn?
-    """
-    goal = dspy.InputField(desc="User analytical goal — what main insight or question they want to answer")
-    dataset_info = dspy.InputField(desc="A description of the dataset: what it represents, and the main columns with data types")
-    deep_questions:str = dspy.OutputField(desc="A list of up to five questions that help deeply explore the analytical goal using the dataset")
-
-class deep_synthesizer(dspy.Signature):
-    """
-You are a data analysis synthesis expert.
-Your job is to take the outputs from a multi-agent data analytics system - including the original user query, the code summaries from each agent, and the actual printed results from running those code blocks - and synthesize them into a comprehensive, well-structured final report.
-This report should:
-- Explain what steps were taken and why (based on the query)
-- Summarize the code logic used by each agent, without including raw code
-- Highlight key findings and results from the code outputs
-- Offer clear, actionable insights tied back to the user's original question
-- Be structured, readable, and suitable for decision-makers or analysts
-Instructions:
-- Begin with a brief restatement of the original query and what it aimed to solve
-- Organize your report step-by-step or by analytical theme (e.g., segmentation, trend analysis, etc.)
-- For each part, summarize what was analyzed, how (based on code summaries), and what the result was (based on printed output)
-- End with a final set of synthesized conclusions and potential next steps or recommendations
-Inputs:
-- query: The user's original analytical question or goal
-- summaries: A list of natural language descriptions of what each agent's code did
-- print_outputs: A list of printed outputs (results) from running each agent's code
-Output:
-- synthesized_report: A structured and readable report that ties all parts together, grounded in the code logic and results
-Example use:
-You are not just summarizing outputs - you're telling a story that answers the user's query using real data.
-    """
-
-    query = dspy.InputField(desc="The original user query or analytical goal")
-    summaries = dspy.InputField(desc="List of code summaries - each describing what a particular agent's code did")
-    print_outputs = dspy.InputField(desc="List of print outputs - the actual data insights generated by the code")
-    synthesized_report = dspy.OutputField(desc="The final, structured report that synthesizes all the information into clear insights")
-
-def clean_and_store_code(code, session_datasets=None):
-    """
-    Cleans and stores code execution results in a standardized format.
-    
-    Args:
-        code (str): Raw code text to execute
-        session_datasets (dict): Dictionary of datasets from session state
-        
-    Returns:
-        dict: Execution results containing printed_output, plotly_figs, and error info
-    """
-    import io
-    import sys
-    import re
-    import plotly.express as px
-    import plotly.graph_objects as go
-    from plotly.subplots import make_subplots
-    import plotly.io as pio
-    
-    # Make all session datasets available globally if provided
-    if session_datasets is not None:
-        for dataset_name, dataset_df in session_datasets.items():
-            globals()[dataset_name] = dataset_df
-    
-    # Initialize output containers
-    output_dict = {
-        'exec_result': None,
-        'printed_output': '',
-        'plotly_figs': [],
-        'error': None
-    }
-    
-    try:
-        # Clean the code
-        cleaned_code = code.strip()
-        
-        cleaned_code = cleaned_code.replace('```python', '').replace('```', '')
-
-    
-        # Fix try statement syntax
-        cleaned_code = cleaned_code.replace('try\n', 'try:\n')
-    
-        # Remove code patterns that would make the code unrunnable
-        invalid_patterns = [
-            '```', # Code block markers
-            '\\n', # Raw newlines
-            '\\t', # Raw tabs
-            '\\r', # Raw carriage returns
-        ]
-        
-        for pattern in invalid_patterns:
-            if pattern in cleaned_code:
-                cleaned_code = cleaned_code.replace(pattern, '')
-        
-        
-        # Remove reading the csv file if it's already in the context
-        cleaned_code = re.sub(r"df\s*=\s*pd\.read_csv\([\"\'].*?[\"\']\).*?(\n|$)", '', cleaned_code)
-        
-        # Only match assignments at top level (not indented)
-        # 1. Remove 'df = pd.DataFrame()' if it's at the top level
-        cleaned_code = re.sub(
-            r"^df\s*=\s*pd\.DataFrame\(\s*\)\s*(#.*)?$",
-            '',
-        cleaned_code,
-            flags=re.MULTILINE
-        )
-        cleaned_code = re.sub(r"plt\.show\(\).*?(\n|$)", '', cleaned_code)
-        # Remove all .show() method calls more comprehensively
-        cleaned_code = re.sub(r'\b\w*\.show\(\)', '', cleaned_code)
-        cleaned_code = re.sub(r'^\s*\w*fig\w*\.show\(\)\s*;?\s*$', '', cleaned_code, flags=re.MULTILINE)
-        
-        # Additional patterns to catch more .show() variations
-        cleaned_code = re.sub(r'\.show\(\s*\)', '', cleaned_code)  # .show() with optional spaces
-        cleaned_code = re.sub(r'\.show\(\s*renderer\s*=\s*[\'"][^\'\"]*[\'"]\s*\)', '', cleaned_code)  # .show(renderer='...')
-        cleaned_code = re.sub(r'plotly_figs\[\d+\]\.show\(\)', '', cleaned_code)  # plotly_figs[0].show()
-        
-        # More comprehensive patterns
-        cleaned_code = re.sub(r'\.show\([^)]*\)', '', cleaned_code)  # .show(any_args)
-        cleaned_code = re.sub(r'fig\w*\.show\(\s*[^)]*\s*\)', '', cleaned_code)  # fig*.show(any_args)
-        cleaned_code = re.sub(r'\w+_fig\w*\.show\(\s*[^)]*\s*\)', '', cleaned_code)  # *_fig*.show(any_args)
-        
-        cleaned_code = remove_main_block(cleaned_code)
-        
-        # Clean Unicode characters that might cause encoding issues
-        cleaned_code = clean_unicode_chars(cleaned_code)
-        
-        # Capture printed output
-        old_stdout = sys.stdout
-        captured_output = io.StringIO()
-        sys.stdout = captured_output
-        
-        # Create execution environment with common imports and session data
-        exec_globals = {
-            '__builtins__': __builtins__,
-            'pd': __import__('pandas'),
-            'np': __import__('numpy'),
-            'px': px,
-            'go': go,
-            'make_subplots': make_subplots,
-            'plotly_figs': [],
-            'print': print,
-        }
-        
-        # Add session DataFrame if available
-        if session_datasets is not None:
-            for dataset_name, dataset_df in session_datasets.items():
-                exec_globals[dataset_name] = dataset_df
-        
-        # Add other common libraries that might be needed
-        try:
-            exec_globals['sm'] = __import__('statsmodels.api', fromlist=[''])
-            exec_globals['train_test_split'] = __import__('sklearn.model_selection', fromlist=['train_test_split']).train_test_split
-            exec_globals['LinearRegression'] = __import__('sklearn.linear_model', fromlist=['LinearRegression']).LinearRegression
-            exec_globals['mean_absolute_error'] = __import__('sklearn.metrics', fromlist=['mean_absolute_error']).mean_absolute_error
-            exec_globals['r2_score'] = __import__('sklearn.metrics', fromlist=['r2_score']).r2_score
-            exec_globals['LabelEncoder'] = __import__('sklearn.preprocessing', fromlist=['LabelEncoder']).LabelEncoder
-            exec_globals['warnings'] = __import__('warnings')
-        except ImportError as e:
-            print(f"Warning: Could not import some optional libraries: {e}")
-        
-        # Execute the code
-        exec(cleaned_code, exec_globals)
-        
-        # Restore stdout
-        sys.stdout = old_stdout
-        
-        # Get the captured output
-        printed_output = captured_output.getvalue()
-        output_dict['printed_output'] = printed_output
-        # Extract plotly figures from the execution environment
-        if 'plotly_figs' in exec_globals:
-            plotly_figs = exec_globals['plotly_figs']
-            if isinstance(plotly_figs, list):
-                output_dict['plotly_figs'] = plotly_figs
-            else:
-                output_dict['plotly_figs'] = [plotly_figs] if plotly_figs else []
-        
-        # Also check for any figure variables that might have been created
-        for var_name, var_value in exec_globals.items():
-            if hasattr(var_value, 'to_json') and hasattr(var_value, 'show'):
-                # This looks like a Plotly figure
-                if var_value not in output_dict['plotly_figs']:
-                    output_dict['plotly_figs'].append(var_value)
-        
-    except Exception as e:
-        # Restore stdout in case of error
-        sys.stdout = old_stdout
-        error_msg = str(e)
-        output_dict['error'] = error_msg
-        output_dict['printed_output'] = f"Error executing code: {error_msg}"
-        print(f"Code execution error: {error_msg}")
-        
-    return output_dict
-
-def score_code(args, code, datasets=None):
-    """
-    Cleans and stores code execution results in a standardized format.
-    Safely handles execution errors and returns clean output even if execution fails.
-    Ensures plotly figures are properly created and captured.
-    
-    Args:
-        args: Arguments (unused but required for dspy.Refine)
-        code: Code object with combined_code attribute
-        datasets: Dictionary of datasets from session state (optional)
-        
-    Returns:
-        int: Score (0=error, 1=success, 2=success with plots)
-    """
-
-    code_text = code.combined_code
-    try:
-        # Fix try statement syntax
-        code_text = code_text.replace('try\n', 'try:\n')
-        code_text = code_text.replace('```python', '').replace('```', '')
-        
-        
-        # Remove code patterns that would make the code unrunnable
-        invalid_patterns = [
-            '```', '\\n', '\\t', '\\r'
-        ]
-        
-        for pattern in invalid_patterns:
-            if pattern in code_text:
-                code_text = code_text.replace(pattern, '')
-
-        cleaned_code = re.sub(r"plt\.show\(\).*?(\n|$)", '', code_text)
-        # Remove all .show() method calls more comprehensively
-        cleaned_code = re.sub(r'\b\w*\.show\(\)', '', cleaned_code)
-        cleaned_code = re.sub(r'^\s*\w*fig\w*\.show\(\)\s*;?\s*$', '', cleaned_code, flags=re.MULTILINE)
-        
-        # Additional patterns to catch more .show() variations
-        cleaned_code = re.sub(r'\.show\(\s*\)', '', cleaned_code)  # .show() with optional spaces
-        cleaned_code = re.sub(r'\.show\(\s*renderer\s*=\s*[\'"][^\'\"]*[\'"]\s*\)', '', cleaned_code)  # .show(renderer='...')
-        cleaned_code = re.sub(r'plotly_figs\[\d+\]\.show\(\)', '', cleaned_code)  # plotly_figs[0].show()
-        
-        # More comprehensive patterns
-        cleaned_code = re.sub(r'\.show\([^)]*\)', '', cleaned_code)  # .show(any_args)
-        cleaned_code = re.sub(r'fig\w*\.show\(\s*[^)]*\s*\)', '', cleaned_code)  # fig*.show(any_args)
-        cleaned_code = re.sub(r'\w+_fig\w*\.show\(\s*[^)]*\s*\)', '', cleaned_code)  # *_fig*.show(any_args)
-            
-        cleaned_code = remove_main_block(cleaned_code)
-        
-        # Capture stdout using StringIO
-        from io import StringIO
-        import sys
-        import plotly.graph_objects as go
-        import pandas as pd
-        import numpy as np
-        
-        stdout_capture = StringIO()
-        original_stdout = sys.stdout
-        sys.stdout = stdout_capture
-        
-        # Execute code in a new namespace with datasets available
-        local_vars = {}
-        
-        # Add datasets to the execution context if provided
-        if datasets:
-            local_vars.update(datasets)
-        
-        # Add common imports to the execution context
-        local_vars.update({
-            'pd': pd,
-            'np': np,
-            'go': go,
-            'plt': __import__('matplotlib.pyplot'),
-            'sns': __import__('seaborn'),
-        })
-        
-        exec(cleaned_code, globals(), local_vars)
-        
-        # Capture any plotly figures from local namespace
-        plotly_figs = []
-        for var_name, var in local_vars.items():
-            if isinstance(var, go.Figure):
-                if not var.layout.title:
-                    var.update_layout(title=f"Figure {len(plotly_figs) + 1}")
-                if not var.layout.template:
-                    var.update_layout(template="plotly_white")
-                plotly_figs.append(var)
-            elif isinstance(var, (list, tuple)):
-                for item in var:
-                    if isinstance(item, go.Figure):
-                        if not item.layout.title:
-                            item.update_layout(title=f"Figure {len(plotly_figs) + 1}")
-                        if not item.layout.template:
-                            item.update_layout(template="plotly_white")
-                        plotly_figs.append(item)
-        
-        # Restore stdout and get captured output
-        sys.stdout = original_stdout
-        captured_output = stdout_capture.getvalue()
-        stdout_capture.close()
-        
-        # Calculate score based on execution and plot generation
-        score = 2 if plotly_figs else 1
-        
-        return score
-    
-    except Exception as e:
-        # Restore stdout in case of error
-        if 'stdout_capture' in locals():
-            sys.stdout = original_stdout
-            stdout_capture.close()
-            
-        return 0
-    
-
-class deep_planner(dspy.Signature):
-    """
-    You are an advanced multi-question planning agent. Your task is to generate the most optimized and minimal plan
-    to answer up to 5 analytical questions using available agents.
-    Your responsibilities:
-    1. Feasibility: Verify that the goal is achievable using the provided datasets and agent descriptions.
-    2. Optimization: 
-       - Batch up to 2 similar questions per agent call.
-       - Reuse outputs across questions wherever possible.
-       - Avoid unnecessary agents or redundant processing.
-       - Minimize total agent calls while preserving correctness.
-    3. Clarity: 
-       - Define clear variable usage (create/use).
-       - Specify concise step-by-step instructions per agent.
-       - Use dependency arrows (->) to indicate required agent outputs used by others.
-    Inputs:
-    - deep_questions: A list of up to 5 deep analytical questions (e.g., ["q1", "q2", ..., "q5"])
-    - dataset: The available dataset(s) in memory or context
-    - agents_desc: Dictionary containing each agent's name and its capabilities or descriptions
-    Outputs:
-    - plan_instructions: Detailed per-agent variable flow and functionality in the format:
-        {
-            "agent_x": {
-                "create": ["cleaned_data: DataFrame - cleaned version of the input dataset"],
-                "use": ["df: DataFrame - raw input dataset"],
-                "instruction": "Clean the dataset by handling null values and standardizing formats."
-            },
-            "agent_y": {
-                "create": ["analysis_results: dict - results of correlation analysis"],
-                "use": ["cleaned_data: DataFrame - output from @agent_x"],
-                "instruction": "Perform correlation analysis to identify strong predictors."
-            }
-        }
-    Output Goal:
-    Generate a small, clean, optimized execution plan using minimal agent calls, reusable outputs, and well-structured dependencies.
-    USE THE EXACT NAME OF THE AGENTS IN THE INSTRUCTIONS
-    """
-
-    deep_questions = dspy.InputField(desc="List of up to 5 deep analytical questions to answer")
-    dataset = dspy.InputField(desc="Available datasets, use 'df' as the working dataset")
-    agents_desc = dspy.InputField(desc="Descriptions of available agents and their functions")
-    plan_instructions = dspy.OutputField(desc="Variable-level instructions for each agent used in the plan")
-
-class deep_plan_fixer(dspy.Signature):
-    """
-    You are a plan instruction fixer agent. Your task is to take potentially malformed plan instructions
-    and convert them into a properly structured dictionary format that can be safely evaluated.
-    Your responsibilities:
-    1. Parse and validate the input plan instructions
-    2. Convert the instructions into a proper dictionary format
-    3. Ensure all agent instructions follow the required structure:
-       {
-           "@agent_name": {
-               "create": ["variable: type - description"],
-               "use": ["variable: type - description"],
-               "instruction": "clear instruction text"
-           }
-       }
-    4. Handle any malformed or missing components
-    5. Return a properly formatted dictionary string that can be safely evaluated
-    Inputs:
-    - plan_instructions: The potentially malformed plan instructions to fix
-    Outputs:
-    - fixed_plan: A properly formatted dictionary string that can be safely evaluated
-    """
-
-    plan_instructions = dspy.InputField(desc="The potentially malformed plan instructions to fix")
-    fixed_plan = dspy.OutputField(desc="Properly formatted dictionary string that can be safely evaluated")
-
-class final_conclusion(dspy.Signature):
-    """
-You are a high-level analytics reasoning engine.
-Your task is to take multiple synthesized analytical results (each answering part of the original query) and produce a cohesive final conclusion that directly addresses the user's original question.
-This is not just a summary — it's a judgment. Use evidence from the synthesized findings to:
-- Answer the original question with clarity
-- Highlight the most important insights
-- Offer any causal reasoning or patterns discovered
-- Suggest next steps or strategic recommendations where appropriate
-Instructions:
-- Focus on relevance to the original query
-- Do not just repeat what the synthesized sections say — instead, infer, interpret, and connect dots
-- Prioritize clarity and insight over detail
-- End with a brief "Next Steps" section if applicable
-Inputs:
-- query: The original user question or goal
-- synthesized_sections: A list of synthesized result sections from the deep_synthesizer step (each covering part of the analysis)
-Output:
-- final_summary: A cohesive final conclusion that addresses the query, draws insight, and offers high-level guidance
----
-Example Output Structure:
-**Conclusion**  
-Summarize the overall answer to the user's question, using the most compelling evidence across the synthesized sections.
-**Key Takeaways**  
-- Bullet 1  
-- Bullet 2  
-- Bullet 3  
-**Recommended Next Steps**  
-(Optional based on context)
-    """
-
-    query = dspy.InputField(desc="The user's original query or analytical goal")
-    synthesized_sections = dspy.InputField(desc="List of synthesized outputs — each one corresponding to a sub-part of the analysis")
-    final_conclusion = dspy.OutputField(desc="A cohesive, conclusive answer that addresses the query and integrates key insights")
-
-
-
-
-class deep_code_synthesizer(dspy.Signature):
-    """
-You are a code synthesis and optimization engine that combines and fixes code from multiple analytical agents.
-Your task is to take code outputs from preprocessing, statistical analysis, machine learning, and visualization agents, then:
-- Combine them into a single, coherent analysis pipeline
-- Fix any errors or inconsistencies between agent outputs
-- Ensure proper data flow between steps
-- Optimize the combined code for efficiency
-- Add necessary imports and dependencies
-- Handle any data type mismatches or conversion issues
-- Validate and normalize data types between agent outputs (e.g., ensure DataFrame operations maintain DataFrame type)
-- Convert between common data structures (lists, dicts, DataFrames) as needed
-- Add type hints and validation checks
-- Ensure consistent variable naming across agents
-- Ensure all visualizations use Plotly exclusively
-- Create comprehensive visualizations that show all important variables and relationships
-- Store all Plotly figures in a list for later use in the report
-Instructions:
-- Review each agent's code for correctness and completeness
-- Ensure variables are properly passed between steps with consistent types
-- Fix any syntax errors or logical issues
-- Add error handling and type validation where needed
-- Optimize code structure and performance
-- Maintain consistent coding style
-- Add clear comments explaining the analysis flow
-- Add data type conversion functions where needed
-- Validate input/output types between agent steps
-- Handle edge cases where agents might return different data structures
-- Convert any non-Plotly visualizations to Plotly format
-- Ensure all important variables are visualized appropriately
-- Store all Plotly figures in a list called plotly_figs
-- Include appropriate titles, labels, and legends for all visualizations
-- Use consistent styling across all Plotly visualizations
-- DONOT COMMENT OUT ANYTHING AS THE CODE SHOULD RUN & SHOW OUTPUTS
-- THE DATASET IS ALREADY LOADED, DON'T CREATE FAKE DATA. 'df' is always loaded
-Inputs:
-- deep_questions- The five deep questions this system is answering
-- dataset_info - Information about the dataset structure and types
-- planner_instructions - the plan according to the planner, ensure that the final code makes everything coherent
-- code - List of all agent code
-Output:
-- combined_code: - A single, optimized Python script that combines all analysis steps with proper type handling and Plotly visualizations
-"""
-    deep_questions = dspy.InputField(desc="The five deep questions this system is answering")
-    dataset_info = dspy.InputField(desc="Information about the dataset")
-    planner_instructions = dspy.InputField(desc="The planner instructions for each")
-    code = dspy.InputField(desc="The code generated by all agents")
-    combined_code = dspy.OutputField(desc="A single, optimized Python script that combines all analysis steps")
-
-class deep_code_fix(dspy.Signature):
-    """
-    You are a code debugging and fixing agent that analyzes and repairs code errors.
-    
-    Your task is to:
-    - Analyze error messages and identify root causes
-    - Fix syntax errors, logical issues, and runtime problems
-    - Ensure proper data type handling and conversions
-    - Add appropriate error handling and validation
-    - Maintain code style and documentation
-    - Preserve the original analysis intent
-    
-    Instructions:
-    - Carefully analyze the error message and stack trace
-    - Identify the specific line(s) causing the error
-    - Determine if the issue is syntax, logic, or runtime related
-    - Fix the code while maintaining its original purpose
-    - Add appropriate error handling if needed
-    - Ensure the fix doesn't introduce new issues
-    - Document the changes made
-    
-    Inputs:
-    - code: The code that generated the error
-    - error: The error message and stack trace
-    
-    Output:
-    - fixed_code: The repaired code with error handling
-    - fix_explanation: Explanation of what was fixed and why
-    """
-    code = dspy.InputField(desc="The code that generated the error")
-    error = dspy.InputField(desc="The error message and stack trace")
-    fixed_code = dspy.OutputField(desc="The repaired code with error handling")
-    fix_explanation = dspy.OutputField(desc="Explanation of what was fixed and why")
-
-
-chart_instructions = """
-Chart Styling Guidelines:
-1. General Styling:
-   - Use a clean, professional color palette (e.g., Tableau, ColorBrewer)
-   - Include clear titles and axis labels
-   - Add appropriate legends
-   - Use consistent font sizes and styles
-   - Include grid lines where helpful
-   - Add hover information for interactive plots
-2. Specific Chart Types:
-   - Bar Charts:
-     * Use horizontal bars for many categories
-     * Sort bars by value when appropriate
-     * Use consistent bar widths
-     * Add value labels on bars
-   
-   - Line Charts:
-     * Use distinct line styles/colors
-     * Add markers at data points
-     * Include trend lines when relevant
-     * Show confidence intervals if applicable
-   
-   - Scatter Plots:
-     * Use appropriate marker sizes
-     * Add regression lines when needed
-     * Use color to show additional dimensions
-     * Include density contours for large datasets
-   
-   - Heatmaps:
-     * Use diverging color schemes for correlation
-     * Include value annotations
-     * Sort rows/columns by similarity
-     * Add clear color scale legend
-3. Data Visualization Best Practices:
-   - Start axes at zero when appropriate
-   - Use log scales for wide-ranging data
-   - Include reference lines/benchmarks
-   - Add annotations for important points
-   - Show uncertainty where relevant
-   - Use consistent color encoding
-   - Include data source and timestamp
-   - Add clear figure captions
-4. Interactive Features:
-   - Enable zooming and panning
-   - Add tooltips with detailed information
-   - Include download options
-   - Allow toggling of data series
-   - Enable cross-filtering between charts
-5. Accessibility:
-   - Use colorblind-friendly palettes
-   - Include alt text for all visualizations
-   - Ensure sufficient contrast
-   - Make interactive elements keyboard accessible
-   - Provide text alternatives for key insights
-"""
-
-
-
-
-class deep_analysis_module(dspy.Module):
-    def __init__(self,agents, agents_desc):
-        # logger.log_message(f"Initializing deep_analysis_module with {agents} agents: {list(agents.keys())}", level=logging.INFO)
-        
-        # self.agent = agents
-        self.agents = {}
-        for key in agents.keys():
-            self.agents[key] = dspy.asyncify(dspy.Predict(agents[key]))
-
-        # for sig in self.agen
-        # Make all dspy operations async using asyncify
-        self.deep_questions = dspy.asyncify(dspy.Predict(deep_questions))
-        self.deep_planner = dspy.asyncify(dspy.Predict(deep_planner))
-        self.deep_synthesizer = dspy.asyncify(dspy.Predict(deep_synthesizer))
-        # Keep both asyncified and non-asyncified versions for code synthesizer
-        self.deep_code_synthesizer_sync = dspy.Predict(deep_code_synthesizer)  # For dspy.Refine
-        self.deep_code_synthesizer = dspy.asyncify(dspy.Predict(deep_code_synthesizer))  # For async use
-        self.deep_plan_fixer = dspy.asyncify(dspy.Predict(deep_plan_fixer))
-        self.deep_code_fixer = dspy.asyncify(dspy.Predict(deep_code_fix))
-        self.styling_instructions = chart_instructions
-        self.agents_desc = agents_desc
-        self.final_conclusion = dspy.asyncify(dspy.ChainOfThought(final_conclusion))
-        
-        # logger.log_message(f"Deep analysis module initialized successfully with agents: {list(self.agents.keys())}", level=logging.INFO)
-
-    async def execute_deep_analysis_streaming(self, goal, dataset_info, session_datasets=None):
-        """
-        Execute deep analysis with streaming progress updates.
-        This is an async generator that yields progress updates incrementally.
-        """
-        logger.log_message(f"🔍 DEEP ANALYSIS STREAMING START - goal: {goal[:100]}...", level=logging.DEBUG)
-        logger.log_message(f"🔍 DEEP ANALYSIS STREAMING START - dataset_info type: {type(dataset_info)}, length: {len(dataset_info) if isinstance(dataset_info, str) else 'N/A'}", level=logging.DEBUG)
-        logger.log_message(f"🔍 DEEP ANALYSIS STREAMING START - session_datasets type: {type(session_datasets)}, keys: {list(session_datasets.keys()) if session_datasets else 'None'}", level=logging.DEBUG)
-        
-        # Make all session datasets available globally for code execution
-        if session_datasets is not None:
-            for dataset_name, dataset_df in session_datasets.items():
-                globals()[dataset_name] = dataset_df
-                logger.log_message(f"🔍 MADE DATASET AVAILABLE GLOBALLY - {dataset_name}: shape {dataset_df.shape}", level=logging.DEBUG)
-        
-        try:
-            # Step 1: Generate deep questions (20% progress)
-            yield {
-                "step": "questions",
-                "status": "processing",
-                "message": "Generating analytical questions...",
-                "progress": 10
-            }
-            
-            logger.log_message(f"🔍 CALLING DEEP_QUESTIONS - dataset_info type: {type(dataset_info)}, length: {len(dataset_info) if isinstance(dataset_info, str) else 'N/A'}", level=logging.DEBUG)
-            questions = await self.deep_questions(goal=goal, dataset_info=dataset_info)
-            logger.log_message("Questions generated")
-            
-            yield {
-                "step": "questions", 
-                "status": "completed",
-                "content": questions.deep_questions,
-                "progress": 20
-            }
-            
-            # Step 2: Create analysis plan (40% progress)
-            yield {
-                "step": "planning",
-                "status": "processing", 
-                "message": "Creating analysis plan...",
-                "progress": 25
-            }
-            
-            question_list = [q.strip() for q in questions.deep_questions.split('\n') if q.strip()]
-            logger.log_message(f" CALLING DEEP_PLANNER - dataset_info type: {type(dataset_info)}, length: {len(dataset_info) if isinstance(dataset_info, str) else 'N/A'}", level=logging.DEBUG)
-            deep_plan = await self.deep_planner(
-                deep_questions=questions.deep_questions, 
-                dataset=dataset_info, 
-                agents_desc=str(self.agents_desc)
-            )
-            logger.log_message("Plan created")
-            
-            # Parse plan instructions
-            try:
-                plan_instructions = ast.literal_eval(deep_plan.plan_instructions)
-                if not isinstance(plan_instructions, dict):
-                    plan_instructions = json.loads(deep_plan.plan_instructions)
-                keys = [key for key in plan_instructions.keys()]
-                
-                if not all(key in self.agents for key in keys):
-                    raise ValueError(f"Invalid agent key(s) in plan instructions. Available agents: {list(self.agents.keys())}")
-                logger.log_message(f"Plan instructions: {plan_instructions}", logging.INFO)
-                logger.log_message(f"Keys: {keys}", logging.INFO)
-            except (ValueError, SyntaxError, json.JSONDecodeError) as e:
-                try:
-                    deep_plan = await self.deep_plan_fixer(plan_instructions=deep_plan.plan_instructions)
-                    plan_instructions = ast.literal_eval(deep_plan.fixed_plan)
-                    if not isinstance(plan_instructions, dict):
-                        plan_instructions = json.loads(deep_plan.fixed_plan)
-                    keys = [key for key in plan_instructions.keys()]
-                except (ValueError, SyntaxError, json.JSONDecodeError) as e:
-                    logger.log_message(f"Error parsing plan instructions: {e}", logging.ERROR)
-                    raise e
-            
-            logger.log_message("Instructions parsed")
-            
-            yield {
-                "step": "planning",
-                "status": "completed",
-                "content": deep_plan.plan_instructions,
-                "progress": 40
-            }
-            
-            # Step 3: Execute agent tasks (60% progress)
-            yield {
-                "step": "agent_execution",
-                "status": "processing",
-                "message": "Executing analysis agents...",
-                "progress": 45
-            }
-            
-            logger.log_message(f"🔍 CREATING DSPY EXAMPLES - dataset_info type: {type(dataset_info)}, length: {len(dataset_info) if isinstance(dataset_info, str) else 'N/A'}", level=logging.DEBUG)
-
-
-            # tasks = [self.agents[key](**q) for q, key in zip(queries, keys)]
-            tasks = []
-            for key in  keys:
-                logger.log_message(f"Preparing agent task for key: {key}", level=logging.DEBUG)
-                logger.log_message(f"Plan instructions for {key}: {plan_instructions[key]}", level=logging.DEBUG)
-                if "data_viz" in key or "viz" in key.lower() or "visual" in key.lower() or "plot" in key.lower() or "chart" in key.lower():
-                    logger.log_message(f"Agent {key} identified as visualization agent. Adding styling_index.", level=logging.DEBUG)
-                    task = self.agents[key](
-                        goal=questions.deep_questions,
-                        dataset=dataset_info,
-                        plan_instructions=str(plan_instructions[key]),
-                        styling_index=self.styling_instructions
-                    )
-                    logger.log_message(f"Visualization agent task for {key} created.", level=logging.DEBUG)
-                    tasks.append(task)
-                else:
-                    logger.log_message(f"Agent {key} identified as non-visualization agent.", level=logging.DEBUG)
-                    task = self.agents[key](
-                        goal=questions.deep_questions,
-                        dataset=dataset_info,
-                        plan_instructions=str(plan_instructions[key])
-                    )
-                    logger.log_message(f"Non-visualization agent task for {key} created.", level=logging.DEBUG)
-                    tasks.append(task)
-
-
-
-
-
-
-            logger.log_message("Passed that stage")
-            # DEBUG: Log what parameters each agent will receive
-                  
-            # Await all tasks to complete
-            summaries = []
-            codes = []
-            logger.log_message("Tasks started")
-            
-            completed_tasks = 0
-            for task in asyncio.as_completed(tasks):
-                result = await task
-                summaries.append(result.summary)
-                codes.append(result.code)
-                completed_tasks += 1
-                
-                # Update progress for each completed agent
-                agent_progress = 45 + (completed_tasks / len(tasks)) * 15  # 45% to 60%
-                yield {
-                    "step": "agent_execution",
-                    "status": "processing",
-                    "message": f"Completed {completed_tasks}/{len(tasks)} analysis agents...",
-                    "progress": int(agent_progress)
-                }
-                logger.log_message(f"Done with agent {completed_tasks}/{len(tasks)}")
-
-            yield {
-                "step": "agent_execution",
-                "status": "completed", 
-                "message": "All analysis agents completed",
-                "progress": 60
-            }
-            
-            # Step 4: Code synthesis (80% progress)
-            yield {
-                "step": "code_synthesis",
-                "status": "processing",
-                "message": "Analyzing code...",
-                "progress": 65
-            }
-            
-            # Safely extract code from agent outputs
-            code = []
-            for c in codes:
-                try:
-                    cleaned_code = remove_main_block(c)
-                    if "```python" in cleaned_code:
-                        parts = cleaned_code.split("```python")
-                        if len(parts) > 1:
-                            extracted = parts[1].split("```")[0] if "```" in parts[1] else parts[1]
-                            code.append(extracted.replace('try\n','try:\n'))
-                        else:
-                            code.append(cleaned_code.replace('try\n','try:\n'))
-                    else:
-                        code.append(cleaned_code.replace('try\n','try:\n'))
-                except Exception as e:
-                    logger.log_message(f"Warning: Error processing code block: {e}", logging.WARNING)
-                    code.append(c.replace('try\n','try:\n'))
-            
-            # Create deep coder without asyncify to avoid source inspection issues
-            def create_score_code_with_datasets(datasets):
-                """
-                Creates a score function that includes the session datasets.
-                
-                Args:
-                    datasets: Dictionary of datasets from session_state['datasets']
-                    
-                Returns:
-                    A reward function compatible with dspy.Refine
-                """
-                def score_code_with_datasets(args, pred):
-                    return score_code(args, pred, datasets)  # FIXED: Use positional argument
-                
-                return score_code_with_datasets
-
-            # Then in your deep analysis method:
-            # Create score function with datasets
-            score_fn = create_score_code_with_datasets(self.datasets)
-            deep_coder = dspy.Refine(module=self.deep_code_synthesizer_sync, N=5, reward_fn=score_fn, threshold=1.0, fail_count=10)
-            
-            # Check if we have valid API key
-            anthropic_key = os.environ.get('ANTHROPIC_API_KEY')
-            if not anthropic_key:
-                raise ValueError("ANTHROPIC_API_KEY environment variable is not set")
-            
-            try:
-                # Create the LM instance that will be used
-                thread_lm = dspy.LM("anthropic/claude-sonnet-4-6", api_key=anthropic_key, max_tokens=17000)
-                
-                logger.log_message("Starting code generation...")
-                start_time = datetime.datetime.now()
-                logger.log_message(f"Code generation started at: {start_time.strftime('%Y-%m-%d %H:%M:%S')}")
-                
-                # Define the blocking function to run in thread
-                def run_deep_coder():
-                    with dspy.context(lm=thread_lm):
-                        return deep_coder(
-                            deep_questions=str(questions.deep_questions), 
-                            dataset_info=dataset_info,
-                            planner_instructions=str(plan_instructions), 
-                            code=str(code)
-                        )
-                
-                # Use asyncio.to_thread for better async integration
-                deep_code = await asyncio.to_thread(run_deep_coder)
-                
-                logger.log_message(f"Code generation completed at: {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
-            except Exception as e:
-                logger.log_message(f"Error during code generation: {str(e)}", logging.ERROR)
-                raise e
-
-            code = deep_code.combined_code
-            code = code.replace('```python', '').replace('```', '')
-            
-            # Clean Unicode characters that might cause encoding issues
-            code = clean_unicode_chars(code)
-            
-            yield {
-                "step": "code_synthesis",
-                "status": "completed",
-                "message": "Code synthesis completed",
-                "progress": 80
-            }
-            
-            # Step 5: Execute code (85% progress)
-            yield {
-                "step": "code_execution",
-                "status": "processing",
-                "message": "Executing code...",
-                "progress": 82
-            }
-            
-            # Execute the code with error handling and session DataFrame
-            try:
-                # Run code execution in thread pool to avoid blocking
-                import concurrent.futures
-                with concurrent.futures.ThreadPoolExecutor() as executor:
-                    future = executor.submit(clean_and_store_code, code, session_datasets)
-                    output = future.result(timeout=300)  # 5 minute timeout
-                
-                logger.log_message(f"Deep Code executed")
-                
-                if output.get('error'):
-                    logger.log_message(f"Warning: Code execution had errors: {output['error']}", logging.ERROR)
-                
-                print_outputs = [output['printed_output']]
-                plotly_figs = [output['plotly_figs']]
-                
-            except Exception as e:
-                logger.log_message(f"Error during code execution: {str(e)}", logging.ERROR)
-                output = {
-                    'exec_result': None,
-                    'printed_output': f"Code execution failed: {str(e)}",
-                    'plotly_figs': [],
-                    'error': str(e)
-                }
-                print_outputs = [output['printed_output']]
-                plotly_figs = [output['plotly_figs']]
-
-            yield {
-                "step": "code_execution",
-                "status": "completed",
-                "message": "Code execution completed",
-                "progress": 85
-            }
-            
-            # Step 6: Synthesis (90% progress)
-            yield {
-                "step": "synthesis",
-                "status": "processing",
-                "message": "Synthesizing results...",
-                "progress": 87
-            }
-            
-            synthesis = []
-            try:
-                synthesis_result = await self.deep_synthesizer(
-                    query=goal, 
-                    summaries=str(summaries), 
-                    print_outputs=str(output['printed_output'])
-                )
-                synthesis.append(synthesis_result)
-            except Exception as e:
-                logger.log_message(f"Error during synthesis: {str(e)}", logging.ERROR)
-                synthesis.append(type('obj', (object,), {'synthesized_report': f"Synthesis failed: {str(e)}"})())
-            
-            logger.log_message("Synthesis done")
-            
-            yield {
-                "step": "synthesis",
-                "status": "completed",
-                "message": "Synthesis completed",
-                "progress": 90
-            }
-            
-            # Step 7: Final conclusion (100% progress)
-            yield {
-                "step": "conclusion",
-                "status": "processing",
-                "message": "Generating final conclusion...",
-                "progress": 95
-            }
-            
-            try:
-                final_conclusion = await self.final_conclusion(
-                    query=goal, 
-                    synthesized_sections=str([s.synthesized_report for s in synthesis])
-                )
-            except Exception as e:
-                logger.log_message(f"Error during final conclusion: {str(e)}", logging.ERROR)
-                final_conclusion = type('obj', (object,), {'final_conclusion': f"Final conclusion failed: {str(e)}"})()
-
-            logger.log_message("Conclusion Made")
-            
-            return_dict = {
-                'goal': goal, 
-                'deep_questions': questions.deep_questions, 
-                'deep_plan': deep_plan.plan_instructions, 
-                'summaries': summaries, 
-                'code': code,
-                'plotly_figs': plotly_figs,
-                'synthesis': [s.synthesized_report for s in synthesis], 
-                'final_conclusion': final_conclusion.final_conclusion 
-            }
-            
-            yield {
-                "step": "conclusion",
-                "status": "completed",
-                "message": "Analysis completed successfully",
-                "progress": 100,
-                "final_result": return_dict
-            }
-            
-            logger.log_message("Return dict created")
-            
-        except Exception as e:
-            logger.log_message(f"Error in deep analysis: {str(e)}", logging.ERROR)
-            yield {
-                "step": "error",
-                "status": "failed",
-                "message": f"Deep analysis failed: {str(e)}",
-                "progress": 0,
-                "error": str(e)
-            }
-
-
-    async def execute_deep_analysis(self, goal, dataset_info, session_df=None):
-        """
-        Legacy method for backward compatibility.
-        Executes the streaming analysis and returns the final result.
-        """
-        final_result = None
-        async for update in self.execute_deep_analysis_streaming(goal, dataset_info, session_df):
-            if update.get("step") == "conclusion" and update.get("status") == "completed":
-                final_result = update.get("final_result")
-            elif update.get("step") == "error":
-                raise Exception(update.get("message", "Unknown error"))
-        
-        return final_result

src/agents/retrievers/retrievers.py

@@ -2,7 +2,7 @@
 
 import pandas as pd
 import numpy as np
-from datetime import datetime
+import datetime
 
 # instructions also stored here
 instructions ="""
@@ -34,7 +34,7 @@ PLEASE READ THE INSTRUCTIONS! Thank you
 def return_vals(df,c):
     if isinstance(df[c].iloc[10], (int, float, complex)):
         return {'max_value':max(df[c]),'min_value': min(df[c]), 'mean_value':np.mean(df[c])}
-    elif(isinstance(df[c].iloc[10],datetime)):
+    elif(isinstance(df[c].iloc[10],datetime.datetime)):
         return {str(max(df[c])), str(min(df[c])), str(np.mean(df[c]))}
     else:
         return {'top_10_values':df[c].value_counts()[:10], 'total_categoy_count':len(df[c].unique())}
@@ -52,8 +52,9 @@ def correct_num(df,c):
 # does most of the pre-processing
 def make_data(df, desc):
     dict_ = {}
-    dict_['dataset_python_name'] = "The data is loaded as df"       
+    dict_['df_name'] = "The data is loaded as df"
     dict_['Description'] = desc
+    dict_['dataframe_head_view'] = df.head(2).to_markdown()
     # dict_['all_column_names'] = str(list(df.columns[:20]))
 
     # for c in df.columns:
@@ -65,3 +66,88 @@ def make_data(df, desc):
     return dict_
 
 
+
+# These are stored styling instructions for data_viz_agent, helps generate good graphs
+styling_instructions =[
+    """
+        Dont ignore any of these instructions.
+        For a line chart always use plotly_white template, reduce x axes & y axes line to 0.2 & x & y grid width to 1. 
+        Always give a title and make bold using html tag axis label and try to use multiple colors if more than one line
+        Annotate the min and max of the line
+        Display numbers in thousand(K) or Million(M) if larger than 1000/100000 
+        Show percentages in 2 decimal points with '%' sign
+        Default size of chart should be height =1200 and width =1000
+        
+        """
+        
+   , """
+        Dont ignore any of these instructions.
+        For a bar chart always use plotly_white template, reduce x axes & y axes line to 0.2 & x & y grid width to 1. 
+        Always give a title and make bold using html tag axis label 
+        Always display numbers in thousand(K) or Million(M) if larger than 1000/100000. 
+        Annotate the values of the bar chart
+        If variable is a percentage show in 2 decimal points with '%' sign.
+        Default size of chart should be height =1200 and width =1000
+        """
+        ,
+
+          """
+        For a histogram chart choose a bin_size of 50
+        Do not ignore any of these instructions
+        always use plotly_white template, reduce x & y axes line to 0.2 & x & y grid width to 1. 
+        Always give a title and make bold using html tag axis label 
+        Always display numbers in thousand(K) or Million(M) if larger than 1000/100000. Add annotations x values
+        If variable is a percentage show in 2 decimal points with '%'
+        Default size of chart should be height =1200 and width =1000
+        """,
+
+
+          """
+        For a pie chart only show top 10 categories, bundle rest as others
+        Do not ignore any of these instructions
+        always use plotly_white template, reduce x & y axes line to 0.2 & x & y grid width to 1. 
+        Always give a title and make bold using html tag axis label 
+        Always display numbers in thousand(K) or Million(M) if larger than 1000/100000. Add annotations x values
+        If variable is a percentage show in 2 decimal points with '%'
+        Default size of chart should be height =1200 and width =1000
+        """,
+
+          """
+        Do not ignore any of these instructions
+        always use plotly_white template, reduce x & y axes line to 0.2 & x & y grid width to 1. 
+        Always give a title and make bold using html tag axis label 
+        Always display numbers in thousand(K) or Million(M) if larger than 1000/100000. Add annotations x values
+        Don't add K/M if number already in , or value is not a number
+        If variable is a percentage show in 2 decimal points with '%'
+        Default size of chart should be height =1200 and width =1000
+        """,
+"""
+    For a heat map
+    Use the 'plotly_white' template for a clean, white background. 
+    Set a chart title 
+    Style the X-axis with a black line color, 0.2 line width, 1 grid width, format 1000/1000000 as K/M
+    Do not format non-numerical numbers 
+    .style the Y-axis with a black line color, 0.2 line width, 1 grid width format 1000/1000000 as K/M
+    Do not format non-numerical numbers 
+
+    . Set the figure dimensions to a height of 1200 pixels and a width of 1000 pixels.
+""",
+"""
+    For a Histogram, used for returns/distribution plotting
+    Use the 'plotly_white' template for a clean, white background. 
+    Set a chart title 
+    Style the X-axis  1 grid width, format 1000/1000000 as K/M
+    Do not format non-numerical numbers 
+    .style the Y-axis, 1 grid width format 1000/1000000 as K/M
+    Do not format non-numerical numbers 
+    
+    Use an opacity of 0.75
+
+     Set the figure dimensions to a height of 1200 pixels and a width of 1000 pixels.
+"""
+       
+         ]
+
+
+
+

src/agents/test_agent.py

@@ -0,0 +1,1223 @@
+import dspy
+import src.agents.memory_agents as m
+import asyncio
+from concurrent.futures import ThreadPoolExecutor
+import os
+from dotenv import load_dotenv
+# import logging
+from src.utils.logger import Logger
+load_dotenv()
+
+logger = Logger("agents", see_time=True, console_log=False)
+
+
+AGENTS_WITH_DESCRIPTION = {
+    "preprocessing_agent": "Cleans and prepares a DataFrame using Pandas and NumPy—handles missing values, detects column types, and converts date strings to datetime.",
+    "statistical_analytics_agent": "Performs statistical analysis (e.g., regression, seasonal decomposition) using statsmodels, with proper handling of categorical data and missing values.",
+    "sk_learn_agent": "Trains and evaluates machine learning models using scikit-learn, including classification, regression, and clustering with feature importance insights.",
+    "data_viz_agent": "Generates interactive visualizations with Plotly, selecting the best chart type to reveal trends, comparisons, and insights based on the analysis goal."
+}
+
+PLANNER_AGENTS_WITH_DESCRIPTION = {
+    "planner_preprocessing_agent": (
+        "Cleans and prepares a DataFrame using Pandas and NumPy—"
+        "handles missing values, detects column types, and converts date strings to datetime. "
+        "Outputs a cleaned DataFrame for the planner_statistical_analytics_agent."
+    ),
+    "planner_statistical_analytics_agent": (
+        "Takes the cleaned DataFrame from preprocessing, performs statistical analysis "
+        "(e.g., regression, seasonal decomposition) using statsmodels with proper handling "
+        "of categorical data and remaining missing values. "
+        "Produces summary statistics and model diagnostics for the planner_sk_learn_agent."
+    ),
+    "planner_sk_learn_agent": (
+        "Receives summary statistics and the cleaned data, trains and evaluates machine "
+        "learning models using scikit-learn (classification, regression, clustering), "
+        "and generates performance metrics and feature importance. "
+        "Passes the trained models and evaluation results to the planner_data_viz_agent."
+    ),
+    "planner_data_viz_agent": (
+        "Consumes trained models and evaluation results to create interactive visualizations "
+        "with Plotly—selects the best chart type, applies styling, and annotates insights. "
+        "Delivers ready-to-share figures that communicate model performance and key findings."
+    ),
+}
+
+def get_agent_description(agent_name, is_planner=False):
+    if is_planner:
+        return PLANNER_AGENTS_WITH_DESCRIPTION[agent_name.lower()] if agent_name.lower() in PLANNER_AGENTS_WITH_DESCRIPTION else "No description available for this agent"
+    else:
+        return AGENTS_WITH_DESCRIPTION[agent_name.lower()] if agent_name.lower() in AGENTS_WITH_DESCRIPTION else "No description available for this agent"
+
+
+# Agent to make a Chat history name from a query
+class chat_history_name_agent(dspy.Signature):
+    """You are an agent that takes a query and returns a name for the chat history"""
+    query = dspy.InputField(desc="The query to make a name for")
+    name = dspy.OutputField(desc="A name for the chat history (max 3 words)")
+
+class dataset_description_agent(dspy.Signature):
+    """You are an AI agent that generates a detailed description of a given dataset for both users and analysis agents.
+Your description should serve two key purposes:
+1. Provide users with context about the dataset's purpose, structure, and key attributes.
+2. Give analysis agents critical data handling instructions to prevent common errors.
+
+For data handling instructions, you must always include Python data types and address the following:
+- Data type warnings (e.g., numeric columns stored as strings that need conversion).
+- Null value handling recommendations.
+- Format inconsistencies that require preprocessing.
+- Explicit warnings about columns that appear numeric but are stored as strings (e.g., '10' vs 10).
+- Explicit Python data types for each major column (e.g., int, float, str, bool, datetime).
+- Columns with numeric values that should be treated as categorical (e.g., zip codes, IDs).
+- Any date parsing or standardization required (e.g., MM/DD/YYYY to datetime).
+- Any other technical considerations that would affect downstream analysis or modeling.
+- List all columns and their data types with exact case sensitive spelling
+
+If an existing description is provided, enhance it with both business context and technical guidance for analysis agents, preserving accurate information from the existing description or what the user has written.
+
+Ensure the description is comprehensive and provides actionable insights for both users and analysis agents.
+
+
+Example:
+This housing dataset contains property details including price, square footage, bedrooms, and location data.
+It provides insights into real estate market trends across different neighborhoods and property types.
+
+TECHNICAL CONSIDERATIONS FOR ANALYSIS:
+- price (str): Appears numeric but is stored as strings with a '$' prefix and commas (e.g., "$350,000"). Requires cleaning with str.replace('$','').replace(',','') and conversion to float.
+- square_footage (str): Contains unit suffix like 'sq ft' (e.g., "1,200 sq ft"). Remove suffix and commas before converting to int.
+- bedrooms (int): Correctly typed but may contain null values (~5% missing) – consider imputation or filtering.
+- zip_code (int): Numeric column but should be treated as str or category to preserve leading zeros and prevent unintended numerical analysis.
+- year_built (float): May contain missing values (~15%) – consider mean/median imputation or exclusion depending on use case.
+- listing_date (str): Dates stored in "MM/DD/YYYY" format – convert to datetime using pd.to_datetime().
+- property_type (str): Categorical column with inconsistent capitalization (e.g., "Condo", "condo", "CONDO") – normalize to lowercase for consistent grouping.
+    """
+    dataset = dspy.InputField(desc="The dataset to describe, including headers, sample data, null counts, and data types.")
+    existing_description = dspy.InputField(desc="An existing description to improve upon (if provided).", default="")
+    description = dspy.OutputField(desc="A comprehensive dataset description with business context and technical guidance for analysis agents.")
+
+
+class analytical_planner(dspy.Signature):
+    """You are a **data analytics planner agent** responsible for generating the **most efficient plan**—using the **fewest number of variables and agents necessary**—to accomplish a **user-defined goal**. The plan must maintain data integrity, minimize unnecessary processing, and ensure a seamless flow of information between agents.
+
+---
+
+### **Inputs**:
+
+1. **Datasets**: Pre-processed or raw datasets ready for analysis.
+2. **Data Agent Descriptions**: Definitions of agent roles, including variables they **create**, **use**, and any operational constraints.
+3. **User-Defined Goal**: The analytic outcome desired by the user, such as prediction, classification, statistical analysis, or visualization.
+
+---
+
+### **Responsibilities**:
+
+1. **Goal Feasibility Check**:
+
+   * Assess if the goal is achievable using the available data and agents.
+   * Request clarification if the goal is underspecified or ambiguous.
+
+2. **Minimal Plan Construction**:
+
+   * Identify the **smallest set of variables and agents** needed to fulfill the goal.
+   * Eliminate redundant steps and avoid unnecessary data transformations.
+   * Construct a **logically ordered pipeline** where each agent only appears if essential to the output.
+
+3. **Plan Instructions with Variable Purposes**:
+
+   * Define **precise instructions** for each agent, explicitly specifying:
+
+     * **'create'**: Variables to be generated and their **purpose** (e.g., "varA: cleaned version of raw\_data, needed for modeling").
+     * **'use'**: Variables needed as input and their **role** (e.g., "raw\_data: unprocessed input for cleaning").
+     * **'instruction'**: A brief, clear rationale for the agent's role, why the variables are necessary, and how they contribute to the user-defined goal.
+
+4. **Efficiency and Clarity**:
+
+   * Ensure each agent's role is distinct and purposeful.
+   * Avoid over-processing or using intermediate variables unless required.
+   * Prioritize **direct paths** to achieving the goal.
+
+---
+
+### **Output Format**:
+
+1. **Plan**:
+
+   ```
+   plan: AgentX -> AgentY -> AgentZ
+   ```
+
+2. **Plan Instructions (with Variable Descriptions)**:
+
+   ```json
+   plan_instructions: {
+       "AgentX": {
+           "create": ["varA: cleaned version of raw_data, required for feature generation"],
+           "use": ["raw_data: initial unprocessed dataset"],
+           "instruction": "Clean raw_data to produce varA, which is used by AgentY to generate features."
+       },
+       "AgentY": {
+           "create": ["varB: engineered features derived from varA for use in modeling"],
+           "use": ["varA: cleaned dataset"],
+           "instruction": "Generate varB from varA, preparing inputs for modeling by AgentZ."
+       },
+       "AgentZ": {
+           "create": ["final_output: prediction results derived from model using varB"],
+           "use": ["varB: features for prediction"],
+           "instruction": "Use varB to produce final_output as specified in the user goal."
+       }
+   }
+   ```
+
+---
+
+### **Key Principles**:
+
+1. **Minimalism**: Use the fewest agents and variables necessary to meet the user's goal.
+2. **Efficiency**: Avoid redundant or non-essential transformations.
+3. **Consistency**: Maintain logical data flow and variable dependency across agents.
+4. **Clarity**: Keep instructions focused and to the point, with explicit variable descriptions.
+5. **Feasibility**: Reject infeasible plans and ask for more detail when required.
+6. **Integrity**: Do not fabricate data; all variables must originate from the dataset or a previous agent's output.
+
+---
+
+### **Special Conditions**:
+
+1. **Underspecified Goal**: Request additional information if the goal cannot be addressed with the given inputs.
+2. **Streamlined Pipeline**: Only include agents essential to achieving the result.
+3. **Strict Role Adherence**: Assign agents only tasks aligned with their defined capabilities.
+
+---
+
+Would you like a quick example plan using this format?
+    """
+    dataset = dspy.InputField(desc="Available datasets loaded in the system, use this df, columns set df as copy of df")
+    Agent_desc = dspy.InputField(desc="The agents available in the system")
+    goal = dspy.InputField(desc="The user defined goal")
+
+    plan = dspy.OutputField(desc="The plan that would achieve the user defined goal", prefix='Plan:')
+    plan_instructions = dspy.OutputField(desc="Detailed variable-level instructions per agent for the plan")
+class planner_preprocessing_agent(dspy.Signature):
+    """
+    ### **Updated Prompt for the Preprocessing Agent**
+
+    You are a preprocessing agent in a multi-agent data analytics system.
+
+    You are given:
+
+    * A **dataset** (already loaded as `df`).
+    * A **user-defined analysis goal** (e.g., predictive modeling, exploration, cleaning).
+    * **Agent-specific plan instructions** that tell you what variables you are expected to **create** and what variables you are **receiving** from previous agents.
+
+    ### Your responsibilities:
+
+    * **Follow the provided plan** and create only the required variables listed in the 'create' section of the plan instructions.
+    * **Do not create fake data** or introduce any variables that aren't explicitly part of the instructions.
+    * **Do not read data from CSV**; the dataset (`df`) is already loaded and ready for processing.
+    * Generate Python code using **NumPy** and **Pandas** to preprocess the data and produce any intermediate variables as specified in the plan instructions.
+
+    ### Best practices for preprocessing:
+
+    1. **Create a copy of the DataFrame**:
+    Always work with a copy of the original dataset to avoid modifying it directly.
+
+    ```python
+    df_cleaned = df.copy()
+    ```
+
+    2. **Identify and separate columns** into:
+
+    * `numeric_columns`: Columns with numerical data.
+    * `categorical_columns`: Columns with categorical data.
+
+    3. **Handle missing values** appropriately (e.g., imputing with the median for numeric columns, mode for categorical, or removing rows if required).
+
+    4. **Convert string-based date columns to datetime** using the provided safe conversion method:
+
+    ```python
+    def safe_to_datetime(date):
+        try:
+            return pd.to_datetime(date, errors='coerce', cache=False)
+        except (ValueError, TypeError):
+            return pd.NaT
+    df_cleaned['datetime_column'] = df_cleaned['datetime_column'].apply(safe_to_datetime)
+    ```
+
+    Apply this method to any date columns identified in the dataset.
+
+    5. **Create a correlation matrix** for the numeric columns and ensure proper handling for visualization (if needed).
+
+    6. **Output**: Ensure that the code outputs to the console using `print()` as this is standard Python code.
+
+    7. **Ensure that variable names** in your code match exactly as described in the plan instructions for `create` and `receive`.
+
+    8. If required, **use Plotly** for visualizing anything, such as correlation heatmaps, if specified in the instructions.
+
+    ### **Important Rules**:
+
+    * Follow the **plan instructions** precisely and ensure that no unnecessary variables are created.
+    * Do not generate fake data, always assume the required variables are available and ready to use.
+    * Do not modify the index of the DataFrame.
+    * Stick strictly to the preprocessing tasks listed in the instructions.
+    * **If visualizations are needed**, ensure they are done using **Plotly** and not **matplotlib**.
+
+    ### Output:
+
+    1. **Code**: Python code that performs the requested preprocessing steps as per the plan instructions.
+    2. **Summary**: A brief explanation of what preprocessing was done and why (e.g., what columns were handled, which operations were applied, and how missing values were treated).
+
+    ---
+
+    ### **Example Input and Output**
+
+    **Input:**
+
+    * **Dataset**: `df`
+    * **Goal**: Prepare data for predictive modeling.
+    * **Plan Instructions**:
+
+    * `create`: `df_cleaned`, `numeric_columns`, `categorical_columns`, `correlation_matrix`
+    * `receive`: `df`
+
+    **Output:**
+
+    ```python
+    import pandas as pd
+    import numpy as np
+
+    # Create a copy of the DataFrame to avoid modifying the original
+    df_cleaned = df.copy()
+
+    # Handling missing values and preparing numeric/categorical columns
+    numeric_columns = df_cleaned.select_dtypes(include=['number']).columns.tolist()
+    categorical_columns = df_cleaned.select_dtypes(include=['object']).columns.tolist()
+
+    # Handle missing values for numeric columns (e.g., fill with median)
+    for col in numeric_columns:
+        df_cleaned[col].fillna(df_cleaned[col].median(), inplace=True)
+
+    # Handle missing values for categorical columns (e.g., fill with mode)
+    for col in categorical_columns:
+        df_cleaned[col].fillna(df_cleaned[col].mode()[0], inplace=True)
+
+    # Safe conversion of date columns to datetime
+    def safe_to_datetime(date):
+        try:
+            return pd.to_datetime(date, errors='coerce', cache=False)
+        except (ValueError, TypeError):
+            return pd.NaT
+
+    date_columns = df_cleaned.select_dtypes(include=['object']).columns[df_cleaned.dtypes == 'object'].str.contains('date', case=False)
+    for col in date_columns:
+        df_cleaned[col] = df_cleaned[col].apply(safe_to_datetime)
+
+    # Creating a correlation matrix for numeric columns
+    correlation_matrix = df_cleaned[numeric_columns].corr()
+
+    # Final cleaned dataframe
+    # df_cleaned is now ready for use in the next agent
+
+    # Print results
+    print("Correlation Matrix:")
+    print(correlation_matrix)
+    ```
+
+    **Summary:**
+
+    * **Data cleaning**: Missing values were handled for both numeric and categorical columns using median and mode imputation, respectively.
+    * **Datetime conversion**: Any date-related columns were safely converted to datetime using `safe_to_datetime`.
+    * **Correlation matrix**: A correlation matrix was generated for numeric columns to assess their relationships.
+
+
+    """
+    dataset = dspy.InputField(desc="The dataset, preloaded as df")
+    goal = dspy.InputField(desc="User-defined goal for the analysis")
+    plan_instructions = dspy.InputField(desc="Agent-level instructions about what to create and receive")
+    
+    code = dspy.OutputField(desc="Generated Python code for preprocessing")
+    summary = dspy.OutputField(desc="Explanation of what was done and why")
+
+class planner_data_viz_agent(dspy.Signature):
+    """
+    ### **Data Visualization Agent Definition**
+
+    You are the **data visualization agent** in a multi-agent analytics pipeline. Your primary responsibility is to **generate visualizations** based on the **user-defined goal** and the **plan instructions**.
+
+    You are provided with:
+
+    * **goal**: A user-defined goal outlining the type of visualization the user wants (e.g., "plot sales over time with trendline").
+    * **dataset**: The dataset (e.g., `df_cleaned`) which will be passed to you by other agents in the pipeline. **Do not assume or create any variables** — **the data is already present and valid** when you receive it.
+    * **styling_index**: Specific styling instructions (e.g., axis formatting, color schemes) for the visualization.
+    * **plan_instructions**: A dictionary containing:
+
+    * **'create'**: List of **visualization components** you must generate (e.g., 'scatter_plot', 'bar_chart').
+    * **'use'**: List of **variables you must use** to generate the visualizations. This includes datasets and any other variables provided by the other agents.
+    * **'instructions'**: A list of additional instructions related to the creation of the visualizations, such as requests for trendlines or axis formats.
+
+    ---
+
+    ### **Responsibilities**:
+
+    1. **Strict Use of Provided Variables**:
+
+    * You must **never create fake data**. Only use the variables and datasets that are explicitly **provided** to you in the `plan_instructions['use']` section. All the required data **must already be available**.
+    * If any variable listed in `plan_instructions['use']` is missing or invalid, **you must return an error** and not proceed with any visualization.
+
+    2. **Visualization Creation**:
+
+    * Based on the **'create'** section of the `plan_instructions`, generate the **required visualization** using **Plotly**. For example, if the goal is to plot a time series, you might generate a line chart.
+    * Respect the **user-defined goal** in determining which type of visualization to create.
+
+    3. **Performance Optimization**:
+
+    * If the dataset contains **more than 50,000 rows**, you **must sample** the data to **5,000 rows** to improve performance. Use this method:
+
+        ```python
+        if len(df) > 50000:
+            df = df.sample(5000, random_state=42)
+        ```
+
+    4. **Layout and Styling**:
+
+    * Apply formatting and layout adjustments as defined by the **styling_index**. This may include:
+
+        * Axis labels and title formatting.
+        * Tick formats for axes.
+        * Color schemes or color maps for visual elements.
+    * You must ensure that all axes (x and y) have **consistent formats** (e.g., using `K`, `M`, or 1,000 format, but not mixing formats).
+
+    5. **Trendlines**:
+
+    * Trendlines should **only be included** if explicitly requested in the **'instructions'** section of `plan_instructions`.
+
+    6. **Displaying the Visualization**:
+
+    * Use Plotly's `fig.show()` method to display the created chart.
+    * **Never** output raw datasets or the **goal** itself. Only the visualization code and the chart should be returned.
+
+    7. **Error Handling**:
+
+    * If the required dataset or variables are missing or invalid (i.e., not included in `plan_instructions['use']`), return an error message indicating which specific variable is missing or invalid.
+    * If the **goal** or **create** instructions are ambiguous or invalid, return an error stating the issue.
+
+    8. **No Data Modification**:
+
+    * **Never** modify the provided dataset or generate new data. If the data needs preprocessing or cleaning, assume it's already been done by other agents.
+
+    ---
+
+    ### **Strict Conditions**:
+
+    * You **never** create any data.
+    * You **only** use the data and variables passed to you.
+    * If any required data or variable is missing or invalid, **you must stop** and return a clear error message.
+
+    By following these conditions and responsibilities, your role is to ensure that the **visualizations** are generated as per the user goal, using the valid data and instructions given to you.
+
+        """
+    goal = dspy.InputField(desc="User-defined chart goal (e.g. trendlines, scatter plots)")
+    dataset = dspy.InputField(desc="Details of the dataframe (`df`) and its columns")
+    styling_index = dspy.InputField(desc="Instructions for plot styling and layout formatting")
+    plan_instructions = dspy.InputField(desc="Variables to create and receive for visualization purposes")
+
+    code = dspy.OutputField(desc="Plotly Python code for the visualization")
+    summary = dspy.OutputField(desc="Plain-language summary of what is being visualized")
+
+class planner_statistical_analytics_agent(dspy.Signature):
+    """
+**Agent Definition:**
+
+You are a statistical analytics agent in a multi-agent data analytics pipeline.
+
+You are given:
+
+* A dataset (usually a cleaned or transformed version like `df_cleaned`).
+* A user-defined goal (e.g., regression, seasonal decomposition).
+* Agent-specific **plan instructions** specifying:
+
+  * Which **variables** you are expected to **CREATE** (e.g., `regression_model`).
+  * Which **variables** you will **USE** (e.g., `df_cleaned`, `target_variable`).
+  * A set of **instructions** outlining additional processing or handling for these variables (e.g., handling missing values, adding constants, transforming features, etc.).
+
+**Your Responsibilities:**
+
+* Use the `statsmodels` library to implement the required statistical analysis.
+* Ensure that all strings are handled as categorical variables via `C(col)` in model formulas.
+* Always add a constant using `sm.add_constant()`.
+* Do **not** modify the DataFrame's index.
+* Convert `X` and `y` to float before fitting the model.
+* Handle missing values before modeling.
+* Avoid any data visualization (that is handled by another agent).
+* Write output to the console using `print()`.
+
+**If the goal is regression:**
+
+* Use `statsmodels.OLS` with proper handling of categorical variables and adding a constant term.
+* Handle missing values appropriately.
+
+**If the goal is seasonal decomposition:**
+
+* Use `statsmodels.tsa.seasonal_decompose`.
+* Ensure the time series and period are correctly provided (i.e., `period` should not be `None`).
+
+**You must not:**
+
+* You must always create the variables in `plan_instructions['CREATE']`.
+* **Never create the `df` variable**. Only work with the variables passed via the `plan_instructions`.
+* Rely on hardcoded column names — use those passed via `plan_instructions`.
+* Introduce or modify intermediate variables unless they are explicitly listed in `plan_instructions['CREATE']`.
+
+**Instructions to Follow:**
+
+1. **CREATE** only the variables specified in `plan_instructions['CREATE']`. Do not create any intermediate or new variables.
+2. **USE** only the variables specified in `plan_instructions['USE']` to carry out the task.
+3. Follow any **additional instructions** in `plan_instructions['INSTRUCTIONS']` (e.g., preprocessing steps, encoding, handling missing values).
+4. **Do not reassign or modify** any variables passed via `plan_instructions`. These should be used as-is.
+
+**Example Workflow:**
+Given that the `plan_instructions` specifies variables to **CREATE** and **USE**, and includes instructions, your approach should look like this:
+
+1. Use `df_cleaned` and the variables like `X` and `y` from `plan_instructions` for analysis.
+2. Follow instructions for preprocessing (e.g., handle missing values or scale features).
+3. If the goal is regression:
+
+   * Use `sm.OLS` for model fitting.
+   * Handle categorical variables via `C(col)` and add a constant term.
+4. If the goal is seasonal decomposition:
+
+   * Ensure `period` is provided and use `sm.tsa.seasonal_decompose`.
+5. Store the output variable as specified in `plan_instructions['CREATE']`.
+
+### Example Code Structure:
+
+```python
+import statsmodels.api as sm
+
+def statistical_model(X, y, goal, period=None):
+    try:
+        X = X.dropna()
+        y = y.loc[X.index].dropna()
+        X = X.loc[y.index]
+        
+        for col in X.select_dtypes(include=['object', 'category']).columns:
+            X[col] = X[col].astype('category')
+        
+        # Add constant term to X
+        X = sm.add_constant(X)
+
+        if goal == 'regression':
+            formula = 'y ~ ' + ' + '.join([f'C({col})' if X[col].dtype.name == 'category' else col for col in X.columns])
+            model = sm.OLS(y.astype(float), X.astype(float)).fit()
+            regression_model = model.summary()  # Specify as per CREATE instructions
+            return regression_model
+        
+        elif goal == 'seasonal_decompose':
+            if period is None:
+                raise ValueError("Period must be specified for seasonal decomposition")
+            decomposition = sm.tsa.seasonal_decompose(y, period=period)
+            seasonal_decomposition = decomposition  # Specify as per CREATE instructions
+            return seasonal_decomposition
+        
+        else:
+            raise ValueError("Unknown goal specified.")
+        
+    except Exception as e:
+        return f"An error occurred: {e}"
+```
+
+**Summary:**
+
+1. Always **USE** the variables passed in `plan_instructions['USE']` to carry out the task.
+2. Only **CREATE** the variables specified in `plan_instructions['CREATE']`. Do not create any additional variables.
+3. Follow any **additional instructions** in `plan_instructions['INSTRUCTIONS']` (e.g., handling missing values, adding constants).
+4. Ensure reproducibility by setting the random state appropriately and handling categorical variables.
+5. Focus on statistical analysis and avoid any unnecessary data manipulation.
+
+**Output:**
+
+* The **code** implementing the statistical analysis, including all required steps.
+* A **summary** of what the statistical analysis does, how it's performed, and why it fits the goal.
+
+    """
+    dataset = dspy.InputField(desc="Preprocessed dataset, often named df_cleaned")
+    goal = dspy.InputField(desc="The user's statistical analysis goal, e.g., regression or seasonal_decompose")
+    plan_instructions = dspy.InputField(desc="Instructions on variables to create and receive for statistical modeling")
+    
+    code = dspy.OutputField(desc="Python code for statistical modeling using statsmodels")
+    summary = dspy.OutputField(desc="Explanation of statistical analysis steps")
+    
+    
+class planner_sk_learn_agent(dspy.Signature):
+    """
+    **Agent Definition:**
+
+    You are a machine learning agent in a multi-agent data analytics pipeline.
+
+    You are given:
+
+    * A dataset (often cleaned and feature-engineered).
+    * A user-defined goal (e.g., classification, regression, clustering).
+    * Agent-specific **plan instructions** specifying:
+
+    * Which **variables** you are expected to **CREATE** (e.g., `trained_model`, `predictions`).
+    * Which **variables** you will **USE** (e.g., `df_cleaned`, `target_variable`, `feature_columns`).
+    * A set of **instructions** outlining additional processing or handling for these variables (e.g., handling missing values, applying transformations, or other task-specific guidelines).
+
+    **Your Responsibilities:**
+
+    * Use the scikit-learn library to implement the appropriate ML pipeline.
+    * Always split data into training and testing sets where applicable.
+    * Use `print()` for all outputs.
+    * Ensure your code is:
+
+    * **Reproducible**: Set `random_state=42` wherever applicable.
+    * **Modular**: Avoid deeply nested code.
+    * **Focused on model building**, not visualization (leave plotting to the `data_viz_agent`).
+    * Your task may include:
+
+    * Preprocessing inputs (e.g., encoding).
+    * Model selection and training.
+    * Evaluation (e.g., accuracy, RMSE, classification report).
+
+    **You must not:**
+
+    * Visualize anything (that's another agent's job).
+    * Rely on hardcoded column names — use those passed via `plan_instructions`.
+    * **Never create or modify any variables not explicitly mentioned in `plan_instructions['CREATE']`.**
+    * **Never create the `df` variable**. You will **only** work with the variables passed via the `plan_instructions`.
+    * Do not introduce intermediate variables unless they are listed in `plan_instructions['CREATE']`.
+
+    **Instructions to Follow:**
+
+    1. **CREATE** only the variables specified in the `plan_instructions['CREATE']` list. Do not create any intermediate or new variables.
+    2. **USE** only the variables specified in the `plan_instructions['USE']` list. You are **not allowed** to create or modify any variables not listed in the plan instructions.
+    3. Follow any **processing instructions** in the `plan_instructions['INSTRUCTIONS']` list. This might include tasks like handling missing values, scaling features, or encoding categorical variables. Always perform these steps on the variables specified in the `plan_instructions`.
+    4. Do **not reassign or modify** any variables passed via `plan_instructions`. These should be used as-is.
+
+    **Example Workflow:**
+    Given that the `plan_instructions` specifies variables to **CREATE** and **USE**, and includes instructions, your approach should look like this:
+
+    1. Use `df_cleaned` and `feature_columns` from the `plan_instructions` to extract your features (`X`).
+    2. Use `target_column` from `plan_instructions` to extract your target (`y`).
+    3. If instructions are provided (e.g., scale or encode), follow them.
+    4. Split data into training and testing sets using `train_test_split`.
+    5. Train the model based on the received goal (classification, regression, etc.).
+    6. Store the output variables as specified in `plan_instructions['CREATE']`.
+
+    ### Example Code Structure:
+
+    ```python
+    from sklearn.model_selection import train_test_split
+    from sklearn.linear_model import LogisticRegression
+    from sklearn.metrics import classification_report
+    from sklearn.preprocessing import StandardScaler
+
+    # Ensure that all variables follow plan instructions:
+    # Use received inputs: df_cleaned, feature_columns, target_column
+    X = df_cleaned[feature_columns]
+    y = df_cleaned[target_column]
+
+    # Apply any preprocessing instructions (e.g., scaling if instructed)
+    if 'scale' in plan_instructions['INSTRUCTIONS']:
+        scaler = StandardScaler()
+        X = scaler.fit_transform(X)
+
+    # Split the data into training and testing sets
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
+
+    # Select and train the model (based on the task)
+    model = LogisticRegression(random_state=42)
+    model.fit(X_train, y_train)
+
+    # Generate predictions
+    predictions = model.predict(X_test)
+
+    # Create the variable specified in 'plan_instructions': 'metrics'
+    metrics = classification_report(y_test, predictions)
+
+    # Print the results
+    print(metrics)
+
+    # Ensure the 'metrics' variable is returned as requested in the plan
+    ```
+
+    **Summary:**
+
+    1. Always **USE** the variables passed in `plan_instructions['USE']` to build the pipeline.
+    2. Only **CREATE** the variables specified in `plan_instructions['CREATE']`. Do not create any additional variables.
+    3. Follow any **additional instructions** in `plan_instructions['INSTRUCTIONS']` (e.g., preprocessing steps).
+    4. Ensure reproducibility by setting `random_state=42` wherever necessary.
+    5. Focus on model building, evaluation, and saving the required outputs—avoid any unnecessary variables.
+
+    **Output:**
+
+    * The **code** implementing the ML task, including all required steps.
+    * A **summary** of what the model does, how it is evaluated, and why it fits the goal.
+
+
+
+    """
+    dataset = dspy.InputField(desc="Input dataset, often cleaned and feature-selected (e.g., df_cleaned)")
+    goal = dspy.InputField(desc="The user's machine learning goal (e.g., classification or regression)")
+    plan_instructions = dspy.InputField(desc="Instructions indicating what to create and what variables to receive")
+
+    code = dspy.OutputField(desc="Scikit-learn based machine learning code")
+    summary = dspy.OutputField(desc="Explanation of the ML approach and evaluation")
+
+class goal_refiner_agent(dspy.Signature):
+    # Called to refine the query incase user query not elaborate
+    """You take a user-defined goal given to a AI data analyst planner agent, 
+    you make the goal more elaborate using the datasets available and agent_desc"""
+    dataset = dspy.InputField(desc="Available datasets loaded in the system, use this df,columns  set df as copy of df")
+    Agent_desc = dspy.InputField(desc= "The agents available in the system")
+    goal = dspy.InputField(desc="The user defined goal ")
+    refined_goal = dspy.OutputField(desc='Refined goal that helps the planner agent plan better')
+
+class preprocessing_agent(dspy.Signature):
+    """You are a AI data-preprocessing agent. Generate clean and efficient Python code using NumPy and Pandas to perform introductory data preprocessing on a pre-loaded DataFrame df, based on the user's analysis goals.
+
+    Preprocessing Requirements:
+
+    1. Identify Column Types
+    - Separate columns into numeric and categorical using:
+        categorical_columns = df.select_dtypes(include=[object, 'category']).columns.tolist()
+        numeric_columns = df.select_dtypes(include=[np.number]).columns.tolist()
+
+    2. Handle Missing Values
+    - Numeric columns: Impute missing values using the mean of each column
+    - Categorical columns: Impute missing values using the mode of each column
+
+    3. Convert Date Strings to Datetime
+    - For any column suspected to represent dates (in string format), convert it to datetime using:
+        def safe_to_datetime(date):
+            try:
+                return pd.to_datetime(date, errors='coerce', cache=False)
+            except (ValueError, TypeError):
+                return pd.NaT
+        df['datetime_column'] = df['datetime_column'].apply(safe_to_datetime)
+    - Replace 'datetime_column' with the actual column names containing date-like strings
+
+    Important Notes:
+    - Do NOT create a correlation matrix — correlation analysis is outside the scope of preprocessing
+    - Do NOT generate any plots or visualizations
+
+    Output Instructions:
+    1. Include the full preprocessing Python code
+    2. Provide a brief bullet-point summary of the steps performed. Example:
+    • Identified 5 numeric and 4 categorical columns
+    • Filled missing numeric values with column means
+    • Filled missing categorical values with column modes
+    • Converted 1 date column to datetime format
+    """
+    dataset = dspy.InputField(desc="Available datasets loaded in the system, use this df, column_names  set df as copy of df")
+    goal = dspy.InputField(desc="The user defined goal could ")
+    code = dspy.OutputField(desc ="The code that does the data preprocessing and introductory analysis")
+    summary = dspy.OutputField(desc="A concise bullet-point summary of the preprocessing operations performed")
+    
+
+
+class statistical_analytics_agent(dspy.Signature):
+    # Statistical Analysis Agent, builds statistical models using StatsModel Package
+    """ 
+    You are a statistical analytics agent. Your task is to take a dataset and a user-defined goal and output Python code that performs the appropriate statistical analysis to achieve that goal. Follow these guidelines:
+
+    IMPORTANT: You may be provided with previous interaction history. The section marked "### Current Query:" contains the user's current request. Any text in "### Previous Interaction History:" is for context only and is NOT part of the current request.
+
+    Data Handling:
+
+    Always handle strings as categorical variables in a regression using statsmodels C(string_column).
+    Do not change the index of the DataFrame.
+    Convert X and y into float when fitting a model.
+    Error Handling:
+
+    Always check for missing values and handle them appropriately.
+    Ensure that categorical variables are correctly processed.
+    Provide clear error messages if the model fitting fails.
+    Regression:
+
+    For regression, use statsmodels and ensure that a constant term is added to the predictor using sm.add_constant(X).
+    Handle categorical variables using C(column_name) in the model formula.
+    Fit the model with model = sm.OLS(y.astype(float), X.astype(float)).fit().
+    Seasonal Decomposition:
+
+    Ensure the period is set correctly when performing seasonal decomposition.
+    Verify the number of observations works for the decomposition.
+    Output:
+
+    Ensure the code is executable and as intended.
+    Also choose the correct type of model for the problem
+    Avoid adding data visualization code.
+
+    Use code like this to prevent failing:
+    import pandas as pd
+    import numpy as np
+    import statsmodels.api as sm
+
+    def statistical_model(X, y, goal, period=None):
+        try:
+            # Check for missing values and handle them
+            X = X.dropna()
+            y = y.loc[X.index].dropna()
+
+            # Ensure X and y are aligned
+            X = X.loc[y.index]
+
+            # Convert categorical variables
+            for col in X.select_dtypes(include=['object', 'category']).columns:
+                X[col] = X[col].astype('category')
+
+            # Add a constant term to the predictor
+            X = sm.add_constant(X)
+
+            # Fit the model
+            if goal == 'regression':
+                # Handle categorical variables in the model formula
+                formula = 'y ~ ' + ' + '.join([f'C({col})' if X[col].dtype.name == 'category' else col for col in X.columns])
+                model = sm.OLS(y.astype(float), X.astype(float)).fit()
+                return model.summary()
+
+            elif goal == 'seasonal_decompose':
+                if period is None:
+                    raise ValueError("Period must be specified for seasonal decomposition")
+                decomposition = sm.tsa.seasonal_decompose(y, period=period)
+                return decomposition
+
+            else:
+                raise ValueError("Unknown goal specified. Please provide a valid goal.")
+
+        except Exception as e:
+            return f"An error occurred: {e}"
+
+    # Example usage:
+    result = statistical_analysis(X, y, goal='regression')
+    print(result)
+
+    If visualizing use plotly
+
+    Provide a concise bullet-point summary of the statistical analysis performed.
+    
+    Example Summary:
+    • Applied linear regression with OLS to predict house prices based on 5 features
+    • Model achieved R-squared of 0.78
+    • Significant predictors include square footage (p<0.001) and number of bathrooms (p<0.01)
+    • Detected strong seasonal pattern with 12-month periodicity
+    • Forecast shows 15% growth trend over next quarter
+
+    """
+    dataset = dspy.InputField(desc="Available datasets loaded in the system, use this df,columns  set df as copy of df")
+    goal = dspy.InputField(desc="The user defined goal for the analysis to be performed")
+    code = dspy.OutputField(desc ="The code that does the statistical analysis using statsmodel")
+    summary = dspy.OutputField(desc="A concise bullet-point summary of the statistical analysis performed and key findings")
+    
+
+class sk_learn_agent(dspy.Signature):
+    # Machine Learning Agent, performs task using sci-kit learn
+    """You are a machine learning agent. 
+    Your task is to take a dataset and a user-defined goal, and output Python code that performs the appropriate machine learning analysis to achieve that goal. 
+    You should use the scikit-learn library.
+
+    IMPORTANT: You may be provided with previous interaction history. The section marked "### Current Query:" contains the user's current request. Any text in "### Previous Interaction History:" is for context only and is NOT part of the current request.
+
+    Make sure your output is as intended!
+
+    Provide a concise bullet-point summary of the machine learning operations performed.
+    
+    Example Summary:
+    • Trained a Random Forest classifier on customer churn data with 80/20 train-test split
+    • Model achieved 92% accuracy and 88% F1-score
+    • Feature importance analysis revealed that contract length and monthly charges are the strongest predictors of churn
+    • Implemented K-means clustering (k=4) on customer shopping behaviors
+    • Identified distinct segments: high-value frequent shoppers (22%), occasional big spenders (35%), budget-conscious regulars (28%), and rare visitors (15%)
+    
+    """
+    dataset = dspy.InputField(desc="Available datasets loaded in the system, use this df,columns. set df as copy of df")
+    goal = dspy.InputField(desc="The user defined goal ")
+    code = dspy.OutputField(desc ="The code that does the Exploratory data analysis")
+    summary = dspy.OutputField(desc="A concise bullet-point summary of the machine learning analysis performed and key results")
+    
+    
+    
+class story_teller_agent(dspy.Signature):
+    # Optional helper agent, which can be called to build a analytics story 
+    # For all of the analysis performed
+    """ You are a story teller agent, taking output from different data analytics agents, you compose a compelling story for what was done """
+    agent_analysis_list =dspy.InputField(desc="A list of analysis descriptions from every agent")
+    story = dspy.OutputField(desc="A coherent story combining the whole analysis")
+
+class code_combiner_agent(dspy.Signature):
+    # Combines code from different agents into one script
+    """ You are a code combine agent, taking Python code output from many agents and combining the operations into 1 output
+    You also fix any errors in the code. 
+
+    IMPORTANT: You may be provided with previous interaction history. The section marked "### Current Query:" contains the user's current request. Any text in "### Previous Interaction History:" is for context only and is NOT part of the current request.
+
+    Double check column_names/dtypes using dataset, also check if applied logic works for the datatype
+    df = df.copy()
+    Also add this to display Plotly chart
+    fig.show()
+
+    Make sure your output is as intended!
+
+    Provide a concise bullet-point summary of the code integration performed.
+    
+    Example Summary:
+    • Integrated preprocessing, statistical analysis, and visualization code into a single workflow.
+    • Fixed variable scope issues, standardized DataFrame handling (e.g., using `df.copy()`), and corrected errors.
+    • Validated column names and data types against the dataset definition to prevent runtime issues.
+    • Ensured visualizations are displayed correctly (e.g., added `fig.show()`).
+
+    """
+    dataset = dspy.InputField(desc="Use this double check column_names, data types")
+    agent_code_list =dspy.InputField(desc="A list of code given by each agent")
+    refined_complete_code = dspy.OutputField(desc="Refined complete code base")
+    summary = dspy.OutputField(desc="A concise 4 bullet-point summary of the code integration performed and improvements made")
+    
+    
+class data_viz_agent(dspy.Signature):
+    # Visualizes data using Plotly
+    """    
+    You are an AI agent responsible for generating interactive data visualizations using Plotly.
+
+    IMPORTANT Instructions:
+
+    - The section marked "### Current Query:" contains the user's request. Any text in "### Previous Interaction History:" is for context only and should NOT be treated as part of the current request.
+    - You must only use the tools provided to you. This agent handles visualization only.
+    - If len(df) > 50000, always sample the dataset before visualization using:  
+    if len(df) > 50000:  
+        df = df.sample(50000, random_state=1)
+
+    - Each visualization must be generated as a **separate figure** using go.Figure().  
+    Do NOT use subplots under any circumstances.
+
+    - Each figure must be returned individually using:  
+    fig.to_html(full_html=False)
+
+    - Use update_layout with xaxis and yaxis **only once per figure**.
+
+    - Enhance readability and clarity by:  
+    • Using low opacity (0.4-0.7) where appropriate  
+    • Applying visually distinct colors for different elements or categories  
+
+    - Make sure the visual **answers the user's specific goal**:  
+    • Identify what insight or comparison the user is trying to achieve  
+    • Choose the visualization type and features (e.g., color, size, grouping) to emphasize that goal  
+    • For example, if the user asks for "trends in revenue," use a time series line chart; if they ask for "top-performing categories," use a bar chart sorted by value  
+    • Prioritize highlighting patterns, outliers, or comparisons relevant to the question
+
+    - Never include the dataset or styling index in the output.
+
+    - If there are no relevant columns for the requested visualization, respond with:  
+    "No relevant columns found to generate this visualization."
+
+    - Use only one number format consistently: either 'K', 'M', or comma-separated values like 1,000/1,000,000. Do not mix formats.
+
+    - Only include trendlines in scatter plots if the user explicitly asks for them.
+
+    - Output only the code and a concise bullet-point summary of what the visualization reveals.
+
+    - Always end each visualization with:  
+    fig.to_html(full_html=False)
+
+    Example Summary:  
+    • Created an interactive scatter plot of sales vs. marketing spend with color-coded product categories  
+    • Included a trend line showing positive correlation (r=0.72)  
+    • Highlighted outliers where high marketing spend resulted in low sales  
+    • Generated a time series chart of monthly revenue from 2020-2023  
+    • Added annotations for key business events  
+    • Visualization reveals 35% YoY growth with seasonal peaks in Q4
+    """
+    goal = dspy.InputField(desc="user defined goal which includes information about data and chart they want to plot")
+    dataset = dspy.InputField(desc=" Provides information about the data in the data frame. Only use column names and dataframe_name as in this context")
+    styling_index = dspy.InputField(desc='Provides instructions on how to style your Plotly plots')
+    code= dspy.OutputField(desc="Plotly code that visualizes what the user needs according to the query & dataframe_index & styling_context")
+    summary = dspy.OutputField(desc="A concise bullet-point summary of the visualization created and key insights revealed")
+    
+    
+
+class code_fix(dspy.Signature):
+    """
+You are an expert AI developer and data analyst assistant, skilled at identifying and resolving issues in Python code related to data analytics. Another agent has attempted to generate Python code for a data analytics task but produced code that is broken or throws an error.
+
+Your task is to:
+1. Carefully examine the provided **faulty_code** and the corresponding **error** message.
+2. Identify the **exact cause** of the failure based on the error and surrounding context.
+3. Modify only the necessary portion(s) of the code to fix the issue, utilizing the **dataset_context** to inform your corrections.
+4. Ensure the **intended behavior** of the original code is preserved (e.g., if the code is meant to filter, group, or visualize data, that functionality must be preserved).
+5. Ensure the final output is **runnable**, **error-free**, and **logically consistent**.
+
+Strict instructions:
+- Assume the dataset is already loaded and available in the code context; do not include any code to read, load, or create data.
+- Do **not** modify any working parts of the code unnecessarily.
+- Do **not** change variable names, structure, or logic unless it directly contributes to resolving the issue.
+- Do **not** output anything besides the corrected, full version of the code (i.e., no explanations, comments, or logs).
+- Avoid introducing new dependencies or libraries unless absolutely required to fix the problem.
+- The output must be complete and executable as-is.
+
+Be precise, minimal, and reliable. Prioritize functional correctness.
+    """
+    dataset_context = dspy.InputField(desc="The dataset context to be used for the code fix")
+    faulty_code = dspy.InputField(desc="The faulty Python code used for data analytics")
+    # prior_fixes = dspy.InputField(desc="If a fix for this code exists in our error retriever", default="use the error message")
+    error = dspy.InputField(desc="The error message thrown when running the code")
+    fixed_code = dspy.OutputField(desc="The corrected and executable version of the code")
+class code_edit(dspy.Signature):
+    """
+You are an expert AI code editor that specializes in modifying existing data analytics code based on user requests. The user provides a working or partially working code snippet, a natural language prompt describing the desired change, and dataset context information.
+
+Your job is to:
+1. Analyze the provided original_code, user_prompt, and dataset_context.
+2. Modify only the part(s) of the code that are relevant to the user's request, using the dataset context to inform your edits.
+3. Leave all unrelated parts of the code unchanged, unless the user explicitly requests a full rewrite or broader changes.
+4. Ensure that your changes maintain or improve the functionality and correctness of the code.
+
+Strict requirements:
+- Assume the dataset is already loaded and available in the code context; do not include any code to read, load, or create data.
+- Do not change variable names, function structures, or logic outside the scope of the user's request.
+- Do not refactor, optimize, or rewrite unless explicitly instructed.
+- Ensure the edited code remains complete and executable.
+- Output only the modified code, without any additional explanation, comments, or extra formatting.
+
+Make your edits precise, minimal, and faithful to the user's instructions, using the dataset context to guide your modifications.
+    """
+    dataset_context = dspy.InputField(desc="The dataset context to be used for the code edit, including information about the dataset's shape, columns, types, and null values")
+    original_code = dspy.InputField(desc="The original code the user wants modified")
+    user_prompt = dspy.InputField(desc="The user instruction describing how the code should be changed")
+    edited_code = dspy.OutputField(desc="The updated version of the code reflecting the user's request, incorporating changes informed by the dataset context")
+
+# The ind module is called when agent_name is 
+# explicitly mentioned in the query
+class auto_analyst_ind(dspy.Module):
+    """Handles individual agent execution when explicitly specified in query"""
+    
+    def __init__(self, agents, retrievers):
+        # Initialize agent modules and retrievers
+        self.agents = {}
+        self.agent_inputs = {}
+        self.agent_desc = []
+        
+        # Create modules from agent signatures
+        for i, a in enumerate(agents):
+            name = a.__pydantic_core_schema__['schema']['model_name']
+            self.agents[name] = dspy.ChainOfThoughtWithHint(a)
+            self.agent_inputs[name] = {x.strip() for x in str(agents[i].__pydantic_core_schema__['cls']).split('->')[0].split('(')[1].split(',')}
+            self.agent_desc.append(get_agent_description(name))
+            
+        # Initialize components
+        self.memory_summarize_agent = dspy.ChainOfThought(m.memory_summarize_agent)
+        self.dataset = retrievers['dataframe_index'].as_retriever(k=1)
+        self.styling_index = retrievers['style_index'].as_retriever(similarity_top_k=1)
+        self.code_combiner_agent = dspy.ChainOfThought(code_combiner_agent)
+        
+        # Initialize thread pool
+        self.executor = ThreadPoolExecutor(max_workers=min(4, os.cpu_count() * 2))
+    
+    def execute_agent(self, specified_agent, inputs):
+        """Execute agent and generate memory summary in parallel"""
+        try:
+            # Execute main agent
+            agent_result = self.agents[specified_agent.strip()](**inputs)
+            return specified_agent.strip(), dict(agent_result)
+            
+        except Exception as e:
+            return specified_agent.strip(), {"error": str(e)}
+
+    def execute_agent_with_memory(self, specified_agent, inputs, query):
+        """Execute agent and generate memory summary in parallel"""
+        try:
+            # Execute main agent
+            agent_result = self.agents[specified_agent.strip()](**inputs)
+            agent_dict = dict(agent_result)
+            
+            # Generate memory summary
+            memory_result = self.memory_summarize_agent(
+                agent_response=specified_agent+' '+agent_dict['code']+'\n'+agent_dict['summary'],
+                user_goal=query
+            )
+            
+            return {
+                specified_agent.strip(): agent_dict,
+                'memory_'+specified_agent.strip(): str(memory_result.summary)
+            }
+        except Exception as e:
+            return {"error": str(e)}
+
+    def forward(self, query, specified_agent):
+        try:
+            # If specified_agent contains multiple agents separated by commas
+            # This is for handling multiple @agent mentions in one query
+            if "," in specified_agent:
+                agent_list = [agent.strip() for agent in specified_agent.split(",")]
+                return self.execute_multiple_agents(query, agent_list)
+            
+            # Process query with specified agent (single agent case)
+            dict_ = {}
+            dict_['dataset'] = self.dataset.retrieve(query)[0].text
+            dict_['styling_index'] = self.styling_index.retrieve(query)[0].text
+            dict_['hint'] = []
+            dict_['goal'] = query
+            dict_['Agent_desc'] = str(self.agent_desc)
+
+            # Prepare inputs
+            inputs = {x:dict_[x] for x in self.agent_inputs[specified_agent.strip()]}
+            inputs['hint'] = str(dict_['hint']).replace('[','').replace(']','')
+            
+            # Execute agent
+            result = self.agents[specified_agent.strip()](**inputs)
+            output_dict = {specified_agent.strip(): dict(result)}
+
+            if "error" in output_dict:
+                return {"response": f"Error executing agent: {output_dict['error']}"}
+
+            return output_dict
+
+        except Exception as e:
+            return {"response": f"This is the error from the system: {str(e)}"}
+    
+    def execute_multiple_agents(self, query, agent_list):
+        """Execute multiple agents sequentially on the same query"""
+        try:
+            # Initialize resources
+            dict_ = {}
+            dict_['dataset'] = self.dataset.retrieve(query)[0].text
+            dict_['styling_index'] = self.styling_index.retrieve(query)[0].text
+            dict_['hint'] = []
+            dict_['goal'] = query
+            dict_['Agent_desc'] = str(self.agent_desc)
+            
+            results = {}
+            code_list = []
+            
+            # Execute each agent sequentially
+            for agent_name in agent_list:
+                if agent_name not in self.agents:
+                    results[agent_name] = {"error": f"Agent '{agent_name}' not found"}
+                    continue
+                
+                # Prepare inputs for this agent
+                inputs = {x:dict_[x] for x in self.agent_inputs[agent_name] if x in dict_}
+                inputs['hint'] = str(dict_['hint']).replace('[','').replace(']','')
+                
+                # Execute agent
+                agent_result = self.agents[agent_name](**inputs)
+                agent_dict = dict(agent_result)
+                results[agent_name] = agent_dict
+                
+                # Collect code for later combination
+                if 'code' in agent_dict:
+                    code_list.append(agent_dict['code'])
+            
+            return results
+            
+        except Exception as e:
+            return {"response": f"Error executing multiple agents: {str(e)}"}
+
+
+# This is the auto_analyst with planner
+class auto_analyst(dspy.Module):
+    """Main analyst module that coordinates multiple agents using a planner"""
+    
+    def __init__(self, agents, retrievers):
+        # Initialize agent modules and retrievers
+        self.agents = {}
+        self.agent_inputs = {}
+        self.agent_desc = []
+        
+        for i, a in enumerate(agents):
+            name = a.__pydantic_core_schema__['schema']['model_name']
+            self.agents[name] = dspy.ChainOfThought(a)
+            self.agent_inputs[name] = {x.strip() for x in str(agents[i].__pydantic_core_schema__['cls']).split('->')[0].split('(')[1].split(',')}
+            self.agent_desc.append({name: get_agent_description(name)})
+        
+        # Initialize coordination agents
+        self.planner = dspy.ChainOfThought(analytical_planner)
+        self.refine_goal = dspy.ChainOfThought(goal_refiner_agent)
+        self.code_combiner_agent = dspy.ChainOfThought(code_combiner_agent)
+        self.story_teller = dspy.ChainOfThought(story_teller_agent)
+        self.memory_summarize_agent = dspy.ChainOfThought(m.memory_summarize_agent)
+                
+        # Initialize retrievers
+        self.dataset = retrievers['dataframe_index'].as_retriever(k=1)
+        self.styling_index = retrievers['style_index'].as_retriever(similarity_top_k=1)
+        
+        # Initialize thread pool for parallel execution
+        self.executor = ThreadPoolExecutor(max_workers=min(len(agents) + 2, os.cpu_count() * 2))
+
+    def execute_agent(self, agent_name, inputs):
+        """Execute a single agent with given inputs"""
+        try:
+            result = self.agents[agent_name.strip()](**inputs)
+            return agent_name.strip(), dict(result)
+        except Exception as e:
+            return agent_name.strip(), {"error": str(e)}
+
+    def get_plan(self, query):
+        """Get the analysis plan"""
+        dict_ = {}
+        dict_['dataset'] = self.dataset.retrieve(query)[0].text
+        dict_['styling_index'] = self.styling_index.retrieve(query)[0].text
+        dict_['goal'] = query
+        dict_['Agent_desc'] = str(self.agent_desc)
+        
+        plan = self.planner(goal=dict_['goal'], dataset=dict_['dataset'], Agent_desc=dict_['Agent_desc'])
+        return dict(plan)
+
+    async def execute_plan(self, query, plan):
+        """Execute the plan and yield results as they complete"""
+        dict_ = {}
+        dict_['dataset'] = self.dataset.retrieve(query)[0].text
+        dict_['styling_index'] = self.styling_index.retrieve(query)[0].text
+        dict_['hint'] = []
+        dict_['goal'] = query
+        
+        import json
+
+        # Clean and split the plan string into agent names
+        plan_text = plan.plan.replace("Plan", "").replace(":", "").strip()
+        plan_list = [agent.strip() for agent in plan_text.split("->") if agent.strip()]
+        # logger.log(f"Plan list: {plan_list}")
+        # Parse the attached plan_instructions into a dict
+        raw_instr = plan.plan_instructions
+        # logger.log(f"Raw instructions: {raw_instr}")
+        if isinstance(raw_instr, str):
+            try:
+                plan_instructions = json.loads(raw_instr)
+            except Exception:
+                plan_instructions = {}
+        elif isinstance(raw_instr, dict):
+            plan_instructions = raw_instr
+        else:
+            plan_instructions = {}
+        logger
+        # If no plan was produced, short-circuit
+        if not plan_list:
+            yield "plan_not_found", dict(plan), {"error": "No plan found"}
+            return
+
+        # Launch each agent in parallel, attaching its own instructions
+        futures = []
+        for agent_name in plan_list:
+            key = agent_name.strip()
+            # gather input fields except plan_instructions
+            inputs = {
+                param: dict_[param]
+                for param in self.agent_inputs[key]
+                if param != "plan_instructions"
+            }
+            # attach the specific instructions for this agent (or defaults)
+            if "plan_instructions" in self.agent_inputs[key]:
+                inputs['plan_instructions'] = plan_instructions
+                inputs["your_task"] = str(plan_instructions.get(
+                    key, ""
+                ))
+            # logger.log(f"Inputs: {inputs}")
+            future = self.executor.submit(self.execute_agent, agent_name, inputs)
+            futures.append((agent_name, inputs, future))
+        # Yield results as they complete 
+        completed_results = []
+        for agent_name, inputs, future in futures:
+            try:
+                name, result = await asyncio.get_event_loop().run_in_executor(None, future.result)
+                completed_results.append((name, result))
+                yield name, inputs, result
+            except Exception as e:
+                yield agent_name, inputs, {"error": str(e)}

src/db/schemas/models.py

@@ -1,7 +1,7 @@
-from sqlalchemy import create_engine, Column, Integer, String, ForeignKey, DateTime, Text, Float, Boolean, JSON, UniqueConstraint
+from sqlalchemy import create_engine, Column, Integer, String, ForeignKey, DateTime, Text, Float, Boolean, JSON
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker, relationship
-from datetime import datetime, UTC
+from datetime import datetime
 
 # Define the base class for declarative models
 Base = declarative_base()
@@ -11,14 +11,12 @@ class User(Base):
     __tablename__ = 'users'
     
     user_id = Column(Integer, primary_key=True, autoincrement=True)
-    username = Column(String, nullable=False)  # Removed unique=True
+    username = Column(String, unique=True, nullable=False)
     email = Column(String, unique=True, nullable=False)
-    created_at = Column(DateTime, default=lambda: datetime.now(UTC))
+    created_at = Column(DateTime, default=datetime.utcnow)
     # Add relationship for cascade options
     chats = relationship("Chat", back_populates="user", cascade="all, delete-orphan")
     usage_records = relationship("ModelUsage", back_populates="user")
-    deep_analysis_reports = relationship("DeepAnalysisReport", back_populates="user", cascade="all, delete-orphan")
-    template_preferences = relationship("UserTemplatePreference", back_populates="user", cascade="all, delete-orphan")
 
 # Define the Chats table
 class Chat(Base):
@@ -27,7 +25,7 @@ class Chat(Base):
     chat_id = Column(Integer, primary_key=True, autoincrement=True)
     user_id = Column(Integer, ForeignKey('users.user_id', ondelete="CASCADE"), nullable=True)
     title = Column(String, default='New Chat')
-    created_at = Column(DateTime, default=lambda: datetime.now(UTC))
+    created_at = Column(DateTime, default=datetime.utcnow)
     # Add relationships for cascade options
     user = relationship("User", back_populates="chats")
     messages = relationship("Message", back_populates="chat", cascade="all, delete-orphan")
@@ -41,7 +39,7 @@ class Message(Base):
     chat_id = Column(Integer, ForeignKey('chats.chat_id', ondelete="CASCADE"), nullable=False)
     sender = Column(String, nullable=False)  # 'user' or 'ai'
     content = Column(Text, nullable=False)
-    timestamp = Column(DateTime, default=lambda: datetime.now(UTC))
+    timestamp = Column(DateTime, default=datetime.utcnow)
     # Add relationship for cascade options
     chat = relationship("Chat", back_populates="messages")
     feedback = relationship("MessageFeedback", back_populates="message", uselist=False, cascade="all, delete-orphan")
@@ -62,7 +60,7 @@ class ModelUsage(Base):
     query_size = Column(Integer, default=0)  # Size in characters
     response_size = Column(Integer, default=0)  # Size in characters
     cost = Column(Float, default=0.0)  # Cost in USD
-    timestamp = Column(DateTime, default=lambda: datetime.now(UTC))
+    timestamp = Column(DateTime, default=datetime.utcnow)
     is_streaming = Column(Boolean, default=False)
     request_time_ms = Column(Integer, default=0)  # Request processing time in milliseconds
     # Add relationships
@@ -98,8 +96,8 @@ class CodeExecution(Base):
     error_messages = Column(Text, nullable=True)  # JSON map of error messages by agent
     
     # Metadata
-    created_at = Column(DateTime, default=lambda: datetime.now(UTC))
-    updated_at = Column(DateTime, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
+    created_at = Column(DateTime, default=datetime.utcnow)
+    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
     
 class MessageFeedback(Base):
     """Tracks user feedback and model settings for each message."""
@@ -118,120 +116,10 @@ class MessageFeedback(Base):
     max_tokens = Column(Integer, nullable=True)
 
     # Metadata
-    created_at = Column(DateTime, default=lambda: datetime.now(UTC))
-    updated_at = Column(DateTime, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
+    created_at = Column(DateTime, default=datetime.utcnow)
+    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
     
     # Relationship
     message = relationship("Message", back_populates="feedback")
-
-class DeepAnalysisReport(Base):
-    """Stores deep analysis reports with comprehensive analysis data and metadata."""
-    __tablename__ = 'deep_analysis_reports'
-    
-    report_id = Column(Integer, primary_key=True, autoincrement=True)
-    report_uuid = Column(String(100), unique=True, nullable=False)  # Frontend generated ID
-    user_id = Column(Integer, ForeignKey('users.user_id', ondelete="CASCADE"), nullable=True)
-    
-    # Analysis objective and status
-    goal = Column(Text, nullable=False)  # The analysis objective/question
-    status = Column(String(20), nullable=False, default='pending')  # 'pending', 'running', 'completed', 'failed'
-    
-    # Timing information
-    start_time = Column(DateTime, default=lambda: datetime.now(UTC))
-    end_time = Column(DateTime, nullable=True)
-    duration_seconds = Column(Integer, nullable=True)  # Calculated duration
-    
-    # Analysis components (stored as text/JSON)
-    deep_questions = Column(Text, nullable=True)  # Generated analytical questions
-    deep_plan = Column(Text, nullable=True)  # Analysis plan
-    summaries = Column(JSON, nullable=True)  # Array of analysis summaries
-    analysis_code = Column(Text, nullable=True)  # Generated Python code
-    plotly_figures = Column(JSON, nullable=True)  # Array of Plotly figure data
-    synthesis = Column(JSON, nullable=True)  # Array of synthesis insights
-    final_conclusion = Column(Text, nullable=True)  # Final analysis conclusion
-    
-    # Report output
-    html_report = Column(Text, nullable=True)  # Complete HTML report
-    report_summary = Column(Text, nullable=True)  # Brief summary for listing
-    
-    # Execution tracking
-    progress_percentage = Column(Integer, default=0)  # Progress 0-100
-    steps_completed = Column(JSON, nullable=True)  # Array of completed step names
-    error_message = Column(Text, nullable=True)  # Error details if failed
-    
-    # Model and cost tracking
-    model_provider = Column(String(50), nullable=True)
-    model_name = Column(String(100), nullable=True)
-    total_tokens_used = Column(Integer, default=0)
-    estimated_cost = Column(Float, default=0.0)  # Cost in USD
-    credits_consumed = Column(Integer, default=0)  # Credits deducted for this analysis
-    
-    # Metadata
-    created_at = Column(DateTime, default=lambda: datetime.now(UTC))
-    updated_at = Column(DateTime, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
-    
-    # Relationships
-    user = relationship("User", back_populates="deep_analysis_reports")
-    
-class AgentTemplate(Base):
-    """Stores predefined agent templates that users can enable/disable."""
-    __tablename__ = 'agent_templates'
-    
-    template_id = Column(Integer, primary_key=True, autoincrement=True)
-    
-    # Template definition
-    template_name = Column(String(100), nullable=False, unique=True)  # e.g., 'pytorch_specialist', 'data_cleaning_expert'
-    display_name = Column(String(200), nullable=True)  # User-friendly display name
-    description = Column(Text, nullable=False)  # Short description for template selection
-    prompt_template = Column(Text, nullable=False)  # Main prompt/instructions for agent behavior
-    
-    # Template appearance
-    icon_url = Column(String(500), nullable=True)  # URL to template icon (CDN, data URL, or relative path)
-    
-    # Template categorization
-    category = Column(String(50), nullable=True)  # 'Visualization', 'Modelling', 'Data Manipulation'
-    is_premium_only = Column(Boolean, default=False)  # True if template requires premium subscription
-    
-    # Agent variant support
-    variant_type = Column(String(20), default='individual')  # 'planner', 'individual', or 'both'
-    base_agent = Column(String(100), nullable=True)  # Base agent name for variants (e.g., 'preprocessing_agent')
-    
-    # Status and metadata
-    is_active = Column(Boolean, default=True)
-    
-    # Timestamps
-    created_at = Column(DateTime, default=lambda: datetime.now(UTC))
-    updated_at = Column(DateTime, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
-    
-    # Relationships
-    user_preferences = relationship("UserTemplatePreference", back_populates="template", cascade="all, delete-orphan")
-
-class UserTemplatePreference(Base):
-    """Tracks user preferences and usage for agent templates."""
-    __tablename__ = 'user_template_preferences'
-    
-    preference_id = Column(Integer, primary_key=True, autoincrement=True)
-    user_id = Column(Integer, ForeignKey('users.user_id', ondelete="CASCADE"), nullable=False)
-    template_id = Column(Integer, ForeignKey('agent_templates.template_id', ondelete="CASCADE"), nullable=False)
-    
-    # User preferences
-    is_enabled = Column(Boolean, default=True)  # Whether user has this template enabled
-    
-    # Usage tracking
-    usage_count = Column(Integer, default=0)  # Track how many times user has used this template
-    last_used_at = Column(DateTime, nullable=True)  # Last time user used this template
-    
-    # Timestamps
-    created_at = Column(DateTime, default=lambda: datetime.now(UTC))
-    updated_at = Column(DateTime, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
-    
-    # Relationships
-    user = relationship("User", back_populates="template_preferences")
-    template = relationship("AgentTemplate", back_populates="user_preferences")
-    
-    # Constraints - user can only have one preference record per template
-    __table_args__ = (
-        UniqueConstraint('user_id', 'template_id', name='unique_user_template_preference'),
-    )
     
     

src/managers/ai_manager.py

@@ -1,8 +1,10 @@
 import logging
 from typing import Optional, Dict, Any
+import time
 from src.db.schemas.models import ModelUsage
 from src.db.init_db import session_factory
-from datetime import datetime, UTC
+from datetime import datetime
+import tiktoken
 from src.routes.analytics_routes import handle_new_model_usage
 import asyncio
 
@@ -43,7 +45,7 @@ class AI_Manager:
                 query_size=query_size,
                 response_size=response_size,
                 cost=cost,
-                timestamp=datetime.now(UTC),
+                timestamp=datetime.utcnow(),
                 is_streaming=is_streaming,
                 request_time_ms=request_time_ms
             )