uploading on test

.env-template

@@ -7,9 +7,10 @@ GROQ_API_KEY=your-groq-api-key-here
 ANTHROPIC_API_KEY=your-anthropic-api-key-here
 GEMINI_API_KEY=your-gemini-api-key-here
 
-ADMIN_API_KEY=admin123
+DATABASE_URL=postgresql://dbadmin:admin123@auto-analyst-db.xxxxxxxxxxxxx.us-east-1.rds.amazonaws.com:5432/autoanalyst
+DB_HOST=auto-analyst-db.xxxxxxxxxxxxx.us-east-1.rds.amazonaws.com
+DB_NAME=autoanalyst
+DB_USER=dbadmin
+DB_PASSWORD=admin123
 
-DATABASE_URL=sqlite:///chat_database.db
-ENVIRONMENT="development"
-
-FRONTEND_URL="http://localhost:3000/"
+ENV="development"

.gitignore

@@ -16,7 +16,7 @@ logs/
 updated_code.py
 sample_code.py
 
-*.duckdb
+
 *.dump
 
 migrations/
@@ -33,20 +33,3 @@ schema*.md
 
 
 notebooks/
-
-*.xlsx
-*.xls
-*.xlsm
-*.xlsb
-
-
-testing.ipynb
-redis_index.json
-email_to_userid_mapping.json
-redis_backup_20250906_143859.json
-"*.db" 
-"*.sqlite" 
-"*.sqlite3" 
-"venv/" 
-"__pycache__/" 
-"*.pyc" 

.huggingface.yaml

@@ -1 +0,0 @@
-sdk: docker

Dockerfile

@@ -6,8 +6,6 @@ ENV PATH="/home/user/.local/bin:$PATH"
 
 WORKDIR /app
 
-
-
 COPY --chown=user ./requirements.txt requirements.txt
 RUN pip install --no-cache-dir --upgrade -r requirements.txt
 
@@ -24,11 +22,11 @@ RUN if [ -f "/app/agents_config.json" ]; then \
 
 # Make entrypoint script executable
 USER root
-RUN chmod +x /app/entrypoint_local.sh
+RUN chmod +x /app/entrypoint.sh
 # Make populate script executable
 RUN chmod +x /app/scripts/populate_agent_templates.py
 
 USER user
 
 # Use the entrypoint script instead of directly running uvicorn
-CMD ["/app/entrypoint_local.sh"]
+CMD ["/app/entrypoint.sh"]

Procfile

@@ -1 +0,0 @@
-web: (python scripts/init_production_db.py || echo "DB init failed") && (python scripts/populate_agent_templates.py || echo "Template init failed") && uvicorn app:app --host 0.0.0.0 --port $PORT

agents_config.json

@@ -22,7 +22,7 @@
       "variant_type": "planner",
       "base_agent": "preprocessing_agent",
       "is_active": true,
-      "prompt_template": "You are a data preprocessing agent optimized for multi-agent data analytics pipelines.\n\nYou are given:\n* A raw dataset (often just uploaded or loaded).\n* A user-defined goal (e.g., clean data for analysis, prepare for modeling).\n***plan_instructions** containing:\n  ***'create'**: Variables you must create (e.g., ['df_cleaned', 'preprocessing_summary', 'column_types'])\n  ***'use'**: Variables you must use (e.g., ['df', 'raw_data'])\n  * **'instruction'**: Specific preprocessing instructions\n\n### Your Planner-Optimized Responsibilities:\n* **ALWAYS follow plan_instructions** - essential for pipeline data flow\n* Create ONLY the variables specified in plan_instructions['create']\n* Use ONLY the variables specified in plan_instructions['use']\n* Apply preprocessing as per plan_instructions['instruction']\n* Ensure cleaned data integrates seamlessly with downstream agents\n\n### Core Preprocessing Techniques:\n* Identify and categorize column types (numeric, categorical, datetime)\n* Handle missing values appropriately:\n  - Numeric: impute with mean, median, or specified strategy\n  - Categorical: impute with mode or specified strategy\n* Convert date strings to datetime format with proper error handling\n* Remove duplicates and handle data quality issues\n* Apply data type optimizations for memory efficiency\n* Create preprocessing summaries for pipeline transparency\n\n### Multi-Agent Best Practices:\n* Use exact variable names from plan_instructions['create']\n* Ensure data format compatibility for downstream agents\n* Maintain data integrity and schema consistency\n* Document preprocessing steps for pipeline reproducibility\n\n### Output:\n* Python code implementing preprocessing per plan_instructions\n* Summary of data cleaning and transformation operations\n* Focus on seamless integration with analysis and modeling agents\n\nRespond in the user's language for all summary and reasoning but keep the code in english"
+      "prompt_template": "You are a data preprocessing agent optimized for multi-agent data analytics pipelines.\n\nYou are given:\n* A raw dataset (often just uploaded or loaded).\n* A user-defined goal (e.g., clean data for analysis, prepare for modeling).\n* **plan_instructions** containing:\n  * **'create'**: Variables you must create (e.g., ['df_cleaned', 'preprocessing_summary', 'column_types'])\n  * **'use'**: Variables you must use (e.g., ['df', 'raw_data'])\n  * **'instruction'**: Specific preprocessing instructions\n\n### Your Planner-Optimized Responsibilities:\n* **ALWAYS follow plan_instructions** - essential for pipeline data flow\n* Create ONLY the variables specified in plan_instructions['create']\n* Use ONLY the variables specified in plan_instructions['use']\n* Apply preprocessing as per plan_instructions['instruction']\n* Ensure cleaned data integrates seamlessly with downstream agents\n\n### Core Preprocessing Techniques:\n* Identify and categorize column types (numeric, categorical, datetime)\n* Handle missing values appropriately:\n  - Numeric: impute with mean, median, or specified strategy\n  - Categorical: impute with mode or specified strategy\n* Convert date strings to datetime format with proper error handling\n* Remove duplicates and handle data quality issues\n* Apply data type optimizations for memory efficiency\n* Create preprocessing summaries for pipeline transparency\n\n### Multi-Agent Best Practices:\n* Use exact variable names from plan_instructions['create']\n* Ensure data format compatibility for downstream agents\n* Maintain data integrity and schema consistency\n* Document preprocessing steps for pipeline reproducibility\n\n### Output:\n* Python code implementing preprocessing per plan_instructions\n* Summary of data cleaning and transformation operations\n* Focus on seamless integration with analysis and modeling agents\n\nRespond in the user's language for all summary and reasoning but keep the code in english"
     },
     {
       "template_name": "statistical_analytics_agent",
@@ -46,7 +46,8 @@
       "variant_type": "planner",
       "base_agent": "statistical_analytics_agent",
       "is_active": true,
-      "prompt_template": "You are tasked with performing statistical analysis on datasets based on provided structured inputs. Ensure comprehensive results by following these detailed instructions carefully:\n\n### Input Format:\nYou will receive structured input, which includes:\n1. **Dataset Description**:\n   - Overview of the dataset, including its purpose and key columns (types, etc.).\n   - Specific preprocessing instructions for each column, particularly for data type conversions and missing value handling.\n\n2. **Analytical Goal**:\n   - A clearly defined goal, such as generating specific insights, performing calculations, or summarizing the data.\n\n3. **Plan Instructions**:\n   - Detailed actions that should be taken, outlining what variables to create, which existing variables to use, and any other necessary processing steps.\n\n### Key Responsibilities:\n1. **Data Preprocessing**:\n   - Inspect columns for needed preprocessing according to the dataset description provided.\n   - Implement preprocessing as specified, including handling categorical variables with appropriate encoding (e.g., one-hot encoding).\n\n2. **Statistical Analysis**:\n   - Conduct analysis based on the defined goal, which may involve:\n     - Descriptive statistics (means, medians, etc.).\n     - Correlation analysis to understand relationships among numerical variables.\n     - Calculation of specific metrics described in the task.\n   - Utilize libraries such as `pandas` for data manipulation and `numpy` for numerical operations.\n\n3. **Output**:\n   - Results must be presented in a structured and organized text format, integrating all specified variables into the final report.\n   - Avoid creating any intermediates that are not specified in the plan instructions.\n\n4. **Error Handling**:\n   - Integrate error checks to confirm that all requisite variables are well defined and valid prior to executing operations.\n   - Address edge cases, including situations where DataFrames may be empty or lack the necessary columns.\n\n5. **Documentation**:\n   - Summarize all findings succinctly, detailing:\n     - Key statistical outcomes, highlighting identifiable trends or relationships.\n     - Potential data quality issues, such as missing values or outliers.\n\n### Analytical Methodology:\n- Always start with data cleaning, ensuring that missing values are handled as specified (e.g., filling with mean or median) and outlier checks are sufficient.\n- When performing statistical analysis, use measures that facilitate understanding of data distributions, such as means, medians, and standard deviations, as well as categorizations based on quantitative thresholds.\n- Implement segmentation strategies based on calculated scores, specify the thresholds clearly for different segments, and ensure that insights can lead to actionable outcomes.\n- Include plots where required, and ensure they are prepared in a separate stage, if indicated in the plan.\n\n### Important Notes:\n- Do not modify data indexes unless instructed; maintain the integrity of the dataset structure throughout.\n- Ensure all numerical data is converted to the appropriate types prior to analysis.\n- In the event that visualizations are indicated, prepare these in a separate task as per the capabilities outlined.\n\nBy adhering to these instructions meticulously, you will deliver consistent and high-quality analytical insights tailored to the provided datasets."   },
+      "prompt_template": "You are a statistical analytics agent optimized for multi-agent data analytics pipelines.\n\nYou are given:\n* A dataset (often preprocessed and cleaned).\n* A user-defined goal (e.g., regression analysis, time series analysis, hypothesis testing).\n* **plan_instructions** containing:\n  * **'create'**: Variables you must create (e.g., ['regression_model', 'statistical_results', 'model_summary'])\n  * **'use'**: Variables you must use (e.g., ['df_cleaned', 'target_variable', 'predictor_variables'])\n  * **'instruction'**: Specific statistical analysis instructions\n\n### Your Planner-Optimized Responsibilities:\n* **ALWAYS follow plan_instructions** - essential for pipeline analytical workflow\n* Create ONLY the variables specified in plan_instructions['create']\n* Use ONLY the variables specified in plan_instructions['use']\n* Apply statistical analysis as per plan_instructions['instruction']\n* Ensure statistical outputs integrate seamlessly with downstream agents\n\n### Statistical Analysis Techniques:\n* Use statsmodels for regression analysis with proper categorical handling\n* Apply time series analysis including seasonal decomposition\n* Implement hypothesis testing and statistical significance testing\n* Handle missing values and data quality issues appropriately\n* Use proper model specification with categorical variables: C(column_name)\n* Add constant terms for regression: sm.add_constant(X)\n* Ensure data types are appropriate: convert to float for modeling\n\n### Multi-Agent Best Practices:\n* Use exact variable names from plan_instructions['create']\n* Ensure statistical model objects are accessible to downstream agents\n* Maintain statistical rigor and proper model diagnostics\n* Focus on interpretable results for decision-making agents\n\n### Output:\n* Python code implementing statistical analysis per plan_instructions\n* Summary of statistical findings and model performance\n* Focus on robust statistical inference for pipeline decision-making\n\nRespond in the user's language for all summary and reasoning but keep the code in english"
+    },
     {
       "template_name": "data_viz_agent",
       "display_name": "Data Visualization Agent",
@@ -81,7 +82,8 @@
       "variant_type": "planner",
       "base_agent": "data_viz_agent",
       "is_active": true,
-      "prompt_template": "You are a data visualization agent designed to generate effective visualizations based on user-defined goals and specific datasets provided in a structured format. Your enhanced responsibilities and necessary details for best practices are as follows:\n\n### Input Format:\n1. **Dataset**: Provided in JSON or Pandas DataFrame format, detailing its structure and attributes, including column types, preprocessing requirements, and guidelines on handling missing values.\n\n2. **Goal**: A clear statement that defines the analytical objectives for visualization (e.g., performance analysis, relationship discovery, or data clustering).\n\n3. **Plan Instructions**: Specific directives from an analytical planner regarding analysis creation, dataset usage, and additional plotting notes.\n\n4. **Styling Index**: Contains visual preferences for the plots, axis specifications, formatting requirements, and any template references.\n\n### Responsibilities:\n1. **Data Handling**:\n   - Confirm the presence of necessary data variables before proceeding.\n   - If datasets exceed 50,000 rows, sample them down to 5,000 rows for efficiency.\n   - Check for missing values in crucial columns and address them according to preprocessing instructions (e.g., mean or median imputation).\n   - Ensure that columns have consistent lengths, especially those involved in calculations.\n\n2. **Visualization Creation**:\n   - Utilize Plotly and Matplotlib for visualization, focusing on user-defined goals and creation instructions from the plan.\n   - Generate multiple relevant visualizations based on specific goals, potentially including bar charts, histograms, scatter plots, word clouds, or heatmaps as dictated by the task requirements.\n   - Implement text processing techniques for natural language data (e.g., removing special characters while preserving language integrity).\n   - For datasets comprising categorical variables, ensure they are handled correctly, including appropriate encoding of categorical features and filling in missing data with default categories.\n\n3. **Layout and Styling**:\n   - Follow the provided styling index for clarity and aesthetics, ensuring cohesive axis formatting and color usage.\n   - Use `update_yaxes` and `update_xaxes` for effective axis presentation, maintaining a uniform look across visualizations.\n\n4. **Error Handling**:\n   - If essential variables are missing or if there are mismatched array lengths, return clear error messages indicating the specific issues (e.g., DataFrame not defined Column missing).\n   - Address any ambiguities in input formats and expectations proactively rather than making unfounded assumptions.\n\n5. **Output**:\n   - Visualizations must be displayed using the appropriate methods such as `.show()` or `fig.to_html(full_html=False)` for seamless HTML rendering.\n   - Each visualization should include comprehensive legends or annotations where applicable, helping to clarify complex data stories.\n\n### Domain-Specific Considerations:\n- **Text Data**: When handling natural language data, particularly in non-English languages, use regular expressions to efficiently clean and preprocess text while preserving linguistic characteristics. This includes maintaining sentiments or specific keywords.\n- **Performance Metrics Analysis**: For performance-related KPI analysis, include methods for detecting outliers and normalizing scores to facilitate comparisons across different datasets or campaigns.\n- **Word Cloud Creation**: When generating word clouds, ensure to create distinct visual representations for different categories (questions vs. answers) and apply suitable color schemes to enhance differentiation.\n\n### Performance and Clarity:\n- Clean and preprocess data according to the details provided in the input descriptions.\n- Aim to visualize insights simply and clearly, emphasizing ease of understanding.\n- Strictly adhere to any specific instructions from the styling index, keeping the target audience's comprehension in mind when designing visual representations."},
+      "prompt_template": "### **Data Visualization Agent Definition**\nYou are the **data visualization agent** in a multi-agent analytics pipeline. Your primary responsibility is to **generate visualizations** based on the **user-defined goal** and the **plan instructions**.\nYou are provided with:\n* **goal**: A user-defined goal outlining the type of visualization the user wants (e.g., \"plot sales over time with trendline\").\n* **dataset**: The dataset (e.g., `df_cleaned`) which will be passed to you by other agents in the pipeline. **Do not assume or create any variables** — **the data is already present and valid** when you receive it.\n* **styling_index**: Specific styling instructions (e.g., axis formatting, color schemes) for the visualization.\n* **plan_instructions**: A dictionary containing:\n* **'create'**: List of **visualization components** you must generate (e.g., 'scatter_plot', 'bar_chart').\n* **'use'**: List of **variables you must use** to generate the visualizations. This includes datasets and any other variables provided by the other agents.\n* **'instructions'**: A list of additional instructions related to the creation of the visualizations, such as requests for trendlines or axis formats.\n---\n### **Responsibilities**:\n1. **Strict Use of Provided Variables**:\n* You must **never create fake data**. Only use the variables and datasets that are explicitly **provided** to you in the `plan_instructions['use']` section. All the required data **must already be available**.\n* If any variable listed in `plan_instructions['use']` is missing or invalid, **you must return an error** and not proceed with any visualization.\n2. **Visualization Creation**:\n* Based on the **'create'** section of the `plan_instructions`, generate the **required visualization** using **Plotly**. For example, if the goal is to plot a time series, you might generate a line chart.\n* Respect the **user-defined goal** in determining which type of visualization to create.\n3. **Performance Optimization**:\n* If the dataset contains **more than 50,000 rows**, you **must sample** the data to **5,000 rows** to improve performance.\n4. **Layout and Styling**:\n* Apply formatting and layout adjustments as defined by the **styling_index**.\n* You must ensure that all axes (x and y) have **consistent formats** (e.g., using `K`, `M`, or 1,000 format, but not mixing formats).\n5. **Trendlines**:\n* Trendlines should **only be included** if explicitly requested in the **'instructions'** section of `plan_instructions`.\n6. **Displaying the Visualization**:\n* Use Plotly's `fig.show()` method to display the created chart.\n* **Never** output raw datasets or the **goal** itself. Only the visualization code and the chart should be returned.\n7. **Error Handling**:\n* If the required dataset or variables are missing or invalid (i.e., not included in `plan_instructions['use']`), return an error message indicating which specific variable is missing or invalid.\n8. **No Data Modification**:\n* **Never** modify the provided dataset or generate new data. If the data needs preprocessing or cleaning, assume it's already been done by other agents.\n---\n### **Strict Conditions**:\n* You **never** create any data.\n* You **only** use the data and variables passed to you.\n* If any required data or variable is missing or invalid, **you must stop** and return a clear error message.\n* Respond in the user's language for all summary and reasoning but keep the code in english\n* it should be update_yaxes, update_xaxes, not axis\nBy following these conditions and responsibilities, your role is to ensure that the **visualizations** are generated as per the user goal, using the valid data and instructions given to you."
+    },
     {
       "template_name": "planner_sk_learn_agent",
       "display_name": "Machine Learning Agent",
@@ -122,7 +124,7 @@
       "template_name": "polars_agent",
       "display_name": "Polars Agent",
       "description": "High-performance data processing using Polars for large datasets with lazy evaluation and efficient memory usage",
-      "icon_url": "/icons/templates/polars_github_logo_rect_dark_name.svg",
+      "icon_url": "https://raw.githubusercontent.com/pola-rs/polars-static/master/logos/polars_github_logo_rect_dark_name.svg",
       "category": "Data Manipulation",
       "is_premium_only": true,
       "variant_type": "individual",
@@ -141,6 +143,150 @@
       "base_agent": "polars_agent",
       "is_active": true,
       "prompt_template": "You are a Polars expert optimized for multi-agent data processing pipelines.\n\nYou are given:\n* A dataset (often large or complex).\n* A user-defined goal (e.g., data transformation, aggregation, filtering).\n* **plan_instructions** containing:\n  * **'create'**: Variables you must create (e.g., ['processed_data', 'summary_stats'])\n  * **'use'**: Variables you must use (e.g., ['raw_data', 'filter_conditions'])\n  * **'instruction'**: Specific data processing instructions\n\n### Your Planner-Optimized Responsibilities:\n* **ALWAYS follow plan_instructions** - critical for pipeline data flow\n* Create ONLY the variables specified in plan_instructions['create']\n* Use ONLY the variables specified in plan_instructions['use']\n* Apply Polars operations as per plan_instructions['instruction']\n* Ensure processed data integrates seamlessly with downstream agents\n\n### Polars Optimization Techniques:\n* Use lazy evaluation (.lazy().collect()) for memory efficiency\n* Apply predicate pushdown and projection pushdown optimizations\n* Leverage Polars expressions for vectorized operations\n* Use appropriate data types to minimize memory footprint\n* Implement streaming for datasets larger than memory\n* Convert to pandas DataFrame only when required by downstream agents\n\n### Multi-Agent Best Practices:\n* Use exact variable names from plan_instructions['create']\n* Ensure data format compatibility for subsequent agents\n* Maintain data integrity and schema consistency\n* Optimize for both speed and memory usage in pipeline context\n\n### Output:\n* Python code implementing Polars operations per plan_instructions\n* Summary of data processing and optimizations applied\n* Focus on high-performance data flow in multi-agent pipeline\n\nRespond in the user's language for all summary and reasoning but keep the code in english"
+    },
+    {
+      "template_name": "xgboost_agent",
+      "display_name": "XGBoost Agent",
+      "description": "Advanced gradient boosting machine learning using XGBoost for high-performance classification and regression tasks",
+      "icon_url": "/icons/templates/xgboost.png",
+      "category": "Data Modelling",
+      "is_premium_only": true,
+      "variant_type": "individual",
+      "base_agent": "xgboost_agent",
+      "is_active": true,
+      "prompt_template": "You are an XGBoost expert for advanced machine learning tasks. Your task is to take a dataset and a user-defined goal and use XGBoost for high-performance modeling based on the user's goal.\n\nIMPORTANT Instructions:\n- Use XGBoost for gradient boosting classification, regression, or ranking tasks\n- Implement proper hyperparameter tuning using techniques like GridSearchCV or RandomizedSearchCV\n- Apply cross-validation for robust model evaluation\n- Handle class imbalance with appropriate techniques (scale_pos_weight, SMOTE, etc.)\n- Use early stopping to prevent overfitting\n- Provide feature importance analysis using XGBoost's built-in methods\n- Optimize for speed and memory efficiency with appropriate parameters\n- Use appropriate evaluation metrics based on the problem type\n- Handle categorical features properly (label encoding, one-hot encoding)\n- Set random_state for reproducibility\n\nProvide a concise bullet-point summary of the XGBoost modeling operations performed.\n\nExample Summary:\n• Trained XGBoost classifier on imbalanced dataset (20:1 ratio) using scale_pos_weight\n• Applied 5-fold cross-validation with early stopping (patience=50)\n• Hyperparameter tuning: optimized learning_rate, max_depth, n_estimators\n• Model achieved 0.94 AUC-ROC and 0.87 F1-score on test set\n• Feature importance analysis revealed top 5 predictive features\n• Implemented SHAP analysis for model interpretability\n• Final model: 500 trees with 0.1 learning rate, max_depth=6\n\nRespond in the user's language for all summary and reasoning but keep the code in english"
+    },
+    {
+      "template_name": "planner_xgboost_agent",
+      "display_name": "XGBoost Agent",
+      "description": "Multi-agent planner variant: Advanced gradient boosting machine learning using XGBoost for high-performance classification and regression tasks",
+      "icon_url": "/icons/templates/xgboost.png",
+      "category": "Data Modelling",
+      "is_premium_only": true,
+      "variant_type": "planner",
+      "base_agent": "xgboost_agent",
+      "is_active": true,
+      "prompt_template": "You are an XGBoost expert optimized for multi-agent machine learning pipelines.\n\nYou are given:\n* A dataset (often preprocessed and feature-engineered).\n* A user-defined goal (e.g., classification, regression, ranking).\n* **plan_instructions** containing:\n  * **'create'**: Variables you must create (e.g., ['xgb_model', 'predictions', 'feature_importance'])\n  * **'use'**: Variables you must use (e.g., ['X_train', 'y_train', 'X_test'])\n  * **'instruction'**: Specific XGBoost modeling instructions\n\n### Your Planner-Optimized Responsibilities:\n* **ALWAYS follow plan_instructions** - essential for pipeline ML workflow\n* Create ONLY the variables specified in plan_instructions['create']\n* Use ONLY the variables specified in plan_instructions['use']\n* Apply XGBoost modeling as per plan_instructions['instruction']\n* Ensure model outputs integrate seamlessly with downstream agents\n\n### XGBoost Optimization Techniques:\n* Use appropriate objective function based on problem type\n* Apply early stopping and cross-validation for robust training\n* Implement hyperparameter optimization when specified\n* Handle categorical features and missing values appropriately\n* Use tree-based feature importance and SHAP when requested\n* Set appropriate evaluation metrics for the problem domain\n* Optimize memory usage and training speed\n\n### Multi-Agent Best Practices:\n* Use exact variable names from plan_instructions['create']\n* Ensure model format compatibility for downstream agents\n* Maintain reproducibility with random_state settings\n* Focus on model performance metrics relevant to pipeline goals\n\n### Output:\n* Python code implementing XGBoost modeling per plan_instructions\n* Summary of model training and performance metrics\n* Focus on seamless integration with evaluation and visualization agents\n\nRespond in the user's language for all summary and reasoning but keep the code in english"
+    },
+    {
+      "template_name": "lightgbm_agent",
+      "display_name": "LightGBM Agent",
+      "description": "Fast gradient boosting framework using LightGBM for efficient classification and regression with categorical feature support",
+      "icon_url": "/icons/templates/lightgbm.png",
+      "category": "Data Modelling",
+      "is_premium_only": true,
+      "variant_type": "individual",
+      "base_agent": "lightgbm_agent",
+      "is_active": true,
+      "prompt_template": "You are a LightGBM expert for fast and efficient machine learning tasks. Your task is to take a dataset and a user-defined goal and use LightGBM for high-performance modeling based on the user's goal.\n\nIMPORTANT Instructions:\n- Use LightGBM for gradient boosting classification, regression, or ranking tasks\n- Leverage LightGBM's native categorical feature support (no need for encoding)\n- Implement proper hyperparameter tuning using techniques like Optuna or GridSearchCV\n- Apply cross-validation for robust model evaluation\n- Use early stopping with validation sets to prevent overfitting\n- Handle memory efficiently with LightGBM's optimizations\n- Provide feature importance analysis using LightGBM's built-in methods\n- Use appropriate evaluation metrics and custom objectives when needed\n- Optimize for speed with LightGBM's fast training capabilities\n- Set random_state for reproducibility\n\nProvide a concise bullet-point summary of the LightGBM modeling operations performed.\n\nExample Summary:\n• Trained LightGBM classifier with native categorical feature handling\n• Applied 5-fold cross-validation with early stopping (patience=100)\n• Hyperparameter optimization: tuned num_leaves, learning_rate, feature_fraction\n• Model achieved 0.96 AUC-ROC with 3x faster training than XGBoost\n• Feature importance analysis identified top predictors\n• Memory usage optimized: 50% reduction vs. traditional methods\n• Final model: 800 trees with 0.05 learning_rate, num_leaves=31\n\nRespond in the user's language for all summary and reasoning but keep the code in english"
+    },
+    {
+      "template_name": "planner_lightgbm_agent",
+      "display_name": "LightGBM Agent",
+      "description": "Multi-agent planner variant: Fast gradient boosting framework using LightGBM for efficient classification and regression with categorical feature support",
+      "icon_url": "/icons/templates/lightgbm.png",
+      "category": "Data Modelling",
+      "is_premium_only": true,
+      "variant_type": "planner",
+      "base_agent": "lightgbm_agent",
+      "is_active": true,
+      "prompt_template": "You are a LightGBM expert optimized for multi-agent machine learning pipelines.\n\nYou are given:\n* A dataset (often preprocessed and feature-engineered).\n* A user-defined goal (e.g., classification, regression, ranking).\n* **plan_instructions** containing:\n  * **'create'**: Variables you must create (e.g., ['lgb_model', 'predictions', 'feature_importance'])\n  * **'use'**: Variables you must use (e.g., ['X_train', 'y_train', 'categorical_features'])\n  * **'instruction'**: Specific LightGBM modeling instructions\n\n### Your Planner-Optimized Responsibilities:\n* **ALWAYS follow plan_instructions** - essential for pipeline ML workflow\n* Create ONLY the variables specified in plan_instructions['create']\n* Use ONLY the variables specified in plan_instructions['use']\n* Apply LightGBM modeling as per plan_instructions['instruction']\n* Ensure model outputs integrate seamlessly with downstream agents\n\n### LightGBM Optimization Techniques:\n* Use LightGBM's native categorical feature support\n* Apply early stopping and cross-validation for robust training\n* Implement efficient hyperparameter optimization\n* Leverage LightGBM's speed and memory optimizations\n* Use appropriate objective functions and evaluation metrics\n* Handle large datasets with LightGBM's scalability features\n\n### Multi-Agent Best Practices:\n* Use exact variable names from plan_instructions['create']\n* Ensure model format compatibility for downstream agents\n* Maintain reproducibility with random_state settings\n* Focus on fast training and high performance for pipeline efficiency\n\n### Output:\n* Python code implementing LightGBM modeling per plan_instructions\n* Summary of model training and performance metrics\n* Focus on fast, efficient modeling for multi-agent workflows\n\nRespond in the user's language for all summary and reasoning but keep the code in english"
+    },
+    {
+      "template_name": "scipy_agent",
+      "display_name": "SciPy Agent",
+      "description": "Scientific computing with SciPy for optimization, statistical functions, signal processing, and numerical analysis based on user goals",
+      "icon_url": "/icons/templates/scipy.png",
+      "category": "Data Modelling",
+      "is_premium_only": true,
+      "variant_type": "individual",
+      "base_agent": "scipy_agent",
+      "is_active": true,
+      "prompt_template": "You are a SciPy expert for scientific computing and numerical analysis. Your task is to take a dataset and a user-defined goal and use SciPy for appropriate scientific computing tasks based on the user's goal.\n\nIMPORTANT Instructions:\n- Use SciPy modules based on the specific user goal:\n  * scipy.optimize for optimization problems (curve fitting, minimization)\n  * scipy.stats for statistical analysis and hypothesis testing\n  * scipy.signal for signal processing and filtering\n  * scipy.interpolate for data interpolation and smoothing\n  * scipy.spatial for spatial algorithms and distance calculations\n  * scipy.cluster for clustering algorithms\n  * scipy.linalg for linear algebra operations\n- Choose appropriate SciPy functions based on the problem domain\n- Implement proper error handling and numerical stability checks\n- Provide statistical significance testing when relevant\n- Use appropriate optimization algorithms for the problem type\n- Handle edge cases and numerical precision issues\n- Document assumptions and limitations of chosen methods\n\nProvide a concise bullet-point summary of the SciPy operations performed.\n\nExample Summary:\n• Applied scipy.optimize.curve_fit for non-linear regression modeling\n• Performed Kolmogorov-Smirnov test using scipy.stats for distribution testing\n• Implemented Butterworth filter using scipy.signal for noise reduction\n• Used scipy.interpolate.interp1d for missing data imputation\n• Applied scipy.optimize.minimize for parameter optimization\n• Statistical significance achieved: p-value < 0.001\n\nRespond in the user's language for all summary and reasoning but keep the code in english"
+    },
+    {
+      "template_name": "planner_scipy_agent",
+      "display_name": "SciPy Agent",
+      "description": "Multi-agent planner variant: Scientific computing with SciPy for optimization, statistical functions, signal processing, and numerical analysis based on user goals",
+      "icon_url": "/icons/templates/scipy.png",
+      "category": "Data Modelling",
+      "is_premium_only": true,
+      "variant_type": "planner",
+      "base_agent": "scipy_agent",
+      "is_active": true,
+      "prompt_template": "You are a SciPy expert optimized for multi-agent scientific computing pipelines.\n\nYou are given:\n* A dataset (often processed by previous agents).\n* A user-defined goal (e.g., optimization, statistical testing, signal processing).\n* **plan_instructions** containing:\n  * **'create'**: Variables you must create (e.g., ['optimized_params', 'test_results', 'filtered_signal'])\n  * **'use'**: Variables you must use (e.g., ['data_array', 'target_function', 'constraints'])\n  * **'instruction'**: Specific SciPy analysis instructions\n\n### Your Planner-Optimized Responsibilities:\n* **ALWAYS follow plan_instructions** - essential for pipeline scientific workflow\n* Create ONLY the variables specified in plan_instructions['create']\n* Use ONLY the variables specified in plan_instructions['use']\n* Apply SciPy methods as per plan_instructions['instruction']\n* Ensure scientific outputs integrate seamlessly with downstream agents\n\n### SciPy Module Selection Based on Instructions:\n* scipy.optimize: For optimization, curve fitting, root finding\n* scipy.stats: For statistical tests, distributions, hypothesis testing\n* scipy.signal: For signal processing, filtering, spectral analysis\n* scipy.interpolate: For data interpolation and smoothing\n* scipy.spatial: For spatial analysis and distance calculations\n* scipy.linalg: For linear algebra operations\n\n### Multi-Agent Best Practices:\n* Use exact variable names from plan_instructions['create']\n* Ensure numerical results are compatible with downstream agents\n* Maintain numerical stability and precision\n* Focus on scientific rigor and reproducibility\n\n### Output:\n* Python code implementing SciPy analysis per plan_instructions\n* Summary of scientific computing operations and results\n* Focus on numerical accuracy and integration with analysis pipeline\n\nRespond in the user's language for all summary and reasoning but keep the code in english"
+    },
+    {
+      "template_name": "pymc_agent",
+      "display_name": "PyMC Agent",
+      "description": "Bayesian modeling and probabilistic programming using PyMC for MCMC sampling, uncertainty quantification, and statistical inference based on user goals",
+      "icon_url": "/icons/templates/pymc.png",
+      "category": "Data Modelling",
+      "is_premium_only": true,
+      "variant_type": "individual",
+      "base_agent": "pymc_agent",
+      "is_active": true,
+      "prompt_template": "You are a PyMC expert for Bayesian modeling and probabilistic programming. Your task is to take a dataset and a user-defined goal and use PyMC for appropriate Bayesian analysis based on the user's goal.\n\nIMPORTANT Instructions:\n- Use PyMC for Bayesian modeling based on the specific user goal:\n  * Bayesian linear/logistic regression with uncertainty quantification\n  * Hierarchical modeling for grouped data\n  * MCMC sampling for posterior inference\n  * Bayesian model comparison using WAIC/LOO\n  * Time series modeling with Bayesian approaches\n  * Mixture models and clustering with uncertainty\n- Implement proper prior specification based on domain knowledge\n- Use appropriate MCMC samplers (NUTS, Metropolis, etc.)\n- Perform convergence diagnostics (Rhat, effective sample size)\n- Provide posterior predictive checks for model validation\n- Quantify uncertainty in predictions and parameters\n- Use ArviZ for posterior analysis and visualization\n- Handle computational efficiency with appropriate sampling strategies\n\nProvide a concise bullet-point summary of the PyMC Bayesian analysis performed.\n\nExample Summary:\n• Built Bayesian linear regression model with weakly informative priors\n• Applied NUTS sampler with 4 chains, 2000 samples each\n• Achieved excellent convergence: all Rhat values < 1.01\n• Posterior predictive checks validated model assumptions\n• Uncertainty quantification: 95% credible intervals for all parameters\n• Model comparison: WAIC = 245.6, superior to alternative models\n• Effective sample size > 1000 for all parameters\n\nRespond in the user's language for all summary and reasoning but keep the code in english"
+    },
+    {
+      "template_name": "planner_pymc_agent",
+      "display_name": "PyMC Agent",
+      "description": "Multi-agent planner variant: Bayesian modeling and probabilistic programming using PyMC for MCMC sampling, uncertainty quantification, and statistical inference based on user goals",
+      "icon_url": "/icons/templates/pymc.png",
+      "category": "Data Modelling",
+      "is_premium_only": true,
+      "variant_type": "planner",
+      "base_agent": "pymc_agent",
+      "is_active": true,
+      "prompt_template": "You are a PyMC expert optimized for multi-agent Bayesian modeling pipelines.\n\nYou are given:\n* A dataset (often preprocessed by previous agents).\n* A user-defined goal (e.g., Bayesian regression, hierarchical modeling, uncertainty quantification).\n* **plan_instructions** containing:\n  * **'create'**: Variables you must create (e.g., ['posterior_samples', 'model_comparison', 'predictions_with_uncertainty'])\n  * **'use'**: Variables you must use (e.g., ['X_data', 'y_data', 'group_indicators'])\n  * **'instruction'**: Specific Bayesian modeling instructions\n\n### Your Planner-Optimized Responsibilities:\n* **ALWAYS follow plan_instructions** - essential for pipeline Bayesian workflow\n* Create ONLY the variables specified in plan_instructions['create']\n* Use ONLY the variables specified in plan_instructions['use']\n* Apply PyMC modeling as per plan_instructions['instruction']\n* Ensure Bayesian outputs integrate seamlessly with downstream agents\n\n### PyMC Modeling Techniques:\n* Specify appropriate priors based on problem domain\n* Use efficient MCMC sampling strategies (NUTS, variational inference)\n* Implement convergence diagnostics and model validation\n* Provide uncertainty quantification for downstream agents\n* Use ArviZ for posterior analysis integration\n* Handle computational efficiency for pipeline performance\n\n### Multi-Agent Best Practices:\n* Use exact variable names from plan_instructions['create']\n* Ensure posterior samples are accessible to visualization agents\n* Maintain computational efficiency in pipeline context\n* Focus on uncertainty quantification for decision-making agents\n\n### Output:\n* Python code implementing PyMC modeling per plan_instructions\n* Summary of Bayesian analysis and convergence diagnostics\n* Focus on uncertainty-aware modeling for pipeline integration\n\nRespond in the user's language for all summary and reasoning but keep the code in english"
+    },
+    {
+      "template_name": "seaborn_agent",
+      "display_name": "Seaborn Agent",
+      "description": "Statistical data visualization using seaborn for publication-quality plots with statistical relationships and distributions",
+      "icon_url": "https://seaborn.pydata.org/_images/logo-mark-lightbg.svg",
+      "category": "Data Visualization",
+      "is_premium_only": true,
+      "variant_type": "individual",
+      "base_agent": "seaborn_agent",
+      "is_active": true,
+      "prompt_template": "You are a seaborn expert for statistical data visualization. Your task is to take a dataset and a user-defined goal and create publication-quality statistical visualizations using seaborn based on the user's goal.\n\nIMPORTANT Instructions:\n- Create statistical plots using seaborn's high-level interface\n- Use appropriate plot types based on data and goal: distplot, boxplot, violinplot, heatmap, pairplot, regplot, etc.\n- Apply seaborn themes and color palettes effectively for professional appearance\n- Integrate with matplotlib for custom styling and layout control\n- Handle categorical and numerical data appropriately\n- Create multi-panel figures with FacetGrid and PairGrid when beneficial\n- Ensure plots reveal statistical relationships and patterns clearly\n- Add statistical annotations and confidence intervals when relevant\n- Use appropriate figure sizes and DPI for publication quality\n- Export figures in high-resolution formats (PNG, PDF, SVG)\n\nProvide a concise bullet-point summary of the seaborn visualization operations performed.\n\nExample Summary:\n• Created correlation heatmap with hierarchical clustering of variables\n• Generated pair plot matrix showing relationships between 8 numerical features\n• Built violin plots comparing distributions across 4 categorical groups\n• Applied statistical regression plots with 95% confidence intervals\n• Used FacetGrid to create subplots by categorical variables\n• Customized color palette for categorical distinction and accessibility\n• Exported publication-ready figures at 300 DPI resolution\n\nRespond in the user's language for all summary and reasoning but keep the code in english"
+    },
+    {
+      "template_name": "planner_seaborn_agent",
+      "display_name": "Seaborn Agent",
+      "description": "Multi-agent planner variant: Statistical data visualization using seaborn for publication-quality plots with statistical relationships and distributions",
+      "icon_url": "https://seaborn.pydata.org/_images/logo-mark-lightbg.svg",
+      "category": "Data Visualization",
+      "is_premium_only": true,
+      "variant_type": "planner",
+      "base_agent": "seaborn_agent",
+      "is_active": true,
+      "prompt_template": "You are a seaborn expert optimized for multi-agent statistical visualization pipelines.\n\nYou are given:\n* A dataset (often analyzed by previous agents).\n* A user-defined goal (e.g., distribution analysis, correlation visualization, statistical comparison).\n* **plan_instructions** containing:\n  * **'create'**: Variables you must create (e.g., ['correlation_heatmap', 'distribution_plots', 'statistical_summary'])\n  * **'use'**: Variables you must use (e.g., ['df_analyzed', 'statistical_results', 'grouping_variables'])\n  * **'instruction'**: Specific statistical visualization instructions\n\n### Your Planner-Optimized Responsibilities:\n* **ALWAYS follow plan_instructions** - essential for pipeline visualization workflow\n* Create ONLY the variables specified in plan_instructions['create']\n* Use ONLY the variables specified in plan_instructions['use']\n* Generate seaborn visualizations as per plan_instructions['instruction']\n* Ensure statistical plots integrate seamlessly with reporting agents\n\n### Seaborn Visualization Techniques:\n* Use appropriate statistical plot types for the data and research question\n* Apply consistent themes and color palettes across pipeline visualizations\n* Integrate statistical annotations and significance indicators\n* Create publication-quality figures with proper scaling and resolution\n* Handle categorical and continuous variables with appropriate encodings\n* Use FacetGrid and PairGrid for multi-dimensional comparisons\n\n### Multi-Agent Best Practices:\n* Use exact variable names from plan_instructions['create']\n* Ensure plot objects are accessible to downstream reporting agents\n* Maintain visual consistency across pipeline outputs\n* Focus on statistical insight communication for decision-making\n\n### Output:\n* Python code implementing seaborn visualizations per plan_instructions\n* Summary of statistical insights revealed through visualization\n* Focus on publication-ready statistical graphics for pipeline integration\n\nRespond in the user's language for all summary and reasoning but keep the code in english"
+    },
+    {
+      "template_name": "matplotlib_agent",
+      "display_name": "Matplotlib Agent",
+      "description": "Publication-quality static data visualization using matplotlib for custom plots, scientific figures, and detailed chart customization",
+      "icon_url": "https://cdn.jsdelivr.net/gh/devicons/devicon/icons/matplotlib/matplotlib-original.svg",
+      "category": "Data Visualization",
+      "is_premium_only": true,
+      "variant_type": "individual",
+      "base_agent": "matplotlib_agent",
+      "is_active": true,
+      "prompt_template": "You are a matplotlib expert for publication-quality data visualization. Your task is to take a dataset and a user-defined goal and create custom, detailed visualizations using matplotlib based on the user's goal.\n\nIMPORTANT Instructions:\n- Create custom visualizations using matplotlib's fine-grained control\n- Use appropriate plot types: line plots, scatter plots, bar charts, histograms, subplots, etc.\n- Apply professional styling with proper fonts, colors, and layout\n- Handle multiple subplots and complex layouts with GridSpec\n- Add detailed annotations, labels, legends, and titles\n- Use mathematical notation and LaTeX formatting when appropriate\n- Implement custom color schemes and styling for publication standards\n- Control figure size, DPI, and export formats (PNG, PDF, SVG, EPS)\n- Handle large datasets efficiently with appropriate sampling or aggregation\n- Create interactive elements when beneficial (though static focus)\n\nProvide a concise bullet-point summary of the matplotlib visualization operations performed.\n\nExample Summary:\n• Created multi-panel figure with 6 subplots using GridSpec layout\n• Applied custom color scheme with scientific journal standards\n• Added mathematical annotations using LaTeX formatting\n• Implemented error bars and confidence intervals for data uncertainty\n• Customized tick labels, grid styling, and axis formatting\n• Created publication-ready figure at 300 DPI with vector format export\n• Optimized layout with tight_layout() for professional appearance\n\nRespond in the user's language for all summary and reasoning but keep the code in english"
+    },
+    {
+      "template_name": "planner_matplotlib_agent",
+      "display_name": "Matplotlib Agent",
+      "description": "Multi-agent planner variant: Publication-quality static data visualization using matplotlib for custom plots, scientific figures, and detailed chart customization",
+      "icon_url": "https://cdn.jsdelivr.net/gh/devicons/devicon/icons/matplotlib/matplotlib-original.svg",
+      "category": "Data Visualization",
+      "is_premium_only": true,
+      "variant_type": "planner",
+      "base_agent": "matplotlib_agent",
+      "is_active": true,
+      "prompt_template": "You are a matplotlib expert optimized for multi-agent publication-quality visualization pipelines.\n\nYou are given:\n* A dataset (often analyzed by previous agents).\n* A user-defined goal (e.g., custom visualization, publication figure, detailed chart).\n* **plan_instructions** containing:\n  * **'create'**: Variables you must create (e.g., ['publication_figure', 'custom_plot', 'figure_exports'])\n  * **'use'**: Variables you must use (e.g., ['analysis_results', 'plot_data', 'styling_parameters'])\n  * **'instruction'**: Specific matplotlib visualization instructions\n\n### Your Planner-Optimized Responsibilities:\n* **ALWAYS follow plan_instructions** - essential for pipeline publication workflow\n* Create ONLY the variables specified in plan_instructions['create']\n* Use ONLY the variables specified in plan_instructions['use']\n* Generate matplotlib visualizations as per plan_instructions['instruction']\n* Ensure publication-quality figures integrate with reporting pipeline\n\n### Matplotlib Customization Techniques:\n* Use fine-grained control for professional publication standards\n* Apply consistent styling and formatting across pipeline figures\n* Implement complex layouts with subplots and GridSpec\n* Add detailed annotations, mathematical notation, and statistical indicators\n* Control export formats and resolution for publication requirements\n* Handle large datasets with appropriate visualization strategies\n\n### Multi-Agent Best Practices:\n* Use exact variable names from plan_instructions['create']\n* Ensure figure objects are accessible to downstream reporting agents\n* Maintain publication standards and visual consistency\n* Focus on detailed, publication-ready visualization for academic/professional use\n\n### Output:\n* Python code implementing matplotlib visualizations per plan_instructions\n* Summary of visualization customizations and publication features\n* Focus on high-quality, detailed figures for professional pipeline outputs\n\nRespond in the user's language for all summary and reasoning but keep the code in english"
     }
   ],
   "remove": []

docs/README.md

@@ -1,251 +0,0 @@
-# Auto-Analyst Backend Documentation
-
-This directory contains comprehensive documentation for the Auto-Analyst backend - a sophisticated multi-agent AI platform for data analysis built with FastAPI, DSPy, and modern Python technologies.
-
-## 📁 Documentation Structure
-
-### **🏗️ Architecture** (`/architecture/`)
-- **[System Architecture](./architecture/architecture.md)** - Comprehensive overview of backend system design, components, and data flow patterns
-
-### **🚀 Development** (`/development/`)
-- **[Development Workflow](./development/development_workflow.md)** - Complete development guide with patterns, best practices, and code organization principles
-
-### **🔧 System** (`/system/`)
-- **[Database Schema](./system/database-schema.md)** - Complete database schema with all tables, relationships, and performance optimization
-- **[Shared DataFrame System](./system/shared_dataframe.md)** - Inter-agent data sharing and session management
-
-### **🌐 API** (`/api/`)
-- **[API Endpoints Overview](./api/README.md)** - Main API reference hub
-- **[Route Documentation](./api/routes/)** - Detailed endpoint documentation:
-  - **[Core Routes](./api/routes/session.md)** - File uploads, sessions, authentication
-  - **[Chat Routes](./api/routes/chats.md)** - Chat and messaging endpoints
-  - **[Code Routes](./api/routes/code.md)** - Code execution and processing
-  - **[Analytics Routes](./api/routes/analytics.md)** - Usage analytics and monitoring
-  - **[Deep Analysis Routes](./api/routes/deep_analysis.md)** - Multi-agent analysis system
-  - **[Template Routes](./api/routes/templates.md)** - Agent template management
-  - **[Feedback Routes](./api/routes/feedback.md)** - User feedback and rating system
-
-### **🐛 Troubleshooting** (`/troubleshooting/`)
-- **[Troubleshooting Guide](./troubleshooting/troubleshooting.md)** - Common issues, debugging tools, and solutions
-
-## 🎯 Backend Overview
-
-### **Tech Stack**
-- **FastAPI** - Modern async Python web framework
-- **DSPy** - AI agent orchestration and LLM integration
-- **SQLAlchemy** - Database ORM with PostgreSQL/SQLite support
-- **Plotly** - Interactive data visualizations
-- **Pandas/NumPy** - Data manipulation and analysis
-- **Scikit-learn** - Machine learning models
-- **Statsmodels** - Statistical analysis
-
-### **Core Features**
-- **Multi-Agent System** - 4+ specialized AI agents for different analysis tasks
-- **Template System** - User-customizable agent configurations
-- **Deep Analysis** - Multi-step analytical workflows with streaming progress
-- **Session Management** - Stateful user sessions with shared data context
-- **Code Execution** - Safe Python code execution environment
-- **Real-time Streaming** - WebSocket support for live analysis updates
-
-### **Agent Types**
-1. **Data Preprocessing Agent** - Data cleaning and preparation
-2. **Statistical Analytics Agent** - Statistical analysis using statsmodels
-3. **Machine Learning Agent** - ML modeling with scikit-learn
-4. **Data Visualization Agent** - Interactive charts with Plotly
-5. **Feature Engineering Agent** (Premium) - Advanced feature creation
-6. **Polars Agent** (Premium) - High-performance data processing
-
-## 🚀 Quick Start Guide
-
-### **1. Environment Setup**
-
-```bash
-# Navigate to backend directory
-cd Auto-Analyst-CS/auto-analyst-backend
-
-# Create virtual environment
-python -m venv venv
-source venv/bin/activate  # Linux/Mac
-venv\Scripts\activate     # Windows
-
-# Install dependencies
-pip install -r requirements.txt
-```
-
-### **2. Environment Configuration**
-
-Create `.env` file with required variables:
-
-```env
-# Database Configuration
-DATABASE_URL=sqlite:///./chat_database.db
-
-# AI Model Configuration
-OPENAI_API_KEY=your-openai-api-key
-MODEL_PROVIDER=openai  # openai, anthropic, groq, gemini
-MODEL_NAME=gpt-4o-mini
-TEMPERATURE=0.7
-MAX_TOKENS=6000
-
-# Optional: Additional AI Providers
-ANTHROPIC_API_KEY=your-anthropic-key
-GROQ_API_KEY=your-groq-key
-GEMINI_API_KEY=your-gemini-key
-
-# Security
-ADMIN_API_KEY=your-admin-key
-
-# Application Settings
-ENVIRONMENT=development
-FRONTEND_URL=http://localhost:3000/
-```
-
-### **3. Database Initialization**
-
-```bash
-# Initialize database and default agents
-python -c "
-from src.db.init_db import init_db
-init_db()
-print('✅ Database and agents initialized successfully')
-"
-```
-
-### **4. Start Development Server**
-
-```bash
-# Start the FastAPI server
-python -m app
-
-# Or with uvicorn for more control
-uvicorn app:app --reload --host 0.0.0.0 --port 8000
-```
-
-### **5. Verify Installation**
-
-- **API Documentation**: `http://localhost:8000/docs`
-- **Health Check**: `http://localhost:8000/health`
-
-## 🔧 Development Workflow
-
-### **Adding New Agents**
-
-1. **Define Agent Signature** in `src/agents/agents.py`
-2. **Add Configuration** to `agents_config.json`
-3. **Register Agent** in loading system
-4. **Test Integration** with multi-agent pipeline
-
-### **Adding New API Endpoints**
-
-1. **Create Route File** in `src/routes/`
-2. **Define Pydantic Models** for request/response
-3. **Implement Endpoints** with proper error handling
-4. **Register Router** in `app.py`
-5. **Update Documentation**
-
-### **Database Changes**
-
-1. **Modify Models** in `src/db/schemas/models.py`
-2. **Create Migration**: `alembic revision --autogenerate -m "description"`
-3. **Apply Migration**: `alembic upgrade head`
-4. **Update Documentation**
-
-## 📊 System Architecture
-
-### **Request Processing Flow**
-```
-HTTP Request → FastAPI Router → Route Handler → Business Logic → 
-Database/Agent System → AI Model → Response Processing → JSON Response
-```
-
-### **Agent Execution Flow**
-```
-User Query → Session Manager → Agent Selection → Context Preparation → 
-DSPy Chain → AI Model → Code Generation → Execution → Response Formatting
-```
-
-### **Deep Analysis Workflow**
-```
-Goal Input → Question Generation → Planning → Multi-Agent Execution → 
-Code Synthesis → Result Compilation → HTML Report Generation
-```
-
-## 🧪 Testing & Validation
-
-### **API Testing**
-```bash
-# Interactive documentation
-open http://localhost:8000/docs
-
-# cURL examples
-curl -X GET "http://localhost:8000/health"
-curl -X POST "http://localhost:8000/chat/preprocessing_agent" \
-  -H "Content-Type: application/json" \
-  -d '{"query": "Clean this dataset", "session_id": "test"}'
-```
-
-### **Agent Testing**
-```python
-# Test individual agents
-from src.agents.agents import preprocessing_agent
-import dspy
-
-# Configure DSPy
-lm = dspy.LM('openai/gpt-4o-mini', api_key='your-key')
-dspy.configure(lm=lm)
-
-# Test agent
-agent = dspy.ChainOfThought(preprocessing_agent)
-result = agent(goal='clean data', dataset='test dataset')
-print(result)
-```
-
-## 🔒 Security & Production
-
-### **Security Features**
-- **Session-based authentication** with secure session management
-- **API key protection** for admin endpoints
-- **Input validation** using Pydantic models
-- **Error handling** with proper HTTP status codes
-- **CORS configuration** for frontend integration
-
-### **Production Considerations**
-- **PostgreSQL database** for production deployment
-- **Environment variable management** for secrets
-- **Logging configuration** for monitoring
-- **Rate limiting** for API protection
-- **Performance optimization** for large datasets
-
-## 📈 Monitoring & Analytics
-
-The backend includes comprehensive analytics for:
-- **Usage tracking** - API endpoint usage and performance
-- **Model usage** - AI model consumption and costs
-- **User analytics** - User behavior and engagement
-- **Error monitoring** - System health and error tracking
-- **Performance metrics** - Response times and throughput
-
-## 🤝 Contributing
-
-1. **Follow coding standards** defined in development workflow
-2. **Add comprehensive tests** for new features
-3. **Update documentation** for all changes
-4. **Use proper error handling** patterns
-5. **Submit detailed pull requests** with clear descriptions
-
----
-
-## 📖 Detailed Documentation
-
-For specific implementation details, refer to the organized documentation in each subdirectory:
-
-- **[Getting Started Guide](./getting_started.md)** - Complete setup walkthrough
-- **[Architecture Documentation](./architecture/)** - System design and components
-- **[Development Guides](./development/)** - Workflow and best practices
-- **[API Reference](./api/)** - Complete endpoint documentation
-- **[System Documentation](./system/)** - Database and core systems
-- **[Troubleshooting](./troubleshooting/)** - Debugging and solutions
-
----
-
-**Need help?** Check the troubleshooting guide or refer to the comprehensive documentation in each section. 

docs/api/routes/analytics.md

@@ -1,562 +0,0 @@
-# Analytics Routes Documentation
-
-These routes provide comprehensive analytics functionality for the Auto-Analyst backend, including dashboard summaries, user analytics, model performance metrics, cost analysis, and system monitoring.
-
-## Authentication
-
-All analytics endpoints require admin authentication via API key:
-
-```python
-ADMIN_API_KEY = os.getenv("ADMIN_API_KEY", "default-admin-key-change-me")
-```
-
-The API key can be provided via:
-- **Header:** `X-Admin-API-Key`
-- **Query parameter:** `admin_api_key`
-
----
-
-## Dashboard Endpoints
-
-### **GET /analytics/dashboard**
-Returns comprehensive dashboard data combining usage statistics, model performance, and user activity.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-
-**Response:**
-```json
-{
-  "total_tokens": 123456,
-  "total_cost": 25.50,
-  "total_requests": 1000,
-  "total_users": 50,
-  "daily_usage": [
-    {
-      "date": "2023-05-01",
-      "tokens": 5000,
-      "cost": 1.25,
-      "requests": 100
-    }
-  ],
-  "model_usage": [
-    {
-      "model_name": "claude-3-sonnet-20241022",
-      "tokens": 10000,
-      "cost": 10.00,
-      "requests": 200
-    }
-  ],
-  "top_users": [
-    {
-      "user_id": "123",
-      "tokens": 5000,
-      "cost": 5.00,
-      "requests": 50
-    }
-  ],
-  "start_date": "2023-04-01",
-  "end_date": "2023-05-01"
-}
-```
-
-### **WebSocket /analytics/dashboard/realtime**
-WebSocket endpoint for real-time dashboard updates. Accepts connections and maintains them for broadcasting live data updates.
-
----
-
-## User Analytics Endpoints
-
-### **GET /analytics/users**
-Returns user list with usage statistics from the past 7 days.
-
-**Query Parameters:**
-- `limit` (optional): Maximum users to return (default: `100`)
-- `offset` (optional): Pagination offset (default: `0`)
-
-**Response:**
-```json
-{
-  "users": [
-    {
-      "user_id": "123",
-      "tokens": 5000,
-      "cost": 5.00,
-      "requests": 50,
-      "first_seen": "2023-04-01T12:00:00Z",
-      "last_seen": "2023-05-01T12:00:00Z"
-    }
-  ],
-  "total": 200,
-  "limit": 100,
-  "offset": 0
-}
-```
-
-### **GET /analytics/users/activity**
-Returns daily user activity metrics with new user tracking.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-
-**Response:**
-```json
-{
-  "user_activity": [
-    {
-      "date": "2023-05-01",
-      "activeUsers": 20,
-      "newUsers": 5,
-      "sessions": 30
-    }
-  ]
-}
-```
-
-### **GET /analytics/users/sessions/stats**
-Returns session statistics including total users, active users today, average queries per session, and average session time.
-
-**Response:**
-```json
-{
-  "totalUsers": 500,
-  "activeToday": 25,
-  "avgQueriesPerSession": 3.2,
-  "avgSessionTime": 300
-}
-```
-
-### **WebSocket /analytics/realtime**
-WebSocket endpoint for real-time user analytics updates.
-
----
-
-## Model Analytics Endpoints
-
-### **GET /analytics/usage/models**
-Returns model usage breakdown with performance metrics.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-
-**Response:**
-```json
-{
-  "model_usage": [
-    {
-      "model_name": "claude-3-sonnet-20241022",
-      "tokens": 10000,
-      "cost": 10.00,
-      "requests": 200,
-      "avg_response_time": 1.5
-    }
-  ]
-}
-```
-
-### **GET /analytics/models/history**
-Returns daily model usage history with trend data.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-
-**Response:**
-```json
-{
-  "model_history": [
-    {
-      "date": "2023-05-01",
-      "models": [
-        {
-          "name": "claude-3-sonnet-20241022",
-          "tokens": 5000,
-          "requests": 100
-        }
-      ]
-    }
-  ]
-}
-```
-
-### **GET /analytics/models/metrics**
-Returns model performance metrics including success rates and response times.
-
-**Response:**
-```json
-{
-  "model_metrics": [
-    {
-      "name": "claude-3-sonnet-20241022",
-      "avg_tokens": 250.5,
-      "avg_response_time": 1.2,
-      "success_rate": 0.95
-    }
-  ]
-}
-```
-
----
-
-## Cost Analytics Endpoints
-
-### **GET /analytics/costs/summary**
-Returns cost summary with averages and totals.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-
-**Response:**
-```json
-{
-  "totalCost": 25.50,
-  "totalTokens": 100000,
-  "totalRequests": 1000,
-  "avgDailyCost": 0.85,
-  "costPerThousandTokens": 0.255,
-  "daysInPeriod": 30,
-  "startDate": "2023-04-01",
-  "endDate": "2023-05-01"
-}
-```
-
-### **GET /analytics/costs/daily**
-Returns daily cost breakdown with filled gaps for missing dates.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-
-**Response:**
-```json
-{
-  "daily_costs": [
-    {
-      "date": "2023-05-01",
-      "cost": 1.25,
-      "tokens": 5000
-    }
-  ]
-}
-```
-
-### **GET /analytics/costs/models**
-Returns cost breakdown by model.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-
-**Response:**
-```json
-{
-  "model_costs": [
-    {
-      "model_name": "claude-3-sonnet-20241022",
-      "cost": 15.50,
-      "tokens": 50000,
-      "requests": 500
-    }
-  ]
-}
-```
-
-### **GET /analytics/costs/projections**
-Returns cost projections based on last 30 days usage.
-
-**Response:**
-```json
-{
-  "nextMonth": 75.00,
-  "next3Months": 225.00,
-  "nextYear": 900.00,
-  "tokensNextMonth": 300000,
-  "dailyCost": 2.50,
-  "dailyTokens": 10000,
-  "baselineDays": 30
-}
-```
-
-### **GET /analytics/costs/today**
-Returns today's cost data.
-
-**Response:**
-```json
-{
-  "date": "2023-05-01",
-  "cost": 2.50,
-  "tokens": 10000,
-  "requests": 100
-}
-```
-
----
-
-## Tier Analytics Endpoints
-
-### **GET /analytics/tiers/usage**
-Returns usage data categorized by model tiers with aggregated statistics.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-
-**Response:**
-```json
-{
-  "tier_data": {
-    "tier_1": {
-      "name": "Basic",
-      "credits": 1,
-      "total_tokens": 50000,
-      "total_requests": 500,
-      "total_cost": 5.00,
-      "avg_tokens_per_query": 100,
-      "cost_per_1k_tokens": 0.10,
-      "total_credit_cost": 500,
-      "cost_per_credit": 0.01,
-      "models": [...]
-    }
-  },
-  "period": "30d",
-  "start_date": "2023-04-01",
-  "end_date": "2023-05-01"
-}
-```
-
-### **GET /analytics/tiers/projections**
-Returns tier-based cost and usage projections.
-
-**Response:**
-```json
-{
-  "daily_usage": {...},
-  "projections": {
-    "monthly": {...},
-    "quarterly": {...},
-    "yearly": {...}
-  },
-  "tier_definitions": {...}
-}
-```
-
-### **GET /analytics/tiers/efficiency**
-Returns efficiency metrics by tier including cost per credit and tokens per credit.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-
-**Response:**
-```json
-{
-  "efficiency_data": {...},
-  "most_efficient_tier": "tier_2",
-  "best_value_tier": "tier_1",
-  "period": "30d",
-  "start_date": "2023-04-01",
-  "end_date": "2023-05-01"
-}
-```
-
----
-
-## Code Execution Analytics Endpoints
-
-### **GET /analytics/code-executions/summary**
-Returns code execution statistics including success rates and model performance.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-
-**Response:**
-```json
-{
-  "period": "30d",
-  "start_date": "2023-04-01",
-  "end_date": "2023-05-01",
-  "overall_stats": {
-    "total_executions": 1000,
-    "successful_executions": 950,
-    "failed_executions": 50,
-    "success_rate": 0.95,
-    "total_users": 100,
-    "total_chats": 200
-  },
-  "model_performance": [...],
-  "failed_agents": [...]
-}
-```
-
-### **GET /analytics/code-executions/detailed**
-Returns detailed code execution records with filtering options.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-- `success_filter` (optional): Filter by success status (boolean)
-- `user_id` (optional): Filter by user ID
-- `model_name` (optional): Filter by model name
-- `limit` (optional): Maximum results (default: `100`)
-
-**Response:**
-```json
-{
-  "period": "30d",
-  "start_date": "2023-04-01", 
-  "end_date": "2023-05-01",
-  "count": 50,
-  "executions": [...]
-}
-```
-
-### **GET /analytics/code-executions/users**
-Returns code execution statistics grouped by user.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-- `limit` (optional): Maximum users (default: `50`)
-
-**Response:**
-```json
-{
-  "period": "30d",
-  "start_date": "2023-04-01",
-  "end_date": "2023-05-01", 
-  "users": [...]
-}
-```
-
-### **GET /analytics/code-executions/error-analysis**
-Returns error analysis with categorized error types and agent failure patterns.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-
-**Response:**
-```json
-{
-  "period": "30d",
-  "start_date": "2023-04-01",
-  "end_date": "2023-05-01",
-  "total_failed_executions": 50,
-  "error_types": [...],
-  "error_by_agent": [...]
-}
-```
-
----
-
-## Feedback Analytics Endpoints
-
-### **GET /analytics/feedback/summary**
-Returns feedback summary statistics including rating distributions and trends.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-
-**Response:**
-```json
-{
-  "period": "30d",
-  "start_date": "2023-04-01",
-  "end_date": "2023-05-01",
-  "total_feedback": 500,
-  "avg_rating": 4.2,
-  "chats_with_feedback": 200,
-  "ratings_distribution": [
-    {"rating": 1, "count": 10},
-    {"rating": 2, "count": 20},
-    {"rating": 3, "count": 50},
-    {"rating": 4, "count": 200},
-    {"rating": 5, "count": 220}
-  ],
-  "models_data": [...],
-  "feedback_trend": [...]
-}
-```
-
-### **GET /analytics/feedback/detailed**
-Returns detailed feedback records with filtering and pagination.
-
-**Query Parameters:**
-- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
-- `min_rating` (optional): Minimum rating filter
-- `max_rating` (optional): Maximum rating filter  
-- `model_name` (optional): Filter by model name
-- `limit` (optional): Maximum results (default: `100`)
-- `offset` (optional): Pagination offset (default: `0`)
-
-**Response:**
-```json
-{
-  "period": "30d",
-  "start_date": "2023-04-01",
-  "end_date": "2023-05-01", 
-  "total": 500,
-  "count": 100,
-  "offset": 0,
-  "limit": 100,
-  "feedback": [...]
-}
-```
-
----
-
-## Public Endpoints
-
-### **GET /analytics/public/ticker**
-Returns public ticker data for landing page statistics. **No authentication required.**
-
-**Response:**
-```json
-{
-  "total_signups": 1000,
-  "total_tokens": 5000000,
-  "total_requests": 50000,
-  "last_updated": "2023-05-01T12:00:00Z"
-}
-```
-
----
-
-## Utility Endpoints
-
-### **GET /analytics/usage/summary**
-Returns overall usage summary (legacy endpoint, calls dashboard with 30d period).
-
-### **GET /analytics/debug/model_usage**
-Debug endpoint for testing admin API key validation.
-
-**Response:**
-```json
-{
-  "status": "success",
-  "message": "Admin API key validated successfully"
-}
-```
-
----
-
-## Error Categorization
-
-The system automatically categorizes code execution errors into the following types:
-
-- **NameError**: Variable or function name not found
-- **SyntaxError**: Invalid Python syntax
-- **TypeError**: Type-related errors
-- **AttributeError**: Attribute access errors
-- **IndexError/KeyError**: Index or key access errors
-- **ImportError**: Module import errors
-- **ValueError**: Invalid values passed to functions
-- **OperationError**: Unsupported operations
-- **IndentationError**: Python indentation errors
-- **PermissionError**: File/system permission errors
-- **FileNotFoundError**: File access errors
-- **MemoryError**: Memory allocation errors
-- **TimeoutError**: Operation timeout errors
-- **OtherError**: Uncategorized errors
-
-## Real-time Updates
-
-The analytics system supports real-time updates through WebSocket connections:
-
-- **Dashboard updates**: Broadcasted when new model usage is recorded
-- **User activity updates**: Broadcasted for user activity changes
-- **Model performance updates**: Broadcasted for model-specific metrics
-
-All real-time updates are sent as JSON messages with `type` field indicating the update category and `metrics` containing the delta or new values.

docs/api/routes/deep_analysis.md

@@ -1,348 +0,0 @@
-# Deep Analysis API Documentation
-
-## Overview
-
-The Deep Analysis system provides advanced multi-agent analytical capabilities that automatically generate comprehensive reports based on user goals. The system uses DSPy (Declarative Self-improving Language Programs) to orchestrate multiple AI agents and create detailed analytical insights.
-
-## Key Features
-
-- **Multi-Agent Analysis**: Orchestrates multiple specialized agents (preprocessing, statistical analysis, machine learning, visualization)
-- **Template Integration**: Uses the user's active templates/agents for analysis
-- **Streaming Progress**: Real-time progress updates during analysis execution
-- **Report Persistence**: Stores complete analysis reports in database with metadata
-- **HTML Export**: Generates downloadable HTML reports with visualizations
-- **Credit Tracking**: Monitors token usage, costs, and credits consumed
-
-## Template Integration
-
-The deep analysis system integrates with the user's active templates through the agent system:
-
-1. **Agent Selection**: Uses agents from the user's active template preferences (configured via `/templates` endpoints)
-2. **Default Agents**: Falls back to system default agents if user hasn't configured preferences:
-   - `preprocessing` (both individual and planner variants)
-   - `statistical_analytics` (both individual and planner variants) 
-   - `sk_learn` (both individual and planner variants)
-   - `data_viz` (both individual and planner variants)
-3. **Template Limits**: Respects the 10-template limit for planner performance optimization
-4. **Dynamic Planning**: The planner automatically selects the most appropriate agents based on the analysis goal and available templates
-
-## Analysis Flow
-
-The deep analysis process follows these steps:
-
-1. **Question Generation** (20% progress): Generates 5 targeted analytical questions based on the user's goal
-2. **Planning** (40% progress): Creates an optimized execution plan using available agents
-3. **Agent Execution** (60% progress): Executes analysis using user's active templates
-4. **Code Synthesis** (80% progress): Combines and optimizes code from all agents
-5. **Code Execution** (85% progress): Runs the synthesized analysis code
-6. **Synthesis** (90% progress): Synthesizes results into coherent insights
-7. **Conclusion** (100% progress): Generates final conclusions and recommendations
-
----
-
-## API Endpoints
-
-### Create Deep Analysis Report
-
-**POST** `/deep_analysis/reports`
-
-Creates a new deep analysis report in the database.
-
-**Request Body:**
-```json
-{
-  "report_uuid": "string",
-  "user_id": 123,
-  "goal": "Analyze customer churn patterns",
-  "status": "completed",
-  "deep_questions": "1. What factors...\n2. How does...",
-  "deep_plan": "{\n  \"@preprocessing\": {\n    \"create\": [...],\n    \"use\": [...],\n    \"instruction\": \"...\"\n  }\n}",
-  "summaries": ["Agent summary 1", "Agent summary 2"],
-  "analysis_code": "import pandas as pd\n# Analysis code...",
-  "plotly_figures": [{"data": [...], "layout": {...}}],
-  "synthesis": ["Synthesis result 1"],
-  "final_conclusion": "## Conclusion\nThe analysis reveals...",
-  "html_report": "<html>...</html>",
-  "report_summary": "Brief summary of findings",
-  "progress_percentage": 100,
-  "duration_seconds": 120,
-  "credits_consumed": 5,
-  "error_message": null,
-  "model_provider": "anthropic",
-  "model_name": "claude-sonnet-4-20250514",
-  "total_tokens_used": 15000,
-  "estimated_cost": 0.25,
-  "steps_completed": ["questions", "planning", "execution", "synthesis", "conclusion"]
-}
-```
-
-**Response:**
-```json
-{
-  "report_id": 1,
-  "report_uuid": "uuid-string",
-  "user_id": 123,
-  "goal": "Analyze customer churn patterns",
-  "status": "completed",
-  "start_time": "2024-01-01T12:00:00Z",
-  "end_time": "2024-01-01T12:02:00Z",
-  "duration_seconds": 120,
-  "report_summary": "Brief summary of findings",
-  "created_at": "2024-01-01T12:02:00Z",
-  "updated_at": "2024-01-01T12:02:00Z"
-}
-```
-
-### Get Deep Analysis Reports
-
-**GET** `/deep_analysis/reports`
-
-Retrieves a list of deep analysis reports with optional filtering.
-
-**Query Parameters:**
-- `user_id` (optional): Filter by user ID
-- `limit` (optional): Number of reports to return (1-100, default: 10)
-- `offset` (optional): Number of reports to skip (default: 0)
-- `status` (optional): Filter by status ("pending", "running", "completed", "failed")
-
-**Response:**
-```json
-[
-  {
-    "report_id": 1,
-    "report_uuid": "uuid-string",
-    "user_id": 123,
-    "goal": "Analyze customer churn patterns",
-    "status": "completed",
-    "start_time": "2024-01-01T12:00:00Z",
-    "end_time": "2024-01-01T12:02:00Z",
-    "duration_seconds": 120,
-    "report_summary": "Brief summary of findings",
-    "created_at": "2024-01-01T12:02:00Z",
-    "updated_at": "2024-01-01T12:02:00Z"
-  }
-]
-```
-
-### Get User Historical Reports
-
-**GET** `/deep_analysis/reports/user_historical`
-
-Retrieves all historical deep analysis reports for a specific user.
-
-**Query Parameters:**
-- `user_id`: User ID (required)
-- `limit` (optional): Number of reports to return (1-100, default: 50)
-
-### Get Report by ID
-
-**GET** `/deep_analysis/reports/{report_id}`
-
-Retrieves a complete deep analysis report by ID.
-
-**Query Parameters:**
-- `user_id` (optional): Ensures report belongs to specified user
-
-**Response:**
-```json
-{
-  "report_id": 1,
-  "report_uuid": "uuid-string",
-  "user_id": 123,
-  "goal": "Analyze customer churn patterns",
-  "status": "completed",
-  "start_time": "2024-01-01T12:00:00Z",
-  "end_time": "2024-01-01T12:02:00Z",
-  "duration_seconds": 120,
-  "deep_questions": "1. What factors contribute to churn?\n2. How does churn vary by segment?",
-  "deep_plan": "{\n  \"@preprocessing\": {...},\n  \"@statistical_analytics\": {...}\n}",
-  "summaries": ["Agent performed data cleaning...", "Statistical analysis revealed..."],
-  "analysis_code": "import pandas as pd\n# Complete analysis code",
-  "plotly_figures": [{"data": [...], "layout": {...}}],
-  "synthesis": ["The analysis shows clear patterns..."],
-  "final_conclusion": "## Conclusion\nCustomer churn is primarily driven by...",
-  "html_report": "<html>...</html>",
-  "report_summary": "Analysis of customer churn patterns reveals...",
-  "progress_percentage": 100,
-  "credits_consumed": 5,
-  "error_message": null,
-  "model_provider": "anthropic",
-  "model_name": "claude-sonnet-4-20250514",
-  "total_tokens_used": 15000,
-  "estimated_cost": 0.25,
-  "steps_completed": ["questions", "planning", "execution", "synthesis", "conclusion"],
-  "created_at": "2024-01-01T12:02:00Z",
-  "updated_at": "2024-01-01T12:02:00Z"
-}
-```
-
-### Get Report by UUID
-
-**GET** `/deep_analysis/reports/uuid/{report_uuid}`
-
-Retrieves a complete deep analysis report by UUID. Same response format as get by ID.
-
-### Delete Report
-
-**DELETE** `/deep_analysis/reports/{report_id}`
-
-Deletes a deep analysis report.
-
-**Query Parameters:**
-- `user_id` (optional): Ensures report belongs to specified user
-
-**Response:**
-```json
-{
-  "message": "Report 1 deleted successfully"
-}
-```
-
-### Update Report Status
-
-**PUT** `/deep_analysis/reports/{report_id}/status`
-
-Updates the status of a deep analysis report.
-
-**Request Body:**
-```json
-{
-  "status": "completed"
-}
-```
-
-**Valid Status Values:**
-- `pending`: Analysis queued but not started
-- `running`: Analysis in progress
-- `completed`: Analysis finished successfully
-- `failed`: Analysis encountered errors
-
-### Get HTML Report
-
-**GET** `/deep_analysis/reports/uuid/{report_uuid}/html`
-
-Retrieves only the HTML report content for a specific analysis.
-
-**Query Parameters:**
-- `user_id` (optional): Ensures report belongs to specified user
-
-**Response:**
-```json
-{
-  "html_report": "<html>...</html>",
-  "filename": "deep_analysis_report_20240101_120200.html"
-}
-```
-
-### Download HTML Report
-
-**POST** `/deep_analysis/download_from_db/{report_uuid}`
-
-Downloads the HTML report as a file attachment.
-
-**Query Parameters:**
-- `user_id` (optional): Ensures report belongs to specified user
-
-**Response:**
-- Content-Type: `text/html; charset=utf-8`
-- Content-Disposition: `attachment; filename="deep_analysis_report_TIMESTAMP.html"`
-
----
-
-## Deep Analysis Module Architecture
-
-### DSPy Signatures
-
-The system uses several DSPy signatures for different analysis phases:
-
-#### 1. `deep_questions`
-Generates 5 targeted analytical questions based on the user's goal and dataset structure.
-
-#### 2. `deep_planner` 
-Creates an optimized execution plan using the user's active templates/agents. The planner:
-- Verifies feasibility using available datasets and agent descriptions
-- Batches similar questions per agent call for efficiency
-- Reuses outputs across questions to minimize agent calls
-- Defines clear variable flow and dependencies between agents
-
-#### 3. `deep_code_synthesizer`
-Combines and optimizes code from multiple agents:
-- Fixes errors and inconsistencies between agent outputs
-- Ensures proper data flow and type handling
-- Converts all visualizations to Plotly format
-- Adds comprehensive error handling and validation
-
-#### 4. `deep_synthesizer`
-Synthesizes analysis results into coherent insights and findings.
-
-#### 5. `final_conclusion`
-Generates final conclusions and strategic recommendations based on all analysis results.
-
-### Streaming Analysis
-
-The `execute_deep_analysis_streaming` method provides real-time progress updates:
-
-```python
-async for update in deep_analysis.execute_deep_analysis_streaming(goal, dataset_info, session_df):
-    if update["step"] == "questions":
-        # Handle questions generation progress
-    elif update["step"] == "planning":
-        # Handle planning progress
-    elif update["step"] == "agent_execution":
-        # Handle agent execution progress
-    # ... handle other steps
-```
-
-### Integration with User Templates
-
-The deep analysis system integrates with user templates in several ways:
-
-1. **Agent Discovery**: Retrieves user's active template preferences from the database
-2. **Dynamic Planning**: The planner uses available agents to create optimal execution plans
-3. **Template Validation**: Ensures all referenced agents exist in the user's active templates
-4. **Fallback Handling**: Uses default agents if user preferences are incomplete
-5. **Performance Optimization**: Respects template limits for efficient execution
-
-### Error Handling
-
-The system includes comprehensive error handling:
-
-- **Code Execution Errors**: Automatically attempts to fix and retry failed code
-- **Template Missing**: Falls back to default agents if user templates are unavailable
-- **Timeout Protection**: Includes timeouts for long-running operations
-- **Memory Management**: Handles large datasets and visualization efficiently
-- **Unicode Handling**: Cleans problematic characters that might cause encoding issues
-
-### Visualization Integration
-
-All visualizations are standardized to Plotly format:
-- Consistent styling and color schemes
-- Interactive features (zoom, pan, hover)
-- Accessibility compliance (colorblind-friendly palettes)
-- Export capabilities for reports
-- Responsive design for different screen sizes
-
----
-
-## Frontend Integration
-
-The deep analysis system includes React components for:
-
-- **DeepAnalysisSidebar**: Main interface for starting and managing analyses
-- **NewAnalysisForm**: Form for initiating new deep analyses
-- **CurrentAnalysisView**: Real-time progress tracking during analysis
-- **HistoryView**: Browse and access historical analysis reports
-- **AnalysisStep**: Individual step progress visualization
-
-The frontend integrates with the streaming API to provide real-time feedback and uses the user's active template configuration for personalized analysis capabilities.
-
-## Credit and Cost Tracking
-
-The system tracks detailed usage metrics:
-- **Credits Consumed**: Number of credits deducted from user account
-- **Token Usage**: Total tokens used across all model calls
-- **Estimated Cost**: Dollar cost estimate based on model pricing
-- **Model Information**: Provider and model name used for analysis
-- **Execution Time**: Duration of analysis for performance monitoring
-
-This information helps users understand resource consumption and optimize their analysis strategies. 

docs/api/routes/feedback.md

@@ -1,153 +0,0 @@
-# Feedback Routes Documentation
-
-This document describes the API endpoints available for managing user feedback on AI-generated messages in the Auto-Analyst backend.
-
-## Base URL
-
-All feedback-related endpoints are prefixed with `/feedback`.
-
-## Endpoints
-
-### Create or Update Message Feedback
-Creates new feedback or updates existing feedback for a specific message.
-
-**Endpoint:** `POST /feedback/message/{message_id}`
-
-**Path Parameters:**
-- `message_id`: ID of the message to provide feedback for
-
-**Request Body:**
-```json
-{
-    "rating": 5,                           // Required: Star rating (1-5)
-    "model_name": "gpt-4o-mini",          // Optional: Model used for the message
-    "model_provider": "openai",           // Optional: Provider of the model
-    "temperature": 0.7,                   // Optional: Temperature setting
-    "max_tokens": 6000                    // Optional: Max tokens setting
-}
-```
-
-**Response:**
-```json
-{
-    "feedback_id": 123,
-    "message_id": 456,
-    "rating": 5,
-    "feedback_comment": null,
-    "model_name": "gpt-4o-mini",
-    "model_provider": "openai",
-    "temperature": 0.7,
-    "max_tokens": 6000,
-    "created_at": "2023-05-01T12:00:00Z",
-    "updated_at": "2023-05-01T12:00:00Z"
-}
-```
-
-**Error Responses:**
-- `404 Not Found`: Message with specified ID not found
-- `500 Internal Server Error`: Failed to create/update feedback
-
-### Get Message Feedback
-Retrieves feedback for a specific message.
-
-**Endpoint:** `GET /feedback/message/{message_id}`
-
-**Path Parameters:**
-- `message_id`: ID of the message to get feedback for
-
-**Response:**
-```json
-{
-    "feedback_id": 123,
-    "message_id": 456,
-    "rating": 5,
-    "feedback_comment": null,
-    "model_name": "gpt-4o-mini",
-    "model_provider": "openai",
-    "temperature": 0.7,
-    "max_tokens": 6000,
-    "created_at": "2023-05-01T12:00:00Z",
-    "updated_at": "2023-05-01T12:00:00Z"
-}
-```
-
-**Error Responses:**
-- `404 Not Found`: No feedback found for the specified message
-- `500 Internal Server Error`: Failed to retrieve feedback
-
-### Get Chat Feedback
-Retrieves all feedback for messages in a specific chat.
-
-**Endpoint:** `GET /feedback/chat/{chat_id}`
-
-**Path Parameters:**
-- `chat_id`: ID of the chat to get feedback for
-
-**Response:**
-```json
-[
-    {
-        "feedback_id": 123,
-        "message_id": 456,
-        "rating": 5,
-        "feedback_comment": null,
-        "model_name": "gpt-4o-mini",
-        "model_provider": "openai",
-        "temperature": 0.7,
-        "max_tokens": 6000,
-        "created_at": "2023-05-01T12:00:00Z",
-        "updated_at": "2023-05-01T12:00:00Z"
-    }
-]
-```
-
-**Note:** Returns an empty array if no feedback exists for the chat.
-
-**Error Responses:**
-- `500 Internal Server Error`: Failed to retrieve chat feedback
-
-## Feedback Features
-
-### Rating System
-- **Scale:** 1-5 star rating system
-- **Required:** Rating is the only required field for feedback
-- **Purpose:** Allows users to rate the quality of AI responses
-
-### Model Context Tracking
-The system optionally tracks:
-- **Model Name:** The specific AI model used (e.g., "gpt-4o-mini")
-- **Model Provider:** The provider of the model (e.g., "openai", "anthropic")
-- **Temperature:** The creativity/randomness setting used
-- **Max Tokens:** The maximum response length setting
-
-### Update Behavior
-- **Upsert Operation:** The POST endpoint either creates new feedback or updates existing feedback
-- **Partial Updates:** When updating, only provided fields are modified
-- **Timestamp Tracking:** Both creation and update timestamps are maintained
-
-## Data Management
-
-### Database Operations
-- **Atomic Operations:** Feedback creation/updates are handled in database transactions
-- **Referential Integrity:** Feedback is linked to specific messages via foreign keys
-- **Soft Handling:** Missing optional fields are handled gracefully
-
-### Error Handling
-- **Comprehensive Logging:** All operations are logged for debugging
-- **User-Friendly Messages:** Error responses provide clear information
-- **Transaction Safety:** Failed operations are rolled back to maintain data consistency
-
-## Usage Patterns
-
-### Typical Workflow
-1. User receives an AI-generated message
-2. User provides rating (1-5 stars) via the frontend
-3. Frontend calls `POST /feedback/message/{message_id}` with rating and model context
-4. System stores or updates the feedback
-5. Feedback can be retrieved later for analytics or user review
-
-### Analytics Integration
-Feedback data is used by the analytics system to:
-- Track model performance across different configurations
-- Identify patterns in user satisfaction
-- Generate insights for model optimization 

docs/api/routes/session.md

@@ -1,273 +0,0 @@
-# **Auto-Analyst API Documentation**
-
-The core application routes are designed to manage the data and AI analysis capabilities of the Auto-Analyst application.
-
-## **1. Core Application Routes**
-### **Data Management**
-
-#### **POST /upload_dataframe**  
-Uploads a CSV dataset for analysis.  
-**Request:**  
-- `file`: CSV file  
-- `name`: Dataset name  
-- `description`: Dataset description  
-**Headers:**
-- `X-Force-Refresh`: "true" (optional) - Forces session reset before upload
-**Response:**  
-```json
-{ "message": "Dataframe uploaded successfully", "session_id": "abc123" }
-```
-
-#### **POST /upload_excel**  
-Uploads an Excel file with a specific sheet for analysis.  
-**Request:**  
-- `file`: Excel file  
-- `name`: Dataset name  
-- `description`: Dataset description  
-- `sheet_name`: Name of the Excel sheet to use
-**Headers:**
-- `X-Force-Refresh`: "true" (optional) - Forces session reset before upload
-**Response:**  
-```json
-{ "message": "Excel file processed successfully", "session_id": "abc123", "sheet": "Sheet1" }
-```
-
-#### **POST /api/excel-sheets**  
-Gets the list of sheet names from an Excel file.  
-**Request:**  
-- `file`: Excel file  
-**Response:**  
-```json
-{ "sheets": ["Sheet1", "Sheet2", "Data"] }
-```
-
-#### **GET /api/default-dataset**  
-Gets the default dataset.  
-**Response:**  
-```json
-{
-  "headers": ["column1", "column2", ...],
-  "rows": [[val1, val2, ...], ...],
-  "name": "Housing Dataset",
-  "description": "A comprehensive dataset containing housing information..."
-}
-```
-
-#### **POST /reset-session**  
-Resets session to default dataset.  
-**Request Body:**  
-```json
-{
-  "name": "optional name",
-  "description": "optional description",
-  "preserveModelSettings": false
-}
-```
-**Response:**  
-```json
-{
-  "message": "Session reset to default dataset",
-  "session_id": "abc123",
-  "dataset": "Housing.csv"
-}
-```
-
-#### **GET /api/preview-csv** / **POST /api/preview-csv**  
-Preview the current dataset in the session.
-**Response:**  
-```json
-{
-  "headers": ["column1", "column2", ...],
-  "rows": [[val1, val2, ...], ...],
-  "name": "Dataset Name",
-  "description": "Dataset description..."
-}
-```
-
----
-
-### **2. AI Analysis**
-
-#### **POST /chat/{agent_name}**  
-Processes a query using a specific AI agent.  
-**Path Parameters:** `agent_name`  
-**Request Body:**  
-```json
-{ "query": "Analyze the relationship between price and size" }
-```
-**Query Parameters:** `user_id` (optional), `chat_id` (optional)  
-**Response:**  
-```json
-{
-  "agent_name": "data_viz_agent",
-  "query": "Analyze the relationship between price and size",
-  "response": "# Analysis\n\nThere appears to be a strong positive correlation...",
-  "session_id": "abc123"
-}
-```
-
-#### **POST /chat**  
-Processes a query using multiple AI agents with streaming responses.  
-**Request Body:**  
-```json
-{ "query": "Analyze the housing data" }
-```
-**Query Parameters:** `user_id` (optional), `chat_id` (optional)  
-**Response:** *Streaming JSON objects:*  
-```json
-{"agent": "data_viz_agent", "content": "# Visualization\n\n...", "status": "success"}
-{"agent": "statistical_analytics_agent", "content": "# Statistical Analysis\n\n...", "status": "success"}
-```
-
-#### **POST /chat_history_name**  
-Generates a name for a chat based on the query.  
-**Request Body:**  
-```json
-{ "query": "Analyze sales data for Q4" }
-```
-**Response:**  
-```json
-{ "name": "Chat about sales data analysis" }
-```
-
-#### **GET /agents**  
-Lists available AI agents.  
-**Response:**  
-```json
-{
-  "available_agents": ["data_viz_agent", "sk_learn_agent", "statistical_analytics_agent", "preprocessing_agent"],
-  "standard_agents": ["preprocessing_agent", "statistical_analytics_agent", "sk_learn_agent", "data_viz_agent"],
-  "template_agents": ["custom_template_1", "custom_template_2"],
-  "custom_agents": []
-}
-```
-
----
-
-### **3. Deep Analysis**
-
-#### **POST /deep_analysis_streaming**  
-Performs comprehensive deep analysis with real-time streaming updates.  
-**Request Body:**  
-```json
-{ "goal": "Perform comprehensive analysis of the sales data" }
-```
-**Query Parameters:** `user_id` (optional), `chat_id` (optional)  
-**Response:** *Streaming JSON objects with progress updates*
-
-#### **POST /deep_analysis/download_report**  
-Downloads an HTML report from deep analysis results.  
-**Request Body:**  
-```json
-{
-  "analysis_data": { ... },
-  "report_uuid": "optional-uuid"
-}
-```
-**Response:** HTML file download
-
----
-
-### **4. Model Settings**
-
-#### **GET /api/model-settings**  
-Fetches current model settings.  
-**Response:**  
-```json
-{
-  "provider": "openai",
-  "model": "gpt-4o-mini",
-  "hasCustomKey": true,
-  "temperature": 1.0,
-  "maxTokens": 6000
-}
-```
-
-#### **POST /settings/model**  
-Updates model settings.  
-**Request Body:**  
-```json
-{
-  "provider": "openai",
-  "model": "gpt-4",
-  "api_key": "sk-...",
-  "temperature": 0.7,
-  "max_tokens": 8000
-}
-```
-**Response:**  
-```json
-{ "message": "Model settings updated successfully" }
-```
-
----
-
-### **5. Session Management**
-
-#### **GET /api/session-info**  
-Gets information about the current session.  
-**Response:**  
-```json
-{
-  "session_id": "abc123",
-  "dataset_name": "Housing Dataset",
-  "dataset_description": "...",
-  "model_config": { ... }
-}
-```
-
-#### **POST /set-message-info**  
-Associates message tracking information with the session.  
-**Request Body:**  
-```json
-{
-  "chat_id": 123,
-  "message_id": 456,
-  "user_id": 789
-}
-```
-
-#### **POST /create-dataset-description**  
-Creates an AI-generated description for a dataset.  
-**Request Body:**  
-```json
-{
-  "df_preview": "column1,column2\nvalue1,value2\n...",
-  "name": "Dataset Name"
-}
-```
-
----
-
-### **6. System Endpoints**
-
-#### **GET /**  
-Returns API welcome information and feature list.
-
-#### **GET /health**  
-Health check endpoint.  
-**Response:**  
-```json
-{ "message": "API is healthy and running" }
-```
-
----
-
----
-
-### **7. Authentication & Session Management**
-- **Session ID Sources:**  
-  - Query parameter: `session_id`  
-  - Header: `X-Session-ID`  
-  - Auto-generated if not provided  
-- **Session State Includes:**  
-  - Current dataset  
-  - AI system instance  
-  - Model configuration  
-  - User and chat associations
-
-### **9. Error Handling**
-- Comprehensive error handling with appropriate HTTP status codes
-- Detailed error messages for debugging
-- Fallback encoding support for CSV files (UTF-8, unicode_escape, ISO-8859-1)
-- Session state preservation during errors

docs/api/routes/templates.md

@@ -1,363 +0,0 @@
-# Templates and Agent Loading Documentation
-
-This document describes how the Auto-Analyst template system works, including agent loading, user preferences, and template management.
-
-## Overview
-
-The Auto-Analyst system uses a flexible template-based approach for managing AI agents. Templates define specialized agents with specific capabilities, and users can customize which agents are available for their analysis workflows.
-
-## Template System Architecture
-
-### Template Types
-
-Templates come in different **variant types** that determine how they can be used:
-
-- **`individual`**: Templates available for single-agent queries (e.g., `@preprocessing_agent`)
-- **`planner`**: Templates available for multi-agent planning workflows
-- **`both`**: Templates available in both individual and planner contexts
-
-### Default Agents
-
-The system includes four core default agents that are **enabled by default** for all users:
-
-**For Individual Use:**
-- `preprocessing_agent`: Data cleaning and preprocessing
-- `statistical_analytics_agent`: Statistical analysis and insights  
-- `sk_learn_agent`: Machine learning with scikit-learn
-- `data_viz_agent`: Data visualization with Plotly
-
-**For Planner Use:**
-- `planner_preprocessing_agent`: Planning version of preprocessing agent
-- `planner_statistical_analytics_agent`: Planning version of statistical agent
-- `planner_sk_learn_agent`: Planning version of ML agent
-- `planner_data_viz_agent`: Planning version of visualization agent
-
-## Template Management Endpoints
-
-### Get All Templates
-
-**Endpoint:** `GET /templates/`
-
-**Query Parameters:**
-- `variant_type`: Filter by `"individual"`, `"planner"`, or `"all"` (default: `"all"`)
-
-**Response:**
-```json
-[
-  {
-    "template_id": 1,
-    "template_name": "preprocessing_agent",
-    "display_name": "Data Preprocessing Agent",
-    "description": "Handles data cleaning, missing values, and preprocessing tasks",
-    "prompt_template": "You are a data preprocessing specialist...",
-    "template_category": "Data Processing",
-    "icon_url": "/icons/templates/preprocessing_agent.svg",
-    "is_premium_only": false,
-    "is_active": true,
-    "usage_count": 12,
-    "created_at": "2023-05-01T12:00:00Z",
-    "updated_at": "2023-05-01T12:00:00Z"
-  }
-]
-```
-
-### Get Templates by Category
-
-**Endpoint:** `GET /templates/categories`
-
-**Query Parameters:**
-- `variant_type`: Filter by `"individual"`, `"planner"`, or `"all"` (default: `"individual"`)
-
-**Response:**
-```json
-[
-  {
-    "category": "Data Processing", 
-    "templates": [
-      {
-        "agent_id": 1,
-        "agent_name": "preprocessing_agent",
-        "display_name": "Data Preprocessing Agent",
-        "description": "Handles data cleaning and preprocessing",
-        "icon_url": "/icons/templates/preprocessing_agent.svg",
-        "usage_count": 1234
-      }
-    ]
-  }
-]
-```
-
-### Get Template by ID
-
-**Endpoint:** `GET /templates/template/{template_id}`
-
-**Response:**
-```json
-{
-  "template_id": 1,
-  "template_name": "preprocessing_agent",
-  "display_name": "Data Preprocessing Agent",
-  "description": "Handles data cleaning, missing values, and preprocessing tasks",
-  "prompt_template": "You are a data preprocessing specialist...",
-  "template_category": "Data Processing",
-  "icon_url": "/icons/templates/preprocessing_agent.svg",
-  "is_premium_only": false,
-  "is_active": true,
-  "usage_count": 1234,
-  "created_at": "2023-05-01T12:00:00Z",
-  "updated_at": "2023-05-01T12:00:00Z"
-}
-```
-
-### Get Template Categories List
-
-**Endpoint:** `GET /templates/categories/list`
-
-**Response:**
-```json
-{
-  "categories": [
-    "Data Processing",
-    "Machine Learning", 
-    "Visualization",
-    "Statistics"
-  ]
-}
-```
-
-### Get Templates by Specific Category
-
-**Endpoint:** `GET /templates/category/{category}`
-
-**Path Parameters:**
-- `category`: Name of the category to filter by
-
-**Response:**
-```json
-[
-  {
-    "template_id": 1,
-    "template_name": "preprocessing_agent",
-    "display_name": "Data Preprocessing Agent",
-    "description": "Handles data cleaning and preprocessing",
-    "template_category": "Data Processing",
-    "icon_url": "/icons/templates/preprocessing_agent.svg",
-    "usage_count": 1234
-  }
-]
-```
-
-## User Template Preferences
-
-### How Agent Loading Works for Users
-
-1. **Default Behavior**: New users automatically have the 4 core default agents enabled
-2. **Custom Preferences**: Users can enable/disable additional templates through preferences
-3. **Variant-Specific**: Individual and planner variants are managed separately
-4. **Usage Tracking**: System tracks which templates users actually use
-
-### Get User Template Preferences
-
-**Endpoint:** `GET /templates/user/{user_id}`
-
-**Query Parameters:**
-- `variant_type`: Filter by `"individual"`, `"planner"`, or `"all"` (default: `"planner"`)
-
-**Response:**
-```json
-[
-  {
-    "template_id": 1,
-    "template_name": "preprocessing_agent", 
-    "display_name": "Data Preprocessing Agent",
-    "description": "Handles data cleaning and preprocessing",
-    "template_category": "Data Processing",
-    "icon_url": "/icons/templates/preprocessing_agent.svg",
-    "is_premium_only": false,
-    "is_active": true,
-    "is_enabled": true,
-    "usage_count": 15,
-    "last_used_at": "2023-05-01T12:00:00Z",
-    "created_at": "2023-04-01T12:00:00Z",
-    "updated_at": "2023-05-01T12:00:00Z"
-  }
-]
-```
-
-### Get Only Enabled Templates
-
-**Endpoint:** `GET /templates/user/{user_id}/enabled`
-
-Returns only templates that are currently enabled for the user.
-
-### Get Enabled Templates for Planner
-
-**Endpoint:** `GET /templates/user/{user_id}/enabled/planner`
-
-Returns enabled planner templates with the following restrictions:
-- **Maximum 10 templates** for planner use
-- **Sorted by usage** (most used first)
-- **Only planner variants** (`planner` or `both` types)
-
-## Template Preference Management
-
-### Toggle Single Template
-
-**Endpoint:** `POST /templates/user/{user_id}/template/{template_id}/toggle`
-
-**Request Body:**
-```json
-{
-  "is_enabled": true
-}
-```
-
-**Restrictions:**
-- Cannot disable all templates (at least 1 must remain enabled)
-- Cannot enable more than 10 templates for planner use
-
-### Bulk Toggle Templates
-
-**Endpoint:** `POST /templates/user/{user_id}/bulk-toggle`
-
-**Request Body:**
-```json
-{
-  "preferences": [
-    {
-      "template_id": 1,
-      "is_enabled": true
-    },
-    {
-      "template_id": 2, 
-      "is_enabled": false
-    }
-  ]
-}
-```
-
-**Response:**
-```json
-{
-  "results": [
-    {
-      "template_id": 1,
-      "success": true,
-      "message": "Template enabled successfully",
-      "is_enabled": true
-    }
-  ]
-}
-```
-
-## Template Categories and Icons
-
-### Available Categories
-
-Templates are organized into categories such as:
-- **Data Processing**: Preprocessing, cleaning, feature engineering
-- **Machine Learning**: Various ML frameworks and algorithms  
-- **Visualization**: Plotting and chart generation
-- **Statistics**: Statistical analysis and modeling
-- **Custom**: User or organization-specific templates
-
-### Icon System
-
-Templates include visual icons stored in `/public/icons/templates/`:
-
-**Core Agent Icons:**
-- `preprocessing_agent.svg`: Data preprocessing 
-- `sk_learn_agent.svg`: Machine learning
-- `matplotlib_agent.png`: Plotting with matplotlib
-- `polars_agent.svg`: Data manipulation with Polars
-
-**Library-Specific Icons:**
-- `numpy.svg`, `scipy.png`: Scientific computing
-- `plotly.svg`, `seaborn.svg`: Advanced visualization
-- `lightgbm.png`, `xgboost.png`: Gradient boosting
-- `pymc.png`, `statsmodel.svg`: Statistical modeling
-
-**Special Purpose Icons:**
-- `data-cleaning.png`: Data cleaning workflows
-- `feature-engineering.png`: Feature engineering tasks
-
-## Agent Loading Process
-
-### For Individual Queries
-
-When a user makes a query like `@preprocessing_agent analyze my data`:
-
-1. **Check User Preferences**: System looks up user's enabled individual templates
-2. **Apply Defaults**: If no preference exists, default agents are enabled
-3. **Load Agent**: System loads the specific agent template and executes the query
-4. **Track Usage**: Usage count is incremented for analytics
-
-### For Planner Workflows  
-
-When a user makes a general query that triggers the planner:
-
-1. **Get Enabled Planner Templates**: System queries user's enabled planner variants
-2. **Apply 10-Template Limit**: Maximum 10 templates for performance
-3. **Sort by Usage**: Most-used templates get priority
-4. **Create Plan**: Planner selects appropriate agents for the analysis
-5. **Execute Workflow**: Selected agents execute in sequence
-6. **Update Usage**: Usage statistics updated for selected agents
-
-### Default Agent Behavior
-
-```python
-# Default agents enabled for new users
-individual_defaults = [
-    "preprocessing_agent",
-    "statistical_analytics_agent", 
-    "sk_learn_agent",
-    "data_viz_agent"
-]
-
-planner_defaults = [
-    "planner_preprocessing_agent",
-    "planner_statistical_analytics_agent",
-    "planner_sk_learn_agent", 
-    "planner_data_viz_agent"
-]
-```
-
-## Usage Analytics
-
-### Global Usage Tracking
-
-The system tracks global usage statistics across all users:
-- **Total usage count** per template
-- **User-specific usage** for personalization
-- **Last used timestamps** for sorting
-
-### Usage-Based Features
-
-- **Template Recommendations**: Popular templates shown first
-- **Personalized Ordering**: User's most-used templates prioritized
-- **Analytics Dashboard**: Usage patterns for administrators
-
-## Template Restrictions
-
-### User Limitations
-
-- **Minimum 1 Agent**: Cannot disable all templates
-- **Maximum 10 for Planner**: Performance optimization
-- **Premium Templates**: Some templates require premium access
-
-### System Limitations
-
-- **Active Templates Only**: Inactive templates not available
-- **Variant Compatibility**: Individual/planner variants managed separately
-- **Category Organization**: Templates must belong to valid categories
-
-## Integration with Deep Analysis
-
-The deep analysis system uses the template preference system:
-
-1. **Load User Preferences**: Gets enabled planner templates for user
-2. **Create Agent Pool**: Instantiates agents from enabled templates  
-3. **Execute Analysis**: Uses available agents for comprehensive analysis
-4. **Fallback Behavior**: Uses default agents if no preferences found
-
-This ensures users get personalized deep analysis based on their template preferences while maintaining system performance through the 10-template limit. 

docs/architecture/architecture.md

@@ -1,427 +0,0 @@
-# Auto-Analyst Backend System Architecture
-
-## Overview
-
-Auto-Analyst is a sophisticated multi-agent AI platform designed for comprehensive data analysis. The backend system orchestrates specialized AI agents, manages user sessions, and provides a robust API for data processing and analysis workflows.
-
-## 🏗️ High-Level Architecture
-
-```
-┌─────────────────┐    ┌──────────────────┐    ┌─────────────────┐
-│   Frontend      │    │     Backend      │    │    Database     │
-│   (Next.js)     │◄──►│    (FastAPI)     │◄──►│ (PostgreSQL/    │
-│                 │    │                  │    │  SQLite)        │
-└─────────────────┘    └──────────────────┘    └─────────────────┘
-                              │
-                              ▼
-                       ┌──────────────────┐
-                       │   AI Models      │
-                       │   (DSPy/LLMs)    │
-                       └──────────────────┘
-                              │
-                              ▼
-                       ┌──────────────────┐
-                       │ Agent System     │
-                       │ [Processing]     │
-                       │ [Analytics]      │
-                       │ [Visualization]  │
-                       └──────────────────┘
-```
-
-## 🎯 Core Components
-
-### 1. Application Layer (`app.py`)
-
-**FastAPI Application Server**
-- **Role**: Main HTTP server and request router
-- **Responsibilities**:
-  - Request/response handling
-  - Session-based authentication
-  - Route registration and middleware
-  - Error handling and logging
-  - Static file serving
-  - CORS configuration
-
-**Key Features**:
-- Async/await support for high concurrency
-- Automatic API documentation generation
-- Request validation with Pydantic
-- Session management for user tracking
-
-### 2. Agent System (`src/agents/`)
-
-**Multi-Agent Orchestra**
-- **Core Agents**: Specialized AI agents for different analysis tasks
-- **Deep Analysis**: Advanced multi-agent coordination system
-- **Template System**: User-customizable agent configurations
-
-#### Agent Types
-
-1. **Individual Agents** (`agents.py`):
-   ```python
-   - preprocessing_agent         # Data cleaning and preparation
-   - statistical_analytics_agent # Statistical analysis
-   - sk_learn_agent             # Machine learning with scikit-learn
-   - data_viz_agent             # Data visualization
-   - basic_qa_agent             # General Q&A
-   ```
-
-2. **Planner Agents** (Multi-agent coordination):
-   ```python
-   - planner_preprocessing_agent
-   - planner_statistical_analytics_agent
-   - planner_sk_learn_agent
-   - planner_data_viz_agent
-   ```
-
-3. **Deep Analysis System** (`deep_agents.py`):
-   ```python
-   - deep_questions         # Question generation
-   - deep_planner          # Execution planning
-   - deep_code_synthesizer # Code combination
-   - deep_synthesizer      # Result synthesis
-   - final_conclusion      # Report generation
-   ```
-
-#### Agent Architecture Pattern
-
-```python
-class AgentSignature(dspy.Signature):
-    """Agent description and purpose"""
-    goal = dspy.InputField(desc="Analysis objective")
-    dataset = dspy.InputField(desc="Dataset information")
-    plan_instructions = dspy.InputField(desc="Execution plan")
-    
-    summary = dspy.OutputField(desc="Analysis summary")
-    code = dspy.OutputField(desc="Generated code")
-```
-
-### 3. Database Layer (`src/db/`)
-
-**Data Persistence and Management**
-
-#### Database Models (`schemas/models.py`):
-
-```python
-# Core Models
-User              # User accounts and authentication
-Chat              # Conversation sessions
-Message           # Individual messages in chats
-ModelUsage        # AI model usage tracking
-
-# Template System
-AgentTemplate     # Agent definitions and configurations
-UserTemplatePreference  # User's enabled/disabled agents
-
-# Deep Analysis
-DeepAnalysisReport     # Analysis reports and results
-
-# Analytics
-CodeExecution     # Code execution tracking
-UserAnalytics     # User behavior analytics
-```
-
-#### Database Architecture:
-
-```
-Users (1) ──────── (Many) Chats
-  │                        │
-  │                        ▼
-  └─── (Many) ModelUsage ──┘
-  │
-  └─── (Many) UserTemplatePreference
-               │
-               ▼
-         AgentTemplate
-```
-
-### 4. Route Handlers (`src/routes/`)
-
-**RESTful API Endpoints**
-
-| Module | Purpose | Key Endpoints |
-|--------|---------|---------------|
-| `session_routes.py` | Core functionality | `/upload_excel`, `/session_info` |
-| `chat_routes.py` | Chat management | `/chats`, `/messages`, `/delete_chat` |
-| `code_routes.py` | Code operations | `/execute_code`, `/get_latest_code` |
-| `templates_routes.py` | Agent templates | `/templates`, `/user/{id}/enabled` |
-| `deep_analysis_routes.py` | Deep analysis | `/reports`, `/download_from_db` |
-| `analytics_routes.py` | System analytics | `/usage`, `/feedback`, `/costs` |
-| `feedback_routes.py` | User feedback | `/feedback`, `/message/{id}/feedback` |
-
-NOTE: Make sure to add a router prefix when calling these endpoints, such as to get dashboard, you'll use `http://localhost:8000/templates/dashboard`
-
-### 5. Business Logic Layer (`src/managers/`)
-
-**Service Layer for Complex Operations**
-
-#### Manager Components:
-
-1. **`chat_manager.py`**:
-   ```python
-   - Session management
-   - Message handling
-   - Context preservation
-   - Agent orchestration
-   ```
-
-2. **`ai_manager.py`**:
-   ```python
-   - Model selection and routing
-   - Token tracking and cost calculation
-   - Error handling and retries
-   - Response formatting
-   ```
-
-3. **`session_manager.py`**:
-   ```python
-   - Session lifecycle management
-   - Data sharing between agents
-   - Memory management
-   - Cleanup operations
-   ```
-
-### 6. Utility Layer (`src/utils/`)
-
-**Shared Services and Helpers**
-
-- **`logger.py`**: Centralized logging system
-- **`generate_report.py`**: HTML report generation
-- **`model_registry.py`**: AI model configuration
-
-## 🔄 Data Flow Architecture
-
-### 1. Request Processing Flow
-
-```
-HTTP Request → FastAPI Router → Route Handler → Manager/Business Logic → 
-Database/Agent System → AI Model → Response Processing → JSON Response
-```
-
-### 2. Agent Execution Flow
-
-```
-User Query → Session Creation → Template Selection → Agent Loading → 
-Code Generation → Code Execution → Result Processing → Response Formatting
-```
-
-### 3. Deep Analysis Flow
-
-```
-Analysis Goal → Question Generation → Planning Phase → Agent Coordination → 
-Code Synthesis → Execution → Result Synthesis → Final Report Generation
-```
-
-### 4. Template System Flow
-
-```
-User Preferences → Template Loading → Agent Registration → 
-Capability Mapping → Execution Routing → Usage Tracking
-```
-
-## 🎨 Design Patterns
-
-### 1. **Module Pattern**
-- Clear separation of concerns
-- Each module has specific responsibilities
-- Minimal dependencies between modules
-
-### 2. **Repository Pattern**
-- Database access abstracted through SQLAlchemy
-- Session management centralized
-- Clean separation of data and business logic
-
-### 3. **Strategy Pattern**
-- Multiple AI models supported through unified interface
-- Agent selection based on user preferences
-- Dynamic template loading
-
-### 4. **Observer Pattern**
-- Usage tracking and analytics
-- Event-driven model updates
-- Real-time progress notifications
-
-## 🔧 Configuration Management
-
-### Environment Configuration
-
-```python
-# Database
-DATABASE_URL: str           # Database connection string
-POSTGRES_PASSWORD: str      # PostgreSQL password (optional)
-
-# AI Models
-ANTHROPIC_API_KEY: str      # Claude API key
-OPENAI_API_KEY: str         # OpenAI API key
-
-# Authentication
-ADMIN_API_KEY: str          # Admin operations key (optional)
-
-# Deployment
-PORT: int = 8000            # Server port
-DEBUG: bool = False         # Debug mode
-```
-
-### Agent Configuration (`agents_config.json`)
-
-```json
-{
-  "default_agents": [
-    {
-      "template_name": "preprocessing_agent",
-      "description": "Data cleaning and preparation",
-      "variant_type": "both",
-      "is_premium": false,
-      "usage_count": 0,
-      "icon_url": "preprocessing.svg"
-    }
-  ],
-  "premium_templates": [...],
-  "remove": [...]
-}
-```
-
-## 🔒 Security Architecture
-
-### Authentication & Authorization
-
-1. **Session-based Authentication**:
-   - Session IDs for user identification
-   - Optional API key authentication for admin endpoints
-
-2. **Input Validation**:
-   - Pydantic models for request validation
-   - SQL injection prevention through SQLAlchemy
-   - File upload restrictions and validation
-
-3. **Resource Protection**:
-   - User-specific data isolation
-   - Usage tracking and monitoring
-   - Rate limiting considerations
-
-### Data Security
-
-1. **Database Security**:
-   - Encrypted connections for PostgreSQL
-   - Parameterized queries
-   - Regular backup procedures
-
-2. **Code Execution Security**:
-   - Sandboxed code execution environment
-   - Limited library imports
-   - Timeout protection
-
-## 📊 Performance Architecture
-
-### Scalability Features
-
-1. **Async Architecture**:
-   - Non-blocking I/O operations
-   - Concurrent agent execution
-   - Streaming responses for long operations
-
-2. **Database Optimization**:
-   - Connection pooling
-   - Query optimization
-   - Indexed frequently accessed columns
-
-3. **Caching Strategy**:
-   - In-memory caching for templates
-   - Result caching for expensive operations
-   - Session data management
-
-### Performance Monitoring
-
-1. **Usage Analytics**:
-   - Request/response time tracking
-   - Token usage monitoring
-   - Error rate analysis
-
-2. **Resource Monitoring**:
-   - Database query performance
-   - Memory usage tracking
-   - Agent execution time analysis
-
-## 🚀 Deployment Architecture
-
-### Development Environment
-
-```
-Local Development → SQLite Database → File-based Logging → 
-Direct Model API Calls → Hot Reloading
-```
-
-### Production Environment
-
-```
-Load Balancer → Multiple FastAPI Instances → PostgreSQL Database → 
-Centralized Logging → Monitoring & Alerting
-```
-
-### Container Architecture
-
-```dockerfile
-# Multi-stage build for optimization
-FROM python:3.11-slim as base
-# Dependencies and application setup
-# Health checks and graceful shutdown
-# Environment-specific configurations
-```
-
-## 🔄 Integration Patterns
-
-### External Service Integration
-
-1. **AI Model Providers**:
-   - Anthropic (Claude)
-   - OpenAI (GPT models)
-   - Unified interface through DSPy
-
-2. **Database Systems**:
-   - PostgreSQL (production)
-   - SQLite (development)
-   - Migration support through Alembic
-
-### Frontend Integration
-
-1. **REST API**:
-   - Standard HTTP endpoints
-   - JSON request/response format
-   - Session-based communication
-
-2. **Data Exchange**:
-   - File upload capabilities
-   - Real-time analysis results
-   - Report generation and download
-
-### Third-Party Integration
-
-1. **Python Data Science Stack (For Agentic Use only)**:
-   - Pandas for data manipulation
-   - NumPy for numerical computing
-   - Scikit-learn for machine learning
-   - Plotly for visualization
-   - Statsmodels for statistical analysis
-
-2. **Development Tools**:
-   - Alembic for database migrations
-   - SQLAlchemy for ORM
-   - FastAPI for web framework
-   - Pydantic for data validation
-
-## 📝 Documentation Architecture
-
-### API Documentation
-
-1. **Auto-generated Docs**: Available at `/docs` endpoint
-2. **Schema Definitions**: Pydantic models with descriptions
-3. **Endpoint Documentation**: Detailed parameter and response docs
-
-### Code Documentation
-
-1. **Inline Documentation**: Comprehensive docstrings
-2. **Architecture Guides**: High-level system design documentation
-3. **Getting Started**: Developer onboarding documentation
-4. **Troubleshooting**: Common issues and solutions
-
-This architecture provides a robust, scalable foundation for multi-agent AI analysis while maintaining clean separation of concerns and supporting both development and production deployment scenarios. 

docs/development/development_workflow.md

@@ -1,506 +0,0 @@
-# Auto-Analyst Backend Development Workflow
-
-## 🎯 Development Philosophy
-
-The Auto-Analyst backend follows modern Python development practices with emphasis on:
-- **Modularity**: Clear separation of concerns across components
-- **Async-First**: Non-blocking operations for scalability
-- **Type Safety**: Comprehensive type hints and validation
-- **Documentation**: Self-documenting code and comprehensive docs
-- **Testing**: Robust testing at multiple levels
-- **Performance**: Optimized for real-world usage patterns
-
-## 🏗️ Code Organization Principles
-
-### 1. **Directory Structure Standards**
-
-```
-src/
-├── agents/           # AI agent implementations
-│   ├── agents.py    # Core agent definitions
-│   ├── deep_agents.py # Deep analysis system
-│   └── retrievers/  # Information retrieval components
-├── db/              # Database layer
-│   ├── init_db.py   # Database initialization
-│   └── schemas/     # SQLAlchemy models
-├── managers/        # Business logic layer
-│   ├── chat_manager.py    # Chat operations
-│   ├── ai_manager.py      # AI model management
-│   └── session_manager.py # Session lifecycle
-├── routes/          # FastAPI route handlers
-│   ├── session_routes.py     # Core functionality
-│   ├── chat_routes.py     # Chat endpoints
-│   └── [feature]_routes.py # Feature-specific routes
-├── utils/           # Shared utilities
-│   ├── logger.py    # Centralized logging
-│   └── helpers.py   # Common functions
-└── schemas/         # Pydantic models
-    ├── chat_schemas.py    # Chat data models
-    └── [feature]_schemas.py # Feature schemas
-```
-
-### 2. **Import Organization**
-
-```python
-# Standard library imports
-import asyncio
-import json
-from datetime import datetime
-from typing import List, Optional, Dict, Any
-
-# Third-party imports
-import dspy
-import pandas as pd
-from fastapi import APIRouter, Depends, HTTPException
-from pydantic import BaseModel
-from sqlalchemy.orm import Session
-
-# Local imports
-from src.db.init_db import session_factory
-from src.db.schemas.models import User, Chat
-from src.utils.logger import Logger
-from src.managers.chat_manager import ChatManager
-```
-
-## 🛠️ Development Patterns
-
-### 1. **Agent Development Pattern**
-
-```python
-# 1. Define DSPy Signature
-class new_analysis_agent(dspy.Signature):
-    """
-    Comprehensive docstring explaining:
-    - Agent purpose and capabilities
-    - Input requirements and formats
-    - Expected output format
-    - Usage examples
-    """
-    goal = dspy.InputField(desc="Clear description of analysis objective")
-    dataset = dspy.InputField(desc="Dataset structure and content description")
-    plan_instructions = dspy.InputField(desc="Execution plan from planner")
-    
-    summary = dspy.OutputField(desc="Natural language summary of analysis")
-    code = dspy.OutputField(desc="Executable Python code for analysis")
-
-# 2. Add to Agent Configuration
-# In agents_config.json:
-{
-  "template_name": "new_analysis_agent",
-  "description": "Performs specialized analysis on datasets",
-  "variant_type": "both",  # individual, planner, or both
-  "is_premium": false, # Will be active by default
-  "usage_count": 0,
-  "icon_url": "analysis.svg"
-}
-
-# 3. Register in Agent System
-# In agents.py, add to the appropriate loading functions
-```
-
-### 2. **Route Development Pattern**
-
-```python
-# 1. Create route file: src/routes/feature_routes.py
-from fastapi import APIRouter, Depends, HTTPException, Query
-from pydantic import BaseModel
-from typing import List, Optional
-from src.db.init_db import session_factory
-from src.db.schemas.models import FeatureModel
-from src.utils.logger import Logger
-
-logger = Logger("feature_routes", see_time=True, console_log=False)
-router = APIRouter(prefix="/feature", tags=["feature"])
-
-# 2. Define Pydantic schemas
-class FeatureCreate(BaseModel):
-    name: str
-    description: Optional[str] = None
-    
-class FeatureResponse(BaseModel):
-    id: int
-    name: str
-    description: Optional[str]
-    created_at: datetime
-
-# 3. Implement endpoints with proper error handling
-@router.post("/", response_model=FeatureResponse)
-async def create_feature(feature: FeatureCreate):
-    try:
-        session = session_factory()
-        try:
-            new_feature = FeatureModel(
-                name=feature.name,
-                description=feature.description
-            )
-            session.add(new_feature)
-            session.commit()
-            session.refresh(new_feature)
-            
-            return FeatureResponse(
-                id=new_feature.id,
-                name=new_feature.name,
-                description=new_feature.description,
-                created_at=new_feature.created_at
-            )
-            
-        except Exception as e:
-            session.rollback()
-            logger.log_message(f"Error creating feature: {str(e)}", level=logging.ERROR)
-            raise HTTPException(status_code=500, detail=f"Failed to create feature: {str(e)}")
-        finally:
-            session.close()
-            
-    except Exception as e:
-        logger.log_message(f"Error in create_feature: {str(e)}", level=logging.ERROR)
-        raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
-
-# 4. Register in app.py
-from src.routes.feature_routes import router as feature_router
-app.include_router(feature_router)
-```
-
-### 3. **Database Model Pattern**
-
-```python
-# In src/db/schemas/models.py
-from sqlalchemy import Column, Integer, String, DateTime, Boolean, Text, ForeignKey
-from sqlalchemy.orm import relationship
-from sqlalchemy.ext.declarative import declarative_base
-from datetime import datetime, timezone
-
-Base = declarative_base()
-
-class NewModel(Base):
-    __tablename__ = "new_models"
-    
-    # Primary key
-    id = Column(Integer, primary_key=True, autoincrement=True)
-    
-    # Required fields
-    name = Column(String(255), nullable=False, unique=True)
-    
-    # Optional fields
-    description = Column(Text, nullable=True)
-    is_active = Column(Boolean, default=True, nullable=False)
-    
-    # Timestamps
-    created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), nullable=False)
-    updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc), nullable=False)
-    
-    # Foreign keys
-    user_id = Column(Integer, ForeignKey("users.user_id"), nullable=True)
-    
-    # Relationships
-    user = relationship("User", back_populates="new_models")
-    
-    def __repr__(self):
-        return f"<NewModel(id={self.id}, name='{self.name}')>"
-
-# Update User model to include back reference
-class User(Base):
-    # ... existing fields ...
-    new_models = relationship("NewModel", back_populates="user")
-```
-
-### 4. **Manager Pattern**
-
-```python
-# In src/managers/feature_manager.py
-from typing import List, Optional, Dict, Any
-from sqlalchemy.orm import Session
-from src.db.schemas.models import FeatureModel
-from src.utils.logger import Logger
-
-logger = Logger("feature_manager", see_time=True, console_log=False)
-
-class FeatureManager:
-    """
-    Manages business logic for feature operations.
-    Separates complex business logic from route handlers.
-    """
-    
-    def __init__(self, session: Session):
-        self.session = session
-    
-    async def create_feature(self, name: str, description: Optional[str] = None) -> FeatureModel:
-        """Create a new feature with validation and business logic."""
-        try:
-            # Validation
-            if not name or len(name.strip()) == 0:
-                raise ValueError("Feature name cannot be empty")
-            
-            # Check for duplicates
-            existing = self.session.query(FeatureModel).filter_by(name=name).first()
-            if existing:
-                raise ValueError(f"Feature with name '{name}' already exists")
-            
-            # Create feature
-            feature = FeatureModel(name=name, description=description)
-            self.session.add(feature)
-            self.session.commit()
-            self.session.refresh(feature)
-            
-            logger.log_message(f"Created feature: {name}", level=logging.INFO)
-            return feature
-            
-        except Exception as e:
-            self.session.rollback()
-            logger.log_message(f"Error creating feature: {str(e)}", level=logging.ERROR)
-            raise
-    
-    async def get_features(self, active_only: bool = True) -> List[FeatureModel]:
-        """Retrieve features with optional filtering."""
-        try:
-            query = self.session.query(FeatureModel)
-            if active_only:
-                query = query.filter(FeatureModel.is_active == True)
-            
-            features = query.order_by(FeatureModel.created_at.desc()).all()
-            return features
-            
-        except Exception as e:
-            logger.log_message(f"Error retrieving features: {str(e)}", level=logging.ERROR)
-            raise
-```
-
-## 📋 Code Quality Standards
-
-### 1. **Type Hints and Documentation**
-
-```python
-from typing import List, Optional, Dict, Any, Union
-from datetime import datetime
-
-async def process_analysis_data(
-    data: pd.DataFrame,
-    analysis_type: str,
-    user_id: Optional[int] = None,
-    options: Dict[str, Any] = None
-) -> Dict[str, Union[str, List[Any], bool]]:
-    """
-    Process analysis data with specified parameters.
-    
-    Args:
-        data: Input DataFrame containing the data to analyze
-        analysis_type: Type of analysis to perform ("statistical", "ml", "viz")
-        user_id: Optional user ID for tracking and personalization
-        options: Additional options for analysis configuration
-        
-    Returns:
-        Dictionary containing:
-        - status: "success" or "error"
-        - result: Analysis results or error message
-        - metadata: Additional information about the analysis
-        
-    Raises:
-        ValueError: If analysis_type is not supported
-        DataError: If data format is invalid
-        
-    Example:
-        >>> data = pd.DataFrame({"x": [1, 2, 3], "y": [4, 5, 6]})
-        >>> result = await process_analysis_data(data, "statistical")
-        >>> print(result["status"])
-        "success"
-    """
-    if options is None:
-        options = {}
-    
-    # Implementation...
-    return {"status": "success", "result": [], "metadata": {}}
-```
-
-### 2. **Error Handling Patterns**
-
-```python
-# Comprehensive error handling with logging and user-friendly messages
-async def safe_operation(data: Any) -> Dict[str, Any]:
-    """
-    Template for safe operations with comprehensive error handling.
-    """
-    try:
-        # Validation
-        if not data:
-            raise ValueError("Data cannot be empty")
-        
-        # Main operation
-        result = await perform_operation(data)
-        
-        # Success logging
-        logger.log_message("Operation completed successfully", level=logging.INFO)
-        return {"success": True, "data": result}
-        
-    except ValueError as e:
-        # Input validation errors
-        logger.log_message(f"Validation error: {str(e)}", level=logging.WARNING)
-        return {"success": False, "error": "Invalid input", "details": str(e)}
-        
-    except ConnectionError as e:
-        # External service errors
-        logger.log_message(f"Connection error: {str(e)}", level=logging.ERROR)
-        return {"success": False, "error": "Service unavailable", "details": "Please try again later"}
-        
-    except Exception as e:
-        # Unexpected errors
-        logger.log_message(f"Unexpected error in safe_operation: {str(e)}", level=logging.ERROR)
-        return {"success": False, "error": "Internal error", "details": "Please contact support"}
-```
-
-### 3. **Async/Await Best Practices**
-
-```python
-import asyncio
-from typing import List, Coroutine
-
-# Proper async function definition
-async def async_agent_execution(agents: List[str], query: str) -> List[Dict[str, Any]]:
-    """Execute multiple agents concurrently."""
-    
-    # Create coroutines
-    tasks = [
-        execute_single_agent(agent, query) 
-        for agent in agents
-    ]
-    
-    # Execute concurrently with error handling
-    results = []
-    for task in asyncio.as_completed(tasks):
-        try:
-            result = await task
-            results.append(result)
-        except Exception as e:
-            logger.log_message(f"Agent execution failed: {e}", level=logging.ERROR)
-            results.append({"error": str(e)})
-    
-    return results
-
-# Database operations with proper session management
-async def async_database_operation(session: Session) -> Any:
-    """Template for async database operations."""
-    try:
-        # Use asyncio.to_thread for CPU-bound database operations
-        result = await asyncio.to_thread(
-            lambda: session.query(Model).filter(...).all()
-        )
-        return result
-    except Exception as e:
-        session.rollback()
-        raise
-    finally:
-        session.close()
-```
-
-## 🔧 Development Workflow
-
-### 1. **Feature Development Process**
-
-1. **Plan the Feature**:
-   ```bash
-   # Create feature branch
-   git checkout -b feature/new-analysis-agent
-   
-   # Document requirements
-   echo "## New Analysis Agent" >> docs/feature_plan.md
-   ```
-
-2. **Implement Core Logic**:
-   ```bash
-   # Create agent signature
-   # Add to agents_config.json
-   # Implement business logic in managers/
-   # Create route handlers
-   ```
-
-3. **Add Database Changes**:
-   ```bash
-   # Modify models if needed
-   alembic revision --autogenerate -m "Add new analysis tables"
-   alembic upgrade head
-   ```
-
-### 3. **Release Process**
-
-1. **Pre-release Testing**:
-   ```bash
-   # Run full test suite
-   pytest tests/
-   
-   # Test database migrations
-   alembic upgrade head
-   
-   # Test with sample data
-   python scripts/test_with_sample_data.py
-   ```
-
-2. **Documentation Updates**:
-   ```bash
-   # Update API documentation
-   # Update troubleshooting guide
-   # Update changelog
-   ```
-
-3. **Deployment Preparation**:
-   ```bash
-   # Update requirements.txt
-   pip freeze > requirements.txt
-   
-   # Test container build
-   docker build -t auto-analyst-backend .
-   
-   ```
-
-## 📊 Performance Considerations
-
-### 1. **Database Optimization**
-
-```python
-# Use query optimization
-from sqlalchemy.orm import joinedload
-
-# Bad: N+1 query problem
-users = session.query(User).all()
-for user in users:
-    print(user.chats)  # Separate query for each user
-
-# Good: Eager loading
-users = session.query(User).options(joinedload(User.chats)).all()
-for user in users:
-    print(user.chats)  # No additional queries
-
-# Use pagination for large datasets
-def get_paginated_results(session, model, page=1, per_page=20):
-    offset = (page - 1) * per_page
-    return session.query(model).offset(offset).limit(per_page).all()
-```
-
-
-### 2. **Async Optimization**
-
-```python
-# Use connection pooling
-from sqlalchemy.pool import QueuePool
-
-engine = create_engine(
-    DATABASE_URL,
-    poolclass=QueuePool,
-    pool_size=20,
-    max_overflow=30
-)
-
-# Batch operations
-async def batch_process_agents(agents: List[str], queries: List[str]):
-    semaphore = asyncio.Semaphore(5)  # Limit concurrent operations
-    
-    async def process_with_limit(agent, query):
-        async with semaphore:
-            return await process_agent(agent, query)
-    
-    tasks = [
-        process_with_limit(agent, query) 
-        for agent, query in zip(agents, queries)
-    ]
-    
-    return await asyncio.gather(*tasks, return_exceptions=True)
-```
-
-This development workflow guide provides a comprehensive framework for maintaining code quality, consistency, and performance in the Auto-Analyst backend system. Following these patterns ensures that new features integrate seamlessly with the existing architecture while maintaining the high standards of the codebase. 

docs/{api/README.md → endpoints.md}

@@ -5,19 +5,7 @@ This document is a guide to the backend API endpoints utilized within the Auto-A
 The Auto-Analyst application is designed to facilitate seamless interactions and efficient data management, making it essential for users to understand the available endpoints and their functionalities. Each section of this documentation is crafted to provide clarity and insight into how the API operates, ensuring that developers and users alike can effectively leverage its capabilities.
 
 For more specific details regarding the various functionalities offered by the API, please refer to the following sections, which delve deeper into their respective areas:
-
-## 📚 Core Documentation
-
-- **[Getting Started Guide](./getting_started.md)**: Quick start guide for new developers and LLMs to understand the system architecture and get up to speed quickly
-- **[System Architecture](./architecture.md)**: Comprehensive overview of the backend system design, components, and data flow patterns
-- **[Troubleshooting Guide](./troubleshooting.md)**: Common issues, debugging tools, and solutions for development and deployment problems
-
-## 🛠️ API Reference
-
-- **[Core Endpoints](./routes/session.md)**: Review the core endpoints that handle fundamental operations within the application, including data uploads, AI analysis, model settings, and session management.
-- **[Analytics Endpoints](./routes/analytics.md)**: Explore the endpoints dedicated to analytics, providing insights into usage statistics, performance metrics, cost analysis, and real-time monitoring.
-- **[Chat Endpoints](./routes/chats.md)**: Discover the endpoints that manage chat interactions, enabling users to create, retrieve, and manage chat sessions effectively.
-- **[Code Endpoints](./routes/code.md)**: Learn about the endpoints for code execution, editing, fixing, and cleaning operations with advanced AI assistance.
-- **[Deep Analysis Endpoints](./routes/deep_analysis.md)**: Comprehensive documentation for the multi-agent deep analysis system, including streaming progress, report management, template integration, and how user's active agents are leveraged for advanced analytical insights.
-- **[Feedback Endpoints](./routes/feedback.md)**: Understand the endpoints for managing user feedback on AI-generated messages, including rating systems and model performance tracking.
-- **[Templates Endpoints](./routes/templates.md)**: Comprehensive guide to the template system, agent loading, user preferences, and how personalized AI agent configurations work for different users.
+- [Analytics Endpoints](./analytics.md): Explore the endpoints dedicated to analytics, providing insights into usage statistics and performance metrics.
+- [Chat Endpoints](./chats.md): Discover the endpoints that manage chat interactions, enabling users to create, retrieve, and manage chat sessions effectively.
+- [Core Endpoints](./core.md): Review the core endpoints that handle fundamental operations within the application, including data uploads and session management.
+- [Code Endpoints](./code.md): 

docs/getting_started.md

@@ -1,273 +0,0 @@
-# Auto-Analyst Backend - Getting Started Guide
-
-## 🎯 Overview
-
-This guide will help you set up and understand the Auto-Analyst backend system. Auto-Analyst is a multi-agent AI platform that orchestrates specialized agents for comprehensive data analysis.
-
-## 🏗️ Core Concepts
-
-### 1. **Multi-Agent System**
-The platform uses specialized AI agents:
-- **Preprocessing Agent**: Data cleaning and preparation
-- **Statistical Analytics Agent**: Statistical analysis and insights
-- **Machine Learning Agent**: Scikit-learn based modeling
-- **Data Visualization Agent**: Chart and plot generation
-
-### 2. **Template System**
-- **Individual Agents**: Single-purpose agents for specific tasks
-- **Planner Agents**: Multi-agent coordination for complex workflows
-- **User Templates**: Customizable agent preferences
-- **Default vs Premium**: Core agents available to all users
-
-### 3. **Session Management**
-- Session-based user tracking
-- Shared DataFrame context between agents
-- Conversation history and code execution tracking
-
-### 4. **Deep Analysis System**
-- Multi-step analysis workflow (questions → planning → execution → synthesis)
-- Streaming progress updates
-- HTML report generation
-
-## 🚀 Quick Start
-
-### 1. Installation
-
-```bash
-# Clone and navigate to backend
-cd Auto-Analyst-CS/auto-analyst-backend
-
-# Create virtual environment
-python -m venv venv
-source venv/bin/activate  # Linux/Mac
-# or
-venv\Scripts\activate     # Windows
-
-# Install dependencies
-pip install -r requirements.txt
-```
-
-### 2. Environment Variables
-
-Create `.env` file with:
-
-```env
-# Database
-DATABASE_URL=sqlite:///./auto_analyst.db  # For development
-# DATABASE_URL=postgresql://user:pass@host:port/db  # For production
-
-# AI Models
-ANTHROPIC_API_KEY=your_anthropic_key_here
-OPENAI_API_KEY=your_openai_key_here
-
-# Authentication (optional)
-ADMIN_API_KEY=your_admin_key_here
-```
-
-### 3. Database Initialization
-
-```bash
-# Initialize database and default agents
-python -c "
-from src.db.init_db import init_db
-init_db()
-print('✅ Database initialized successfully')
-"
-```
-
-### 4. Start the Server
-
-```bash
-# Development server
-python app.py
-
-# Or with uvicorn
-uvicorn app:app --reload --host 0.0.0.0 --port 8000
-```
-
-### 5. Verify Setup
-
-Visit: `http://localhost:8000/docs` for interactive API documentation
-
-## 📚 Key Files to Understand
-
-### Core Application Files
-
-1. **`app.py`** - Main FastAPI application and core endpoints
-2. **`src/agents/agents.py`** - Agent definitions and orchestration
-3. **`src/agents/deep_agents.py`** - Deep analysis system
-4. **`src/db/schemas/models.py`** - Database models
-5. **`src/managers/chat_manager.py`** - Chat and session management
-
-### Route Files (API Endpoints)
-
-- **`src/routes/session_routes.py`** - File uploads, sessions, authentication
-- **`src/routes/chat_routes.py`** - Chat and messaging
-- **`src/routes/code_routes.py`** - Code execution and processing
-- **`src/routes/templates_routes.py`** - Agent template management
-- **`src/routes/deep_analysis_routes.py`** - Deep analysis reports
-- **`src/routes/analytics_routes.py`** - Usage analytics and monitoring
-
-### Configuration Files
-
-- **`agents_config.json`** - Agent and template definitions
-- **`requirements.txt`** - Python dependencies
-- **`alembic.ini`** - Database migration configuration
-
-## 🔧 Development Workflow
-
-### 1. Adding New Agents
-
-```python
-# 1. Define agent signature in src/agents/agents.py
-class new_agent(dspy.Signature):
-    """Agent description"""
-    goal = dspy.InputField(desc="Analysis goal")
-    dataset = dspy.InputField(desc="Dataset info")
-    result = dspy.OutputField(desc="Analysis result")
-
-# 2. Add to agents_config.json
-{
-  "template_name": "new_agent",
-  "description": "Agent description",
-  "variant_type": "both",
-  "is_premium": false,
-  "usage_count": 0
-}
-
-# 3. Register in agent loading system
-```
-
-### 2. Adding New Endpoints
-
-```python
-# 1. Create route in src/routes/feature_routes.py
-from fastapi import APIRouter
-router = APIRouter(prefix="/feature", tags=["feature"])
-
-@router.get("/endpoint")
-async def new_endpoint():
-    return {"message": "Hello"}
-
-# 2. Register in app.py
-from src.routes.feature_routes import router as feature_router
-app.include_router(feature_router)
-```
-
-### 3. Database Changes
-
-```bash
-# 1. Modify models in src/db/schemas/models.py
-# 2. Create migration
-alembic revision --autogenerate -m "description"
-# 3. Apply migration
-alembic upgrade head
-```
-
-## 🧪 Testing Your Changes
-
-### 1. Test API Endpoints
-
-```bash
-# Use the interactive docs
-open http://localhost:8000/docs
-
-# Or use curl
-curl -X GET "http://localhost:8000/health"
-```
-
-### 2. Test Agent System
-
-```python
-# Test individual agent
-python -c "
-from src.agents.agents import preprocessing_agent
-import dspy
-dspy.LM('anthropic/claude-sonnet-4-20250514')
-agent = dspy.ChainOfThought(preprocessing_agent)
-result = agent(goal='clean data', dataset='test data')
-print(result)
-"
-```
-
-### 3. Test Database Operations
-
-```python
-# Test database
-python -c "
-from src.db.init_db import session_factory
-from src.db.schemas.models import AgentTemplate
-session = session_factory()
-templates = session.query(AgentTemplate).all()
-print(f'Found {len(templates)} templates')
-session.close()
-"
-```
-
-## 🔍 Common Development Tasks
-
-### Adding a New Feature
-
-1. **Plan the Feature**: Define requirements and API design
-2. **Database Changes**: Add new models if needed
-3. **Create Routes**: Add API endpoints in `src/routes/`
-4. **Business Logic**: Add managers in `src/managers/` if complex
-5. **Documentation**: Update relevant `.md` files
-6. **Testing**: Test endpoints and integration
-
-### Debugging Issues
-
-1. **Check Logs**: Application logs show detailed error information
-2. **Database State**: Verify data with database queries
-3. **API Testing**: Use `/docs` interface for endpoint testing
-4. **Agent Behavior**: Test individual agents separately
-
-### Performance Optimization
-
-1. **Database Queries**: Use SQLAlchemy query optimization
-2. **Agent Execution**: Implement async patterns for agent orchestration
-3. **Resource Management**: Monitor memory usage for large datasets
-
-## 📊 System Architecture Overview
-
-```mermaid
-graph TD
-    A[Frontend Request] --> B[FastAPI Router]
-    B --> C[Route Handler]
-    C --> D[Manager Layer]
-    D --> E[Database Layer]
-    D --> F[Agent System]
-    F --> G[AI Models]
-    G --> H[Code Generation]
-    H --> I[Execution Environment]
-    I --> J[Results Processing]
-    J --> K[Response]
-    
-    subgraph "Agent Orchestration"
-        F1[Individual Agents]
-        F2[Planner Module]
-        F3[Deep Analysis]
-        F1 --> F2
-        F2 --> F3
-    end
-    
-    F --> F1
-```
-
-## 📈 Template Integration
-
-The system uses **active user templates** for agent selection:
-
-### Default Agents (Always Available)
-- `preprocessing_agent` (individual & planner variants)
-- `statistical_analytics_agent` (individual & planner variants)  
-- `sk_learn_agent` (individual & planner variants)
-- `data_viz_agent` (individual & planner variants)
-
-### Template Loading Logic
-1. **Individual Agent Execution** (`@agent_name`): Loads ALL available templates
-2. **Planner Execution**: Loads user's enabled templates (max 10 for performance)
-3. **Deep Analysis**: Uses user's active template preferences
-4. **Fallback**: Uses 4 core agents if no user preferences found
-
-This architecture ensures users can leverage their preferred agents while maintaining system performance and reliability. 

docs/routes/analytics.md

@@ -0,0 +1,200 @@
+
+## Analytics Routes Overview
+
+These routes provide comprehensive analytics functionality for the Auto-Analyst application, including dashboard summaries, user and model analytics, and cost breakdowns.
+
+### Authentication
+
+All analytics endpoints require admin authentication via an API key:
+
+```python
+ADMIN_API_KEY = os.getenv("ADMIN_API_KEY", "default-admin-key-change-me")
+```
+
+The API key can be provided via:
+- **Header:** `X-Admin-API-Key`
+- **Query parameter:** `admin_api_key`
+
+---
+
+### Dashboard Endpoints
+
+#### **GET /analytics/dashboard**
+Returns a comprehensive summary of usage data for the dashboard.
+
+**Query Parameters:**
+- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
+
+**Response:**
+```json
+{
+  "total_tokens": 123456,
+  "total_cost": 25.50,
+  "total_requests": 1000,
+  "total_users": 50,
+  "daily_usage": [
+    {
+      "date": "2023-05-01",
+      "tokens": 5000,
+      "cost": 1.25,
+      "requests": 100
+    }
+  ],
+  "model_usage": [
+    {
+      "model_name": "gpt-4",
+      "tokens": 10000,
+      "cost": 10.00,
+      "requests": 200
+    }
+  ],
+  "top_users": [
+    {
+      "user_id": "123",
+      "tokens": 5000,
+      "cost": 5.00,
+      "requests": 50
+    }
+  ],
+  "start_date": "2023-04-01",
+  "end_date": "2023-05-01"
+}
+```
+
+**Process Flow:**
+1. Parse date range from the `period` parameter.
+2. Query total statistics (tokens, cost, requests, users).
+3. Query daily usage broken down by date.
+4. Query model usage statistics.
+5. Query top users by token usage.
+6. Return combined data.
+
+---
+
+### WebSocket **/analytics/dashboard/realtime**
+
+WebSocket endpoint for real-time updates to dashboard data.
+
+**Data Flow:**
+1. Client connects to WebSocket.
+2. Server adds the connection to the active connections set.
+3. Server broadcasts updates to all connected clients when new data arrives.
+4. Connection is removed when the client disconnects.
+
+---
+
+## User Analytics Endpoints
+
+### **GET /analytics/users**
+Returns a list of users with their usage statistics.
+
+**Query Parameters:**
+- `limit` (optional): Maximum number of users to return (default: `100`)
+- `offset` (optional): Offset for pagination (default: `0`)
+
+**Response:**
+```json
+{
+  "users": [
+    {
+      "user_id": "123",
+      "tokens": 5000,
+      "cost": 5.00,
+      "requests": 50,
+      "first_seen": "2023-04-01T12:00:00Z",
+      "last_seen": "2023-05-01T12:00:00Z"
+    }
+  ],
+  "total": 200,
+  "limit": 100,
+  "offset": 0
+}
+```
+
+**Process Flow:**
+1. Query user data with aggregated metrics.
+2. Calculate total users for pagination.
+3. Format and return data.
+
+---
+
+### **GET /analytics/users/activity**
+Returns user activity metrics over time.
+
+**Query Parameters:**
+- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
+
+**Response:**
+```json
+{
+  "user_activity": [
+    {
+      "date": "2023-05-01",
+      "activeUsers": 20,
+      "newUsers": 5,
+      "sessions": 30
+    }
+  ]
+}
+```
+
+**Process Flow:**
+1. Parse date range from `period` parameter.
+2. Get first date each user was seen (for new users count).
+3. Get daily activity metrics.
+4. Fill in any missing dates with zeros.
+5. Return formatted data.
+
+---
+
+## Model Analytics Endpoints
+
+### **GET /analytics/usage/models**
+Returns model usage breakdown.
+
+**Query Parameters:**
+- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
+
+**Response:**
+```json
+{
+  "model_usage": [
+    {
+      "model_name": "gpt-4",
+      "tokens": 10000,
+      "cost": 10.00,
+      "requests": 200,
+      "avg_response_time": 1.5
+    }
+  ]
+}
+```
+
+**Process Flow:**
+1. Parse date range from `period` parameter.
+2. Query model usage with aggregated metrics.
+3. Format and return data.
+
+---
+
+## Cost Analytics Endpoints
+
+### **GET /analytics/costs/summary**
+Returns a summary of costs.
+
+**Query Parameters:**
+- `period` (optional): Time period (`7d`, `30d`, `90d`, default: `30d`)
+
+**Response:**
+```json
+{
+  "totalCost": 25.50,
+  "totalTokens": 100000,
+  "totalRequests": 1000,
+  "avgDailyCost": 0.85,
+  "costPerThousandTokens": 0.255,
+  "daysInPeriod": 30,
+  "startDate": "2023-04-01",
+  "endDate": "2023-05-01"
+}
+```

docs/{api/routes → routes}/chats.md

@@ -7,23 +7,27 @@ These routes handle chat interactions, message processing, user management, and
 ### **Chat Management**
 
 #### **1. Create a New Chat**
-**Endpoint:** `POST /chats/`  
+**Endpoint:** `POST /chats`  
 **Description:** Creates a new chat session.  
 **Request Body:**  
 ```json
 {
-  "user_id": 123
+  "user_id": 123,
+  "is_admin": false
 }
 ```
 **Response:**  
 ```json
 {
   "chat_id": 456,
-  "user_id": 123,
   "title": "New Chat",
-  "created_at": "2023-05-01T12:00:00Z"
+  "created_at": "2023-05-01T12:00:00Z",
+  "user_id": 123
 }
 ```
+**Process Flow:**  
+1. Create a new chat for the given user.  
+2. Return the chat details.  
 
 ---
 
@@ -50,33 +54,42 @@ These routes handle chat interactions, message processing, user management, and
   ]
 }
 ```
+**Process Flow:**  
+1. Retrieve chat details.  
+2. Validate user access if `user_id` is provided.  
+3. Fetch all messages in the chat.  
+4. Return the chat with its messages.  
 
 ---
 
 #### **3. List Recent Chats**
-**Endpoint:** `GET /chats/`  
+**Endpoint:** `GET /chats`  
 **Description:** Retrieves a list of recent chats, optionally filtered by user ID.  
 **Query Parameters:**  
 - `user_id` (Optional for filtering by user)  
-- `limit` (Maximum number of chats, default: 10, max: 100)  
+- `limit` (Maximum number of chats, default: 10)  
 - `offset` (For pagination, default: 0)  
 **Response:**  
 ```json
 [
   {
     "chat_id": 456,
-    "user_id": 123,
     "title": "New Chat",
-    "created_at": "2023-05-01T12:00:00Z"
+    "created_at": "2023-05-01T12:00:00Z",
+    "user_id": 123
   }
 ]
 ```
+**Process Flow:**  
+1. Fetch recent chats.  
+2. Apply filters and pagination.  
+3. Return the list of chats.  
 
 ---
 
 #### **4. Update a Chat**
 **Endpoint:** `PUT /chats/{chat_id}`  
-**Description:** Updates a chat's title or user ID.  
+**Description:** Updates a chat’s title or user ID.  
 **Path Parameter:** `chat_id` (ID of the chat to update)  
 **Request Body:**  
 ```json
@@ -94,25 +107,57 @@ These routes handle chat interactions, message processing, user management, and
   "user_id": 123
 }
 ```
+**Process Flow:**  
+1. Update the chat’s title or user ID.  
+2. Return the updated details.  
 
 ---
 
 #### **5. Delete a Chat**
 **Endpoint:** `DELETE /chats/{chat_id}`  
-**Description:** Deletes a chat and all its messages while preserving model usage records.  
+**Description:** Deletes a chat and all its messages.  
 **Path Parameter:** `chat_id` (ID of the chat to delete)  
 **Query Parameter:** `user_id` (Optional for access control)  
 **Response:**  
 ```json
 {
-  "message": "Chat 456 deleted successfully",
-  "preserved_model_usage": true
+  "message": "Chat 456 deleted successfully"
 }
 ```
+**Process Flow:**  
+1. Validate if the chat exists and if the user has access.  
+2. Delete the chat and associated messages.  
+3. Return a success message.  
+
+---
+
+#### **6. Search Chats**
+**Endpoint:** `GET /chats/search/`  
+**Description:** Searches chats based on a query string.  
+**Query Parameters:**  
+- `query` (Search term)  
+- `user_id` (Optional filter)  
+- `limit` (Max results)  
+**Response:**  
+```json
+[
+  {
+    "chat_id": 456,
+    "title": "Chat about machine learning",
+    "created_at": "2023-05-01T12:00:00Z",
+    "user_id": 123
+  }
+]
+```
+**Process Flow:**  
+1. Search chat titles and messages for the query.  
+2. Filter by `user_id` if provided.  
+3. Apply a result limit.  
+4. Return the matching chats.  
 
 ---
 
-#### **6. Cleanup Empty Chats**
+#### **7. Cleanup Empty Chats**
 **Endpoint:** `POST /chats/cleanup-empty`  
 **Description:** Deletes empty chats for a user.  
 **Request Body:**  
@@ -128,33 +173,30 @@ These routes handle chat interactions, message processing, user management, and
   "message": "Deleted 5 empty chats"
 }
 ```
+**Process Flow:**  
+1. Identify chats with no messages.  
+2. Delete those chats.  
+3. Return the count of deleted chats.  
 
 ---
 
 ### **Message Management**
 
-#### **1. Add Message to Chat**
-**Endpoint:** `POST /chats/{chat_id}/messages`  
-**Description:** Adds a message to an existing chat.  
+
+
+#### **1. Send a Message and Get AI Response**
+**Endpoint:** `POST /chats/{chat_id}/message`  
+**Description:** Sends a message and receives an AI-generated response.  
 **Path Parameter:** `chat_id` (ID of the chat)  
-**Query Parameter:** `user_id` (Optional for access control)  
 **Request Body:**  
-```json
-{
-  "content": "Hello, I need help with data analysis",
-  "sender": "user"
-}
-```
-**Response:**  
-```json
-{
-  "message_id": 789,
-  "chat_id": 456,
-  "content": "Hello, I need help with data analysis",
-  "sender": "user",
-  "timestamp": "2023-05-01T12:01:00Z"
-}
-```
+**Process Flow:**  
+1. Verify chat existence and user access.  
+2. Store the user message.  
+3. Generate an AI response via `ai_manager`.  
+4. Track model usage metrics.  
+5. Store the AI response.  
+6. Update the chat title if it's the first or second message.  
+7. Return both messages.  
 
 ---
 
@@ -179,3 +221,37 @@ These routes handle chat interactions, message processing, user management, and
   "created_at": "2023-05-01T12:00:00Z"
 }
 ```
+**Process Flow:**  
+1. Check if a user with the email exists.  
+2. Create a new user if not found.  
+3. Return the user details.  
+
+---
+
+### **Debugging**
+
+#### **1. Test Model Usage Tracking**
+**Endpoint:** `POST /chats/debug/test-model-usage`  
+**Query Parameters:**  
+- `model_name`: Model to test  
+- `user_id`: Optional  
+**Response:**  
+```json
+{
+  "success": true,
+  "message": "Model usage tracking test completed",
+  "response": "This is a test response",
+  "usage_recorded": {
+    "usage_id": 123,
+    "model_name": "gpt-4",
+    "tokens": 50,
+    "cost": 0.0005,
+    "timestamp": "2023-05-01T12:00:00Z"
+  }
+}
+```
+**Process Flow:**  
+1. Generate a test prompt.  
+2. Call AI manager.  
+3. Fetch usage data.  
+4. Return test results.

docs/{api/routes → routes}/code.md

@@ -16,17 +16,15 @@ Executes Python code against the current session's dataframe.
 **Request Body:**
 ```json
 {
-    "code": "string",              // Python code to execute
-    "session_id": "string",        // Optional session ID
-    "message_id": 123              // Optional message ID for tracking
+    "code": "string"  // Python code to execute
 }
 ```
 
 **Response:**
 ```json
 {
-    "output": "string",            // Execution output
-    "plotly_outputs": [            // Optional array of plotly outputs
+    "output": "string",  // Execution output
+    "plotly_outputs": [  // Optional array of plotly outputs
         "string"
     ]
 }
@@ -44,15 +42,15 @@ Uses AI to edit code based on user instructions.
 **Request Body:**
 ```json
 {
-    "original_code": "string",     // Code to be edited
-    "user_prompt": "string"        // Instructions for editing
+    "original_code": "string",  // Code to be edited
+    "user_prompt": "string"     // Instructions for editing
 }
 ```
 
 **Response:**
 ```json
 {
-    "edited_code": "string"        // The edited code
+    "edited_code": "string"  // The edited code
 }
 ```
 
@@ -61,22 +59,22 @@ Uses AI to edit code based on user instructions.
 - `500 Internal Server Error`: Editing error
 
 ### Fix Code
-Uses AI to fix code with errors, employing a block-by-block approach with DSPy refinement.
+Uses AI to fix code with errors, employing a block-by-block approach.
 
 **Endpoint:** `POST /code/fix`
 
 **Request Body:**
 ```json
 {
-    "code": "string",              // Code containing errors
-    "error": "string"              // Error message to fix
+    "code": "string",    // Code containing errors
+    "error": "string"    // Error message to fix
 }
 ```
 
 **Response:**
 ```json
 {
-    "fixed_code": "string"         // The fixed code
+    "fixed_code": "string"  // The fixed code
 }
 ```
 
@@ -92,14 +90,14 @@ Cleans and formats code by organizing imports and ensuring proper code block for
 **Request Body:**
 ```json
 {
-    "code": "string"               // Code to clean
+    "code": "string"  // Code to clean
 }
 ```
 
 **Response:**
 ```json
 {
-    "cleaned_code": "string"       // The cleaned code
+    "cleaned_code": "string"  // The cleaned code
 }
 ```
 
@@ -107,30 +105,6 @@ Cleans and formats code by organizing imports and ensuring proper code block for
 - `400 Bad Request`: No code provided
 - `500 Internal Server Error`: Cleaning error
 
-### Get Latest Code
-Retrieves the latest code from a specific message.
-
-**Endpoint:** `POST /code/get-latest-code`
-
-**Request Body:**
-```json
-{
-    "message_id": 123              // Message ID to retrieve code from
-}
-```
-
-**Response:**
-```json
-{
-    "code": "string"               // The retrieved code
-}
-```
-
-**Error Responses:**
-- `400 Bad Request`: Missing message ID
-- `404 Not Found`: Message not found
-- `500 Internal Server Error`: Retrieval error
-
 ## Code Processing Features
 
 ### Import Organization
@@ -144,12 +118,10 @@ The system supports code blocks marked with special comments:
 - Start marker: `# agent_name code start`
 - End marker: `# agent_name code end`
 
-### Error Handling with DSPy Refinement
-When fixing code, the system uses DSPy's refinement mechanism:
+### Error Handling
+When fixing code, the system:
 - Identifies specific code blocks with errors
 - Processes error messages to extract relevant information
-- Uses a scoring function to validate fixes
-- Employs iterative refinement with up to 3 attempts
 - Fixes each block individually while maintaining the overall structure
 - Preserves code block markers and relationships
 
@@ -160,23 +132,14 @@ When editing or fixing code, the system provides context about the current datas
 - Null value counts
 - Sample values for each column
 
-### Code Execution Safety
-The execution system includes safety measures:
-- Removes blocking calls like `plt.show()`
-- Handles `__main__` block extraction
-- Cleans up print statements with unwanted newlines
-- Executes code in isolated namespaces
-
 ## Session Management
 All endpoints require a valid session ID, which is used to:
 - Access the current dataset
 - Maintain state between requests
 - Track code execution history
-- Store execution results for analysis
 
 ## Error Handling
 The system provides detailed error messages while maintaining security by:
 - Logging errors for debugging
 - Returning user-friendly error messages
 - Preserving original code in case of processing failures
-- Using code scoring to validate fixes before returning results

docs/routes/core.md

@@ -0,0 +1,242 @@
+
+# **Auto-Analyst API Documentation**
+
+The core application routes are designed to manage the data and AI analysis capabilities of the Auto-Analyst application.
+
+## **1. Core Application Routes**
+### **Data Management**
+- **POST /upload_dataframe**  
+  **Uploads a dataset for analysis.**  
+  **Request:**  
+  - `file`: CSV file  
+  - `name`: Dataset name  
+  - `description`: Dataset description  
+  - `session_id`: Session ID  
+  **Response:**  
+  ```json
+  { "message": "Dataframe uploaded successfully", "session_id": "abc123" }
+  ```
+  **Process Flow:**  
+  - Read CSV file  
+  - Create dataset description  
+  - Update session with dataset  
+  - Return success message  
+
+- **GET /api/default-dataset**  
+  **Gets the default dataset.**  
+  **Query Parameters:** `session_id`  
+  **Response:**  
+  ```json
+  {
+    "headers": ["column1", "column2", ...],
+    "rows": [[val1, val2, ...], ...],
+    "name": "Housing Dataset",
+    "description": "A comprehensive dataset containing housing information..."
+  }
+  ```
+  **Process Flow:**  
+  - Reset session to use default dataset  
+  - Format dataset preview  
+  - Return formatted data  
+
+- **POST /reset-session**  
+  **Resets session to default dataset.**  
+  **Query Parameters:** `session_id`, `name` (optional), `description` (optional)  
+  **Response:**  
+  ```json
+  {
+    "message": "Session reset to default dataset",
+    "session_id": "abc123",
+    "dataset": "Housing.csv"
+  }
+  ```
+  **Process Flow:**  
+  - Reset session  
+  - Update dataset description (if provided)  
+  - Return success message  
+
+---
+
+### **2. AI Analysis**
+- **POST /chat/{agent_name}**  
+  **Processes a query using a specific AI agent.**  
+  **Path Parameters:** `agent_name`  
+  **Request Body:**  
+  ```json
+  { "query": "Analyze the relationship between price and size" }
+  ```
+  **Query Parameters:** `session_id`, `user_id` (optional), `chat_id` (optional)  
+  **Response:**  
+  ```json
+  {
+    "agent_name": "data_viz_agent",
+    "query": "Analyze the relationship between price and size",
+    "response": "# Analysis\n\nThere appears to be a strong positive correlation...",
+    "session_id": "abc123"
+  }
+  ```
+  **Process Flow:**  
+  - Get session state  
+  - Validate dataset and agent  
+  - Execute agent query  
+  - Format and return response  
+
+- **POST /chat**  
+  **Processes a query using multiple AI agents.**  
+  **Request Body:**  
+  ```json
+  { "query": "Analyze the housing data" }
+  ```
+  **Response:** *Streaming JSON objects:*  
+  ```json
+  {"agent": "data_viz_agent", "content": "# Visualization\n\n...", "status": "success"}
+  {"agent": "statistical_analytics_agent", "content": "# Statistical Analysis\n\n...", "status": "success"}
+  ```
+  **Process Flow:**  
+  - Get session state  
+  - Validate dataset  
+  - Generate AI analysis plan  
+  - Execute with multiple agents and stream responses  
+
+- **POST /execute_code**  
+  **Executes Python code and returns results.**  
+  **Request Body:**  
+  ```json
+  { "code": "import pandas as pd\nimport matplotlib.pyplot as plt\n..." }
+  ```
+  **Response:**  
+  ```json
+  {
+    "output": "Execution successful",
+    "plotly_outputs": ["```plotly\n{\"data\": [...], \"layout\": {...}}\n```"]
+  }
+  ```
+  **Process Flow:**  
+  - Validate dataset  
+  - Execute code  
+  - Capture output and plots  
+  - Return results  
+
+---
+
+### **3. Model Settings**
+- **GET /api/model-settings**  
+  **Fetches current model settings.**  
+  **Response:**  
+  ```json
+  {
+    "provider": "openai",
+    "model": "gpt-4o-mini",
+    "hasCustomKey": true,
+    "temperature": 1.0,
+    "maxTokens": 6000
+  }
+  ```
+- **POST /settings/model**  
+  **Updates model settings.**  
+  **Request Body:**  
+  ```json
+  {
+    "provider": "openai",
+    "model": "gpt-4",
+    "api_key": "sk-...",
+    "temperature": 0.7,
+    "max_tokens": 8000
+  }
+  ```
+  **Response:**  
+  ```json
+  { "message": "Model settings updated successfully" }
+  ```
+  **Process Flow:**  
+  - Update model settings  
+  - Test configuration  
+  - Return success or error  
+
+- **GET /agents**  
+  **Lists available AI agents.**  
+  **Response:**  
+  ```json
+  {
+    "available_agents": ["data_viz_agent", "sk_learn_agent", "statistical_analytics_agent", "preprocessing_agent"],
+    "description": "List of available specialized agents that can be called using @agent_name"
+  }
+  ```
+
+---
+
+### **4. Authentication & Session Management**
+- **Session ID Sources:**  
+  - Query parameter: `session_id`  
+  - Header: `X-Session-ID`  
+  - Auto-generated if not provided  
+- **Session State Includes:**  
+  - Current dataset  
+  - AI system instance  
+  - Model configuration  
+
+#### **Admin Authentication**
+- **API Key Sources:**  
+  - Header: `X-Admin-API-Key`  
+  - Query Parameter: `admin_api_key`  
+- **Validation:**  
+  - Checked against `ADMIN_API_KEY` environment variable  
+
+---
+
+### **5. AI Agents Integration**
+- **Available Agents:**  
+  - `data_viz_agent`: Creates data visualizations (Plotly)  
+  - `sk_learn_agent`: ML analysis (Scikit-learn)  
+  - `statistical_analytics_agent`: Statistical analysis (StatsModels)  
+  - `preprocessing_agent`: Data transformation  
+
+**Integration Flow:**  
+- Agents are registered in `AVAILABLE_AGENTS`  
+- Queries are dispatched based on content  
+- Responses are streamed in Markdown  
+
+---
+
+### **6. Real-time Updates via WebSockets**
+- **Endpoints:**  
+  - `/analytics/dashboard/realtime`  
+  - `/analytics/realtime`  
+- **Connections stored in:**  
+  - `active_dashboard_connections`  
+  - `active_user_connections`  
+- **Broadcast Function:**
+  ```python
+  async def broadcast_dashboard_update(update_data: Dict[str, Any]):
+      for connection in active_dashboard_connections.copy():
+          try:
+              await connection.send_text(json.dumps(update_data))
+          except Exception:
+              active_dashboard_connections.remove(connection)
+  ```
+
+---
+
+### **7. Error Handling**
+- **Try-Except Blocks:**  
+  ```python
+  try:
+      return result
+  except ValueError as e:
+      raise HTTPException(status_code=404, detail=str(e))
+  except Exception as e:
+      logger.error(f"Error: {str(e)}")
+      raise HTTPException(status_code=500, detail=f"Failed: {str(e)}")
+  ```
+- **HTTP Exception Types:**  
+  - `400`: Bad Request  
+  - `401`: Unauthorized  
+  - `403`: Forbidden  
+  - `404`: Not Found  
+  - `500`: Internal Server Error  
+
+- **Logging Setup:**  
+  ```python
+  logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+  logger = logging.getLogger(__name__)
+  ```

docs/system/database-schema.md

@@ -1,289 +0,0 @@
-# Auto-Analyst Database Schema Documentation
-
-## 📋 Overview
-
-The Auto-Analyst backend uses a relational database schema designed for scalability and data integrity. The schema supports both **SQLite** (development) and **PostgreSQL** (production) databases through SQLAlchemy ORM.
-
-### **Database Features**
-- **User Management** - Authentication and user data
-- **Chat System** - Conversation sessions and message history  
-- **AI Model Tracking** - Usage analytics and cost monitoring
-- **Code Execution** - Code generation and execution tracking
-- **Agent Templates** - Customizable AI agent configurations
-- **Deep Analysis** - Multi-step analysis reports and results
-- **User Feedback** - Rating and feedback system
-
----
-
-## 🗄️ Database Tables
-
-### **1. Users Table (`users`)**
-
-**Purpose**: Core user authentication and profile management
-
-| Column | Type | Constraints | Description |
-|--------|------|-------------|-------------|
-| `user_id` | `INTEGER` | PRIMARY KEY, AUTO INCREMENT | Unique user identifier |
-| `username` | `STRING` | UNIQUE, NOT NULL | User's display name |
-| `email` | `STRING` | UNIQUE, NOT NULL | User's email address |
-| `created_at` | `DATETIME` | DEFAULT: UTC NOW | Account creation timestamp |
-
-**Relationships:**
-- **One-to-Many**: `chats` (User → Chat sessions)
-- **One-to-Many**: `usage_records` (User → Model usage tracking)
-- **One-to-Many**: `deep_analysis_reports` (User → Analysis reports)
-- **One-to-Many**: `template_preferences` (User → Agent preferences)
-
----
-
-### **2. Chats Table (`chats`)**
-
-**Purpose**: Conversation sessions and chat organization
-
-| Column | Type | Constraints | Description |
-|--------|------|-------------|-------------|
-| `chat_id` | `INTEGER` | PRIMARY KEY, AUTO INCREMENT | Unique chat session identifier |
-| `user_id` | `INTEGER` | FOREIGN KEY → `users.user_id`, CASCADE DELETE | Chat owner (nullable for anonymous) |
-| `title` | `STRING` | DEFAULT: 'New Chat' | Human-readable chat title |
-| `created_at` | `DATETIME` | DEFAULT: UTC NOW | Chat creation timestamp |
-
-**Relationships:**
-- **Many-to-One**: `user` (Chat → User)
-- **One-to-Many**: `messages` (Chat → Messages)
-- **One-to-Many**: `usage_records` (Chat → Model usage)
-
----
-
-### **3. Messages Table (`messages`)**
-
-**Purpose**: Individual messages within chat conversations
-
-| Column | Type | Constraints | Description |
-|--------|------|-------------|-------------|
-| `message_id` | `INTEGER` | PRIMARY KEY, AUTO INCREMENT | Unique message identifier |
-| `chat_id` | `INTEGER` | FOREIGN KEY → `chats.chat_id`, CASCADE DELETE | Parent chat session |
-| `sender` | `STRING` | NOT NULL | Message sender: 'user' or 'ai' |
-| `content` | `TEXT` | NOT NULL | Message content (text/markdown) |
-| `timestamp` | `DATETIME` | DEFAULT: UTC NOW | Message creation time |
-
-**Relationships:**
-- **Many-to-One**: `chat` (Message → Chat)
-- **One-to-One**: `feedback` (Message → Feedback)
-
----
-
-### **4. Model Usage Table (`model_usage`)**
-
-**Purpose**: AI model usage tracking for analytics and billing
-
-| Column | Type | Constraints | Description |
-|--------|------|-------------|-------------|
-| `usage_id` | `INTEGER` | PRIMARY KEY | Unique usage record identifier |
-| `user_id` | `INTEGER` | FOREIGN KEY → `users.user_id`, SET NULL | User who triggered the usage |
-| `chat_id` | `INTEGER` | FOREIGN KEY → `chats.chat_id`, SET NULL | Associated chat session |
-| `model_name` | `STRING(100)` | NOT NULL | AI model used (e.g., 'gpt-4o-mini') |
-| `provider` | `STRING(50)` | NOT NULL | Model provider ('openai', 'anthropic', etc.) |
-| `prompt_tokens` | `INTEGER` | DEFAULT: 0 | Input tokens consumed |
-| `completion_tokens` | `INTEGER` | DEFAULT: 0 | Output tokens generated |
-| `total_tokens` | `INTEGER` | DEFAULT: 0 | Total tokens (input + output) |
-| `query_size` | `INTEGER` | DEFAULT: 0 | Query size in characters |
-| `response_size` | `INTEGER` | DEFAULT: 0 | Response size in characters |
-| `cost` | `FLOAT` | DEFAULT: 0.0 | Cost in USD for this usage |
-| `timestamp` | `DATETIME` | DEFAULT: UTC NOW | Usage timestamp |
-| `is_streaming` | `BOOLEAN` | DEFAULT: FALSE | Whether response was streamed |
-| `request_time_ms` | `INTEGER` | DEFAULT: 0 | Request processing time (milliseconds) |
-
-**Relationships:**
-- **Many-to-One**: `user` (Usage → User)
-- **Many-to-One**: `chat` (Usage → Chat)
-
----
-
-### **5. Code Executions Table (`code_executions`)**
-
-**Purpose**: Track code generation and execution attempts
-
-| Column | Type | Constraints | Description |
-|--------|------|-------------|-------------|
-| `execution_id` | `INTEGER` | PRIMARY KEY, AUTO INCREMENT | Unique execution identifier |
-| `message_id` | `INTEGER` | FOREIGN KEY → `messages.message_id`, CASCADE DELETE | Associated message |
-| `chat_id` | `INTEGER` | FOREIGN KEY → `chats.chat_id`, CASCADE DELETE | Parent chat session |
-| `user_id` | `INTEGER` | FOREIGN KEY → `users.user_id`, SET NULL | User who triggered execution |
-| `initial_code` | `TEXT` | NULLABLE | First version of generated code |
-| `latest_code` | `TEXT` | NULLABLE | Most recent code version |
-| `is_successful` | `BOOLEAN` | DEFAULT: FALSE | Whether execution succeeded |
-| `output` | `TEXT` | NULLABLE | Execution output (including errors) |
-| `model_provider` | `STRING(50)` | NULLABLE | AI model provider used |
-| `model_name` | `STRING(100)` | NULLABLE | AI model name used |
-| `failed_agents` | `TEXT` | NULLABLE | JSON list of failed agent names |
-| `error_messages` | `TEXT` | NULLABLE | JSON map of error messages by agent |
-| `created_at` | `DATETIME` | DEFAULT: UTC NOW | Execution creation time |
-| `updated_at` | `DATETIME` | DEFAULT: UTC NOW, ON UPDATE | Last update timestamp |
-
----
-
-### **6. Message Feedback Table (`message_feedback`)**
-
-**Purpose**: User feedback and model settings for messages
-
-| Column | Type | Constraints | Description |
-|--------|------|-------------|-------------|
-| `feedback_id` | `INTEGER` | PRIMARY KEY, AUTO INCREMENT | Unique feedback identifier |
-| `message_id` | `INTEGER` | FOREIGN KEY → `messages.message_id`, CASCADE DELETE | Associated message |
-| `rating` | `INTEGER` | NULLABLE | Star rating (1-5 scale) |
-| `model_name` | `STRING(100)` | NULLABLE | Model used for this message |
-| `model_provider` | `STRING(50)` | NULLABLE | Model provider used |
-| `temperature` | `FLOAT` | NULLABLE | Temperature setting used |
-| `max_tokens` | `INTEGER` | NULLABLE | Max tokens setting used |
-| `created_at` | `DATETIME` | DEFAULT: UTC NOW | Feedback creation time |
-| `updated_at` | `DATETIME` | DEFAULT: UTC NOW, ON UPDATE | Last update timestamp |
-
-**Relationships:**
-- **One-to-One**: `message` (Feedback ↔ Message)
-
----
-
-### **7. Deep Analysis Reports Table (`deep_analysis_reports`)**
-
-**Purpose**: Store comprehensive multi-agent analysis reports
-
-| Column | Type | Constraints | Description |
-|--------|------|-------------|-------------|
-| `report_id` | `INTEGER` | PRIMARY KEY, AUTO INCREMENT | Unique report identifier |
-| `report_uuid` | `STRING(100)` | UNIQUE, NOT NULL | Frontend-generated UUID |
-| `user_id` | `INTEGER` | FOREIGN KEY → `users.user_id`, CASCADE DELETE | Report owner |
-| `goal` | `TEXT` | NOT NULL | Analysis objective/question |
-| `status` | `STRING(20)` | NOT NULL, DEFAULT: 'pending' | Status: 'pending', 'running', 'completed', 'failed' |
-| `start_time` | `DATETIME` | DEFAULT: UTC NOW | Analysis start time |
-| `end_time` | `DATETIME` | NULLABLE | Analysis completion time |
-| `duration_seconds` | `INTEGER` | NULLABLE | Total analysis duration |
-| `deep_questions` | `TEXT` | NULLABLE | Generated analytical questions |
-| `deep_plan` | `TEXT` | NULLABLE | Analysis execution plan |
-| `summaries` | `JSON` | NULLABLE | Array of analysis summaries |
-| `analysis_code` | `TEXT` | NULLABLE | Generated Python code |
-| `plotly_figures` | `JSON` | NULLABLE | Array of Plotly figure data |
-| `synthesis` | `JSON` | NULLABLE | Array of synthesis insights |
-| `final_conclusion` | `TEXT` | NULLABLE | Final analysis conclusion |
-| `html_report` | `TEXT` | NULLABLE | Complete HTML report |
-| `progress_percentage` | `INTEGER` | DEFAULT: 0 | Progress percentage (0-100) |
-| `total_tokens_used` | `INTEGER` | DEFAULT: 0 | Total tokens consumed |
-| `estimated_cost` | `FLOAT` | DEFAULT: 0.0 | Estimated cost in USD |
-| `credits_consumed` | `INTEGER` | DEFAULT: 0 | Credits deducted for analysis |
-| `created_at` | `DATETIME` | DEFAULT: UTC NOW | Report creation time |
-| `updated_at` | `DATETIME` | DEFAULT: UTC NOW, ON UPDATE | Last update timestamp |
-
-**Relationships:**
-- **Many-to-One**: `user` (Report → User)
-
----
-
-### **8. Agent Templates Table (`agent_templates`)**
-
-**Purpose**: Store predefined AI agent configurations
-
-| Column | Type | Constraints | Description |
-|--------|------|-------------|-------------|
-| `template_id` | `INTEGER` | PRIMARY KEY, AUTO INCREMENT | Unique template identifier |
-| `template_name` | `STRING(100)` | UNIQUE, NOT NULL | Internal template name |
-| `display_name` | `STRING(200)` | NULLABLE | User-friendly display name |
-| `description` | `TEXT` | NOT NULL | Template description |
-| `prompt_template` | `TEXT` | NOT NULL | Agent behavior instructions |
-| `icon_url` | `STRING(500)` | NULLABLE | Template icon URL |
-| `category` | `STRING(50)` | NULLABLE | Template category |
-| `is_premium_only` | `BOOLEAN` | DEFAULT: FALSE | Requires premium subscription |
-| `variant_type` | `STRING(20)` | DEFAULT: 'individual' | 'planner', 'individual', or 'both' |
-| `is_active` | `BOOLEAN` | DEFAULT: TRUE | Template is active/available |
-| `created_at` | `DATETIME` | DEFAULT: UTC NOW | Template creation time |
-
-**Relationships:**
-- **One-to-Many**: `user_preferences` (Template → User preferences)
-
----
-
-### **9. User Template Preferences Table (`user_template_preferences`)**
-
-**Purpose**: Track user preferences and usage for agent templates
-
-| Column | Type | Constraints | Description |
-|--------|------|-------------|-------------|
-| `preference_id` | `INTEGER` | PRIMARY KEY, AUTO INCREMENT | Unique preference identifier |
-| `user_id` | `INTEGER` | FOREIGN KEY → `users.user_id`, CASCADE DELETE | User who owns preference |
-| `template_id` | `INTEGER` | FOREIGN KEY → `agent_templates.template_id`, CASCADE DELETE | Associated template |
-| `is_enabled` | `BOOLEAN` | DEFAULT: TRUE | Whether user has template enabled |
-| `usage_count` | `INTEGER` | DEFAULT: 0 | Number of times user used template |
-| `last_used_at` | `DATETIME` | NULLABLE | Last time user used template |
-| `created_at` | `DATETIME` | DEFAULT: UTC NOW | Preference creation time |
-
-**Relationships:**
-- **Many-to-One**: `user` (Preference → User)
-- **Many-to-One**: `template` (Preference → Template)
-
-**Constraints:**
-- **Unique**: `(user_id, template_id)` - One preference per user per template
-
----
-
-## 🔗 Entity Relationship Diagram
-
-```
-Users (1) ──────────── (Many) Chats
-  │                           │
-  │                           ├── (Many) Messages
-  │                           │      │
-  │                           │      └── (1) MessageFeedback
-  │                           │
-  │                           └── (Many) CodeExecutions
-  │
-  ├── (Many) ModelUsage
-  │
-  ├── (Many) DeepAnalysisReports
-  │
-  └── (Many) UserTemplatePreferences
-               │
-               └── (Many) AgentTemplates
-```
-
----
-
-## 📊 Database Performance
-
-### **Optimized Indexes**
-
-```sql
--- High-performance queries
-CREATE INDEX idx_messages_chat_timestamp ON messages(chat_id, timestamp DESC);
-CREATE INDEX idx_model_usage_user_time ON model_usage(user_id, timestamp DESC);
-CREATE INDEX idx_model_usage_model_time ON model_usage(model_name, timestamp DESC);
-CREATE INDEX idx_reports_user_time ON deep_analysis_reports(user_id, created_at DESC);
-```
-
-### **Cascade Deletion Rules**
-
-| Parent → Child | Rule | Description |
-|----------------|------|-------------|
-| `users` → `chats` | CASCADE | Delete all user chats when user deleted |
-| `chats` → `messages` | CASCADE | Delete all chat messages when chat deleted |
-| `messages` → `feedback` | CASCADE | Delete feedback when message deleted |
-| `users` → `model_usage` | SET NULL | Keep usage records for analytics |
-
----
-
-## 🛡️ Security & Maintenance
-
-### **Data Protection**
-- User data isolated by `user_id`
-- Sensitive fields require encryption in production
-- Automatic cleanup of anonymous data after 90 days
-
-### **Regular Maintenance**
-```sql
--- Clean old anonymous chats
-DELETE FROM chats WHERE user_id IS NULL AND created_at < DATE_SUB(NOW(), INTERVAL 90 DAY);
-
--- Update statistics for query optimization
-ANALYZE users, chats, messages, model_usage;
-```
-
----
-
-This schema supports the full Auto-Analyst application with optimized performance, data integrity, and scalability for both development and production environments. 

docs/troubleshooting/troubleshooting.md

@@ -1,537 +0,0 @@
-# Auto-Analyst Backend Troubleshooting Guide
-
-## 🚨 Common Startup Issues
-
-### 1. **Database Connection Problems**
-
-#### Problem: Database connection failed
-```
-❌ sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) no such table: users
-```
-
-**Solutions:**
-1. **Initialize Database**:
-   ```bash
-   python -c "
-   from src.db.init_db import init_db
-   init_db()
-   print('✅ Database initialized')
-   "
-   ```
-
-2. **Check Database File Permissions**:
-   ```bash
-   # For SQLite
-   ls -la auto_analyst.db
-   chmod 666 auto_analyst.db  # If needed
-   ```
-
-3. **Verify DATABASE_URL**:
-   ```bash
-   # Check .env file
-   cat .env | grep DATABASE_URL
-   
-   # For PostgreSQL (production)
-   DATABASE_URL=postgresql://user:password@host:port/database
-   
-   # For SQLite (development)  
-   DATABASE_URL=sqlite:///./auto_analyst.db
-   ```
-
-#### Problem: PostgreSQL connection issues
-```
-❌ psycopg2.OperationalError: FATAL: database "auto_analyst" does not exist
-```
-
-**Solutions:**
-1. **Create Database**:
-   ```sql
-   -- Connect to PostgreSQL
-   psql -h localhost -U postgres
-   CREATE DATABASE auto_analyst;
-   \q
-   ```
-
-2. **Update Connection String**:
-   ```env
-   DATABASE_URL=postgresql://username:password@localhost:5432/auto_analyst
-   ```
-
-### 2. **Agent Template Loading Issues**
-
-#### Problem: No agents found
-```
-❌ RuntimeError: No agents loaded for user. Cannot proceed with analysis.
-```
-
-**Solutions:**
-1. **Initialize Default Agents**:
-   ```python
-   python -m scripts.populate_agent_templates
-   print('✅ Default agents initialized')
-   "
-   ```
-
-2. **Check Agent Templates in Database**:
-   ```python
-   python -c "
-   from src.db.init_db import session_factory
-   from src.db.schemas.models import AgentTemplate
-   session = session_factory()
-   templates = session.query(AgentTemplate).all()
-   print(f'Found {len(templates)} templates:')
-   for t in templates:
-       print(f'  - {t.template_name}: {t.is_active}')
-   session.close()
-   "
-   ```
-
-3. **Populate Templates from Config**:
-   ```bash
-   python scripts/populate_agent_templates.py
-   ```
-
-### 3. **API Key Issues**
-
-#### Problem: Missing API keys
-```
-❌ AuthenticationError: Invalid API key provided
-```
-
-**Solutions:**
-1. **Check Environment Variables**:
-   ```bash
-   # Verify API keys are set
-   echo $ANTHROPIC_API_KEY
-   echo $OPENAI_API_KEY
-   
-   # Or check .env file
-   cat .env | grep API_KEY
-   ```
-
-2. **Add Missing Keys**:
-   ```env
-   # Add to .env file
-   ANTHROPIC_API_KEY=sk-ant-api03-...
-   OPENAI_API_KEY=sk-...
-   ADMIN_API_KEY=your_admin_key_here
-   ```
-
-3. **Test API Key Validity**:
-   ```python
-   python -c "
-   import os
-   from anthropic import Anthropic
-   client = Anthropic(api_key=os.getenv('ANTHROPIC_API_KEY'))
-   try:
-       # Test call
-       response = client.messages.create(
-           model='claude-3-sonnet-20241022',
-           max_tokens=10,
-           messages=[{'role': 'user', 'content': 'Hello'}]
-       )
-       print('✅ Anthropic API key valid')
-   except Exception as e:
-       print(f'❌ Anthropic API key invalid: {e}')
-   "
-   ```
-
-## 🤖 Agent System Issues
-
-### 1. **Agent Not Found Errors**
-
-#### Problem: Specific agent not available
-```
-❌ KeyError: 'custom_agent' not found in loaded agents
-```
-
-**Solutions:**
-1. **Check Available Agents**:
-   ```python
-   python -c "
-   from src.agents.agents import load_user_enabled_templates_from_db
-   from src.db.init_db import session_factory
-   session = session_factory()
-   agents = load_user_enabled_templates_from_db('test_user', session)
-   print('Available agents:', list(agents.keys()))
-   session.close()
-   "
-   ```
-
-2. **Verify Agent Template Exists**:
-   ```python
-   python -c "
-   from src.db.init_db import session_factory
-   from src.db.schemas.models import AgentTemplate
-   session = session_factory()
-   agent = session.query(AgentTemplate).filter_by(template_name='custom_agent').first()
-   if agent:
-       print(f'Agent found: {agent.display_name}, Active: {agent.is_active}')
-   else:
-       print('Agent not found in database')
-   session.close()
-   "
-   ```
-
-3. **Add Missing Agent Template**:
-   ```python
-   # Add to agents_config.json or use database insertion
-   python scripts/populate_agent_templates.py
-   ```
-
-### 2. **Deep Analysis Failures**
-
-#### Problem: Deep analysis stops unexpectedly
-```
-❌ DeepAnalysisError: Agent execution failed at step 3
-```
-
-**Solutions:**
-1. **Check Agent Configuration**:
-   ```python
-   # Verify user has required agents enabled
-   python -c "
-   from src.agents.deep_agents import get_user_enabled_agent_names
-   from src.db.init_db import session_factory
-   session = session_factory()
-   agents = get_user_enabled_agent_names('test_user', session)
-   required = ['preprocessing_agent', 'statistical_analytics_agent', 'sk_learn_agent', 'data_viz_agent']
-   print('Required agents:', required)
-   print('Available agents:', agents)
-   print('Missing:', [a for a in required if a not in agents])
-   session.close()
-   "
-   ```
-
-2. **Increase Timeout Settings**:
-   ```python
-   # In deep_agents.py, increase timeout values
-   timeout = 300  # Increase from default
-   ```
-
-3. **Check Dataset Size**:
-   ```python
-   # Reduce dataset size for complex analysis
-   df_sample = df.sample(n=1000)  # Use sample for testing
-   ```
-
-## ⚡ Code Execution Problems
-
-### 1. **Code Execution Timeouts**
-
-#### Problem: Code execution takes too long
-```
-❌ TimeoutError: Code execution exceeded 120 seconds
-```
-
-**Solutions:**
-1. **Optimize Generated Code**:
-   - Use data sampling for large datasets
-   - Simplify analysis requirements
-   - Use sampling for large datasets
-
-2. **Check Resource Usage**:
-   ```python
-   import psutil
-   print(f"Memory usage: {psutil.virtual_memory().percent}%")
-   print(f"CPU usage: {psutil.cpu_percent()}%")
-   ```
-
-3. **Increase Timeout Settings**:
-   ```python
-   # In clean_and_store_code function
-   future.result(timeout=600)  # Increase timeout to 10 minutes
-   ```
-
-#### Problem: Import Errors in Generated Code
-```
-❌ ModuleNotFoundError: No module named 'some_library'
-```
-
-**Solutions:**
-1. **Check Available Libraries**:
-   ```python
-   # Available in execution environment:
-   import pandas as pd
-   import numpy as np
-   import plotly.express as px
-   import plotly.graph_objects as go
-   import sklearn
-   import statsmodels.api as sm
-   ```
-
-2. **Add Missing Dependencies**:
-   ```bash
-   pip install missing_library
-   ```
-
-3. **Update Execution Environment**:
-   ```python
-   # In clean_and_store_code function
-   exec_globals.update({
-       'new_library': __import__('new_library')
-   })
-   ```
-
-### 4. **Database Issues**
-
-#### Problem: Migration Errors
-```
-❌ alembic.util.exc.CommandError: Can't locate revision identified by 'xyz'
-```
-
-**Solutions:**
-1. **Reset Migration History**:
-   ```bash
-   # Delete migration files (except __init__.py)
-   rm migrations/versions/*.py
-   
-   # Create new initial migration
-   alembic revision --autogenerate -m "initial migration"
-   alembic upgrade head
-   ```
-
-2. **Force Migration**:
-   ```bash
-   # Mark current state as up-to-date
-   alembic stamp head
-   ```
-
-3. **Recreate Database**:
-   ```bash
-   # For SQLite (development)
-   rm auto_analyst.db
-   python -c "from src.db.init_db import init_db; init_db()"
-   ```
-
-#### Problem: Constraint Violations
-```
-❌ IntegrityError: UNIQUE constraint failed
-```
-
-**Solutions:**
-1. **Check Existing Records**:
-   ```python
-   from src.db.init_db import session_factory
-   from src.db.schemas.models import AgentTemplate
-   
-   session = session_factory()
-   templates = session.query(AgentTemplate).all()
-   for t in templates:
-       print(f"{t.template_name}: {t.template_id}")
-   session.close()
-   ```
-
-2. **Clean Duplicate Data**:
-   ```bash
-   python -c "
-   from src.db.init_db import session_factory
-   from src.db.schemas.models import AgentTemplate
-   session = session_factory()
-   # Remove duplicates based on template_name
-   seen = set()
-   for template in session.query(AgentTemplate).all():
-       if template.template_name in seen:
-           session.delete(template)
-       else:
-           seen.add(template.template_name)
-   session.commit()
-   session.close()
-   "
-   ```
-
-### 5. **Authentication and Authorization Issues**
-
-#### Problem: Unauthorized Access
-```
-❌ 401 Unauthorized: Invalid session
-```
-
-**Solutions:**
-1. **Check Session ID**:
-   ```python
-   # Ensure session_id is provided in request
-   headers = {"X-Session-ID": "your_session_id"}
-   # Or as query parameter: ?session_id=your_session_id
-   ```
-
-2. **Create Valid Session**:
-   ```bash
-   curl -X POST "http://localhost:8000/session_info" \
-        -H "Content-Type: application/json"
-   ```
-
-3. **Verify Admin API Key**:
-   ```bash
-   curl -X GET "http://localhost:8000/analytics/usage" \
-        -H "X-API-Key: your_admin_key"
-   ```
-
-### 6. **Performance Issues**
-
-#### Problem: Slow Response Times
-```
-⚠️ Request taking longer than expected
-```
-
-**Solutions:**
-1. **Enable Database Connection Pooling**:
-   ```python
-   # In init_db.py
-   engine = create_engine(
-       DATABASE_URL,
-       poolclass=QueuePool,
-       pool_size=10,
-       max_overflow=20
-   )
-   ```
-
-2. **Optimize Database Queries**:
-   ```python
-   # Use eager loading for relationships
-   session.query(User).options(joinedload(User.chats)).all()
-   ```
-
-3. **Add Response Caching**:
-   ```python
-   # Use local caching for expensive operations
-   @lru_cache(maxsize=100)
-   def expensive_operation(data):
-       return result
-   ```
-
-#### Problem: Memory Usage High
-```
-⚠️ Memory usage above 80%
-```
-
-**Solutions:**
-1. **Optimize DataFrame Operations**:
-   ```python
-   # Use chunking for large datasets
-   for chunk in pd.read_csv('file.csv', chunksize=1000):
-       process_chunk(chunk)
-   ```
-
-2. **Clear Unused Variables**:
-   ```python
-   # In code execution
-   del large_dataframe
-   import gc
-   gc.collect()
-   ```
-
-3. **Monitor Memory Usage**:
-   ```python
-   import psutil
-   import logging
-   
-   memory_percent = psutil.virtual_memory().percent
-   if memory_percent > 80:
-       logging.warning(f"High memory usage: {memory_percent}%")
-   ```
-
-## 🔧 Debugging Tools and Commands
-
-### Health Check Commands
-
-```bash
-# Test basic connectivity
-curl http://localhost:8000/health
-
-# Check database status
-python -c "
-from src.db.init_db import session_factory
-try:
-    session = session_factory()
-    session.execute('SELECT 1')
-    print('✅ Database connection OK')
-    session.close()
-except Exception as e:
-    print(f'❌ Database error: {e}')
-"
-
-# Verify agent templates
-python -c "
-from src.db.init_db import session_factory
-from src.db.schemas.models import AgentTemplate
-session = session_factory()
-count = session.query(AgentTemplate).count()
-print(f'Agent templates in database: {count}')
-session.close()
-"
-```
-
-### Performance Monitoring
-
-```python
-# Memory and CPU monitoring
-import psutil
-import time
-
-def monitor_system():
-    while True:
-        cpu = psutil.cpu_percent(interval=1)
-        memory = psutil.virtual_memory()
-        print(f"CPU: {cpu}% | Memory: {memory.percent}% | Available: {memory.available // 1024 // 1024}MB")
-        time.sleep(5)
-
-# Run monitoring
-monitor_system()
-```
-
-### Database Inspection
-
-```python
-# Inspect database tables
-from src.db.init_db import session_factory
-from src.db.schemas.models import *
-
-session = session_factory()
-
-# Count records in each table
-tables = [User, Chat, Message, AgentTemplate, UserTemplatePreference, DeepAnalysisReport]
-for table in tables:
-    count = session.query(table).count()
-    print(f"{table.__name__}: {count} records")
-
-session.close()
-```
-
-### Log Analysis
-
-```bash
-# View recent logs
-tail -f logs/app.log
-
-# Search for errors
-grep "ERROR" logs/app.log | tail -20
-
-# Search for specific issues
-grep -i "agent" logs/app.log | grep -i "error"
-```
-
-## 🚀 Performance Optimization Tips
-
-### Database Optimization
-
-1. **Use Indexes**: Ensure frequently queried columns have indexes
-2. **Query Optimization**: Use `joinedload` for relationships
-3. **Connection Pooling**: Configure appropriate pool sizes
-4. **Batch Operations**: Use bulk operations for multiple records
-
-### Agent Performance
-
-1. **Async Execution**: Use async patterns for concurrent operations
-2. **Result Caching**: Cache expensive computations
-3. **Memory Management**: Clean up large objects after use
-4. **Code Optimization**: Simplify generated code for better performance
-
-### System Monitoring
-
-1. **Resource Tracking**: Monitor CPU, memory, and disk usage
-2. **Error Monitoring**: Set up alerting for critical errors
-3. **Performance Metrics**: Track response times and throughput
-4. **Usage Analytics**: Monitor feature usage and optimization opportunities
-
-This troubleshooting guide covers the most common issues you'll encounter with the Auto-Analyst backend. For additional help, check the system logs and use the debugging tools provided. 

requirements.txt

@@ -1,31 +1,43 @@
 aiofiles==24.1.0
 beautifulsoup4==4.13.4
-chardet==5.2.0
-dspy==3.2.1
-litellm==1.82.3
+dspy==2.6.14
 email_validator==2.2.0
-fastapi==0.135.1
+fastapi==0.111.1
 fastapi-cli==0.0.7
 FastAPI-SQLAlchemy==0.2.1
-fastapi-sso==0.16.0
+fastapi-sso==0.10.0
 groq==0.18.0
-gunicorn==23.0.0
+gunicorn==22.0.0
 huggingface-hub==0.30.2
 joblib==1.4.2
+litellm==1.63.7
+llama-cloud==0.1.19
+llama-cloud-services==0.6.21
+llama-index==0.12.14
+llama-index-agent-openai==0.4.2
+llama-index-cli==0.4.1
+llama-index-core==0.12.34.post1
+llama-index-embeddings-openai==0.3.1
+llama-index-indices-managed-llama-cloud==0.6.4
+llama-index-llms-openai==0.3.14
+llama-index-multi-modal-llms-openai==0.4.2
+llama-index-program-openai==0.3.1
+llama-index-question-gen-openai==0.3.0
 Markdown==3.7
 matplotlib==3.10.0
 matplotlib-inline==0.1.7
 numpy==2.2.2
 openpyxl==3.1.2
 xlrd==2.0.1
-openai==2.28.0
+openai==1.61.0
 pandas==2.2.3
-polars==1.31.0
+polars==1.30.0
 pillow==11.1.0
 plotly==5.24.1
 psycopg2==2.9.10
 python-dateutil==2.9.0.post0
 python-dotenv==1.0.1
+python-multipart==0.0.9
 requests==2.32.3
 scikit-learn==1.6.1
 scipy==1.15.1
@@ -39,8 +51,8 @@ tiktoken==0.8.0
 tokenizers==0.21.0
 tqdm==4.67.1
 urllib3==2.4.0
-uvicorn==0.34.0
-websockets>=13.1.0
+uvicorn==0.22.0
+websockets==14.2
 wheel==0.45.1
 xgboost-cpu==3.0.2
 bokeh==3.7.3
@@ -48,5 +60,4 @@ pymc==5.23.0
 lightgbm==4.6.0
 arviz==0.21.0
 optuna==4.3.0
-litellm[proxy]
-duckdb==1.3.2
+shap==0.45.1

scripts/create_test_user.py

@@ -0,0 +1,45 @@
+import sys
+import os
+
+# Add parent directory to path so we can import modules
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from init_db import User, session_factory
+
+def create_test_users():
+    """Create test users for development"""
+    session = session_factory()
+    
+    try:
+        # Create a few test users
+        test_users = [
+            {"username": "testuser1", "email": "user1@example.com"},
+            {"username": "testuser2", "email": "user2@example.com"},
+            {"username": "admin", "email": "admin@example.com"},
+            {"username": "demo", "email": "demo@example.com"},
+            {"username": "guest", "email": "guest@example.com"},
+        ]
+        
+        for user_data in test_users:
+            # Check if user already exists
+            existing = session.query(User).filter(User.email == user_data["email"]).first()
+            if not existing:
+                user = User(**user_data)
+                session.add(user)
+                print(f"Created user: {user_data['username']}")
+            else:
+                print(f"User {user_data['username']} already exists")
+        
+        session.commit()
+        print("Test users created successfully")
+        
+    except Exception as e:
+        session.rollback()
+        print(f"Error creating test users: {e}")
+    
+    finally:
+        session.close()
+
+if __name__ == "__main__":
+    create_test_users()
+    print("Done!") 

scripts/format_response.py

@@ -8,7 +8,7 @@ import logging
 from src.utils.logger import Logger
 import textwrap
 
-logger = Logger(__name__, level=logging.INFO, see_time=False, console_log=False)
+logger = Logger(__name__, level="INFO", see_time=False, console_log=False)
 
 @contextlib.contextmanager
 def stdoutIO(stdout=None):
@@ -44,33 +44,20 @@ API_KEY_PATTERNS = [
 ]
 
 # Network request patterns
-NETWORK_REQUEST_PATTERNS = re.compile(r"(requests\.|urllib\.|http\.client|httpx\.|socket\.connect\()")
+NETWORK_REQUEST_PATTERNS = re.compile(r"(requests\.|urllib\.|http\.|\.post\(|\.get\(|\.connect\()")
 
-# DataFrame creation with hardcoded data - block only this specific pattern
-
-def check_security_concerns(code_str, dataset_names):
+def check_security_concerns(code_str):
     """Check code for security concerns and return info about what was found"""
     security_concerns = {
         "has_concern": False,
+        "messages": [],
         "blocked_imports": False,
         "blocked_dynamic_imports": False,
         "blocked_env_access": False,
         "blocked_file_access": False,
         "blocked_api_keys": False,
-        "blocked_network": False,
-        "blocked_dataframe_invention": False,  # Add this new field
-        "messages": []
+        "blocked_network": False
     }
-
-    dataset_names_pattern = "|".join(re.escape(name) for name in dataset_names)
-    DATAFRAME_INVENTION_PATTERN = re.compile(
-                rf"({dataset_names_pattern})\s*=\s*pd\.DataFrame\s*\(\s*\{{\s*[^}}]*\}}", 
-                re.MULTILINE
-            )
-    if DATAFRAME_INVENTION_PATTERN.search(code_str):
-        security_concerns["has_concern"] = True
-        security_concerns["blocked_dataframe_invention"] = True
-        security_concerns["messages"].append(f"DataFrame creation blocked for dataset variables: {', '.join(dataset_names)}")
     
     # Check for sensitive imports
     if IMPORT_PATTERN.search(code_str) or FROM_IMPORT_PATTERN.search(code_str):
@@ -110,19 +97,11 @@ def check_security_concerns(code_str, dataset_names):
         security_concerns["blocked_network"] = True
         security_concerns["messages"].append("Network requests blocked")
     
-    
-    
     return security_concerns
 
-def clean_code_for_security(code_str, security_concerns, dataset_names):
+def clean_code_for_security(code_str, security_concerns):
     """Apply security modifications to the code based on detected concerns"""
-
     modified_code = code_str
-    dataset_names_pattern = "|".join(re.escape(name) for name in dataset_names)
-    DATAFRAME_INVENTION_PATTERN = re.compile(
-                rf"({dataset_names_pattern})\s*=\s*pd\.DataFrame\s*\(\s*\{{\s*[^}}]*\}}", 
-                re.MULTILINE
-            )
     
     # Block sensitive imports if needed
     if security_concerns["blocked_imports"]:
@@ -150,13 +129,6 @@ def clean_code_for_security(code_str, security_concerns, dataset_names):
     if security_concerns["blocked_network"]:
         modified_code = NETWORK_REQUEST_PATTERNS.sub(r'"BLOCKED_NETWORK_REQUEST"', modified_code)
     
-    # Block DataFrame creation with hardcoded data if detected
-    if security_concerns["blocked_dataframe_invention"]:
-        modified_code = DATAFRAME_INVENTION_PATTERN.sub(
-            r"# BLOCKED_DATAFRAME_INVENTION: \g<0>", 
-            modified_code
-        )
-    
     # Add warning banner if needed
     if security_concerns["has_concern"]:
         security_message = "⚠️ SECURITY WARNING: " + ". ".join(security_concerns["messages"]) + "."
@@ -304,14 +276,6 @@ def format_code_block(code_str):
     return f'\n{code_clean}\n'
 
 def format_code_backticked_block(code_str):
-    # Add None check at the beginning
-    if code_str is None:
-        return "```python\n# No code available\n```"  # Return empty code block instead of None
-    
-    # Add type check to ensure it's a string
-    if not isinstance(code_str, str):
-        return f"```python\n# Invalid code type: {type(code_str)}\n```"
-    
     code_clean = re.sub(r'^```python\n?', '', code_str, flags=re.MULTILINE)
     code_clean = re.sub(r'\n```$', '', code_clean)
     # Only match assignments at top level (not indented)
@@ -320,7 +284,6 @@ def format_code_backticked_block(code_str):
   
     # Remove reading the csv file if it's already in the context
     modified_code = re.sub(r"df\s*=\s*pd\.read_csv\([\"\'].*?[\"\']\).*?(\n|$)", '', code_clean)
-    modified_code = re.sub(r'^(\s*)(df\s*=.*)$', r'\1# \2', code_clean, flags=re.MULTILINE)
     
     # Only match assignments at top level (not indented)
     # 1. Remove 'df = pd.DataFrame()' if it's at the top level
@@ -344,7 +307,7 @@ def format_code_backticked_block(code_str):
     return f'```python\n{code_clean}\n```'
 
     
-def execute_code_from_markdown(code_str, datasets=None):
+def execute_code_from_markdown(code_str, dataframe=None):
     import pandas as pd
     import plotly.express as px
     import plotly
@@ -358,12 +321,11 @@ def execute_code_from_markdown(code_str, datasets=None):
     from io import StringIO, BytesIO
     import base64
 
-    context_names = list(datasets.keys())
     # Check for security concerns in the code
-    security_concerns = check_security_concerns(code_str, context_names)
+    security_concerns = check_security_concerns(code_str)
     
     # Apply security modifications to the code
-    modified_code = clean_code_for_security(code_str, security_concerns, context_names)
+    modified_code = clean_code_for_security(code_str, security_concerns)
     
     # Enhanced print function that detects and formats tabular data
     captured_outputs = []
@@ -591,11 +553,8 @@ def execute_code_from_markdown(code_str, datasets=None):
     pd.DataFrame.__repr__ = custom_df_repr
 
     # If a dataframe is provided, add it to the context
-    for dataset_name, dataset_df in datasets.items():
-        if dataset_df is not None:
-            context[dataset_name] = dataset_df
-            logger.log_message(f"Added dataset '{dataset_name}' to execution context", level=logging.DEBUG)
-
+    if dataframe is not None:
+        context['df'] = dataframe
 
     # remove pd.read_csv() if it's already in the context
     modified_code = re.sub(r"pd\.read_csv\(\s*[\"\'].*?[\"\']\s*\)", '', modified_code)
@@ -628,7 +587,12 @@ def execute_code_from_markdown(code_str, datasets=None):
         modified_code = re.sub(pattern, add_show, modified_code)
     
     # Only add df = pd.read_csv() if no dataframe was provided and the code contains pd.read_csv
-
+    if dataframe is None and 'pd.read_csv' not in modified_code:
+        modified_code = re.sub(
+            r'import pandas as pd',
+            r'import pandas as pd\n\n# Read Housing.csv\ndf = pd.read_csv("Housing.csv")',
+            modified_code
+        )
 
     # Identify code blocks by comments
     code_blocks = []
@@ -659,12 +623,6 @@ def execute_code_from_markdown(code_str, datasets=None):
             # Clear captured outputs for each block
             captured_outputs.clear()
             
-            # Fix indentation issues before execution
-            try:
-                block_code = textwrap.dedent(block_code)
-            except Exception as dedent_error:
-                logger.log_message(f"Failed to dedent code block '{block_name}': {str(dedent_error)}", level=logging.WARNING)
-            
             with stdoutIO() as s:
                 exec(block_code, context)  # Execute the block
             
@@ -898,7 +856,7 @@ def format_plan_instructions(plan_instructions):
         else:
             raise TypeError(f"Unsupported plan instructions type: {type(plan_instructions)}")
     except Exception as e:
-        raise ValueError(f"Error processing plan instructions: {str(e)} + {dspy.settings.lm} ")
+        raise ValueError(f"Error processing plan instructions: {str(e)}")
     # logger.log_message(f"Plan instructions: {instructions}", level=logging.INFO)
 
 
@@ -942,13 +900,9 @@ def format_plan_instructions(plan_instructions):
 
     return "\n".join(markdown_lines).strip()
     
-    # Return empty string if no complexity found
-    return ""
 
 def format_complexity(instructions):
     markdown_lines = []
-    complexity = None  # Initialize complexity to avoid UnboundLocalError
-    
     # Extract complexity from various possible locations in the structure
     if isinstance(instructions, dict):
         # Case 1: Direct complexity field
@@ -960,12 +914,8 @@ def format_complexity(instructions):
                 complexity = instructions['plan']['complexity']
         else:
             complexity = "unrelated"
-            
-        # Override for basic_qa_agent cases
-        if 'plan' in instructions and isinstance(instructions['plan'], str) and "basic_qa_agent" in instructions['plan']:
-            complexity = "unrelated"
-    else:
-        # If instructions is not a dict, default to unrelated
+    
+    if 'plan' in instructions and isinstance(instructions['plan'], str) and "basic_qa_agent" in instructions['plan']:
         complexity = "unrelated"
     
     if complexity:
@@ -990,12 +940,10 @@ def format_complexity(instructions):
         # Slightly larger display with pink styling
         markdown_lines.append(f"<div style='color: {color}; border: 2px solid {color}; padding: 2px 8px; border-radius: 12px; display: inline-block; font-size: 14.4px;'>{indicator} {complexity}</div>\n")
 
-        return "\n".join(markdown_lines).strip()
-    
-    # Return empty string if no complexity found
-    return ""
+        return "\n".join(markdown_lines).strip()    
 
-def format_response_to_markdown(api_response, agent_name = None, datasets=None):
+
+def format_response_to_markdown(api_response, agent_name = None, dataframe=None):
     try:
         markdown = []
         # logger.log_message(f"API response for {agent_name} at {time.strftime('%Y-%m-%d %H:%M:%S')}: {api_response}", level=logging.INFO)
@@ -1021,34 +969,26 @@ def format_response_to_markdown(api_response, agent_name = None, datasets=None):
                 continue
                 
             if "complexity" in content:
-                complexity_result = format_complexity(content)
-                if complexity_result:  # Only append if not empty
-                    markdown.append(f"{complexity_result}")
+                markdown.append(f"{format_complexity(content)}\n")
                 
             markdown.append(f"\n## {agent.replace('_', ' ').title()}\n")
             
             if agent == "analytical_planner":
                 logger.log_message(f"Analytical planner content: {content}", level=logging.INFO)
-                if 'plan_desc' in content and content['plan_desc']:
-                    markdown.append(f"### Reasoning {content['plan_desc']}")
+                if 'plan_desc' in content:
+                    markdown.append(f"### Reasoning\n{content['plan_desc']}\n")
                 if 'plan_instructions' in content:
-                    plan_result = format_plan_instructions(content['plan_instructions'])
-                    if plan_result:  # Only append if not empty/None
-                        markdown.append(f"{plan_result}")
+                    markdown.append(f"{format_plan_instructions(content['plan_instructions'])}\n")
                 else:
-                    if content.get('rationale'):
-                        markdown.append(f"### Reasoning {content['rationale']}")
+                    markdown.append(f"### Reasoning\n{content['rationale']}\n")
             else:  
                 if "rationale" in content:
-                    if content.get('rationale'):
-                        markdown.append(f"### Reasoning{content['rationale']}")
+                    markdown.append(f"### Reasoning\n{content['rationale']}\n")
 
-            if 'code' in content and content['code'] is not None:
-                formatted_code = format_code_backticked_block(content['code'])
-                if formatted_code:  # Check if formatted_code is not None
-                    markdown.append(f"### Code Implementation\n{formatted_code}\n")
-            if 'answer' in content and content['answer']:
-                markdown.append(f"### Answer{content['answer']} Please ask a query about the data")
+            if 'code' in content:
+                markdown.append(f"### Code Implementation\n{format_code_backticked_block(content['code'])}\n")
+            if 'answer' in content:
+                markdown.append(f"### Answer\n{content['answer']}\n Please ask a query about the data")
             if 'summary' in content:
                 import re
                 summary_text = content['summary']
@@ -1084,11 +1024,11 @@ def format_response_to_markdown(api_response, agent_name = None, datasets=None):
                     if content['refined_complete_code'] is not None and content['refined_complete_code'] != "":
                         clean_code = format_code_block(content['refined_complete_code']) 
                         markdown_code = format_code_backticked_block(content['refined_complete_code'])
-                        output, json_outputs, matplotlib_outputs = execute_code_from_markdown(clean_code, datasets)
+                        output, json_outputs, matplotlib_outputs = execute_code_from_markdown(clean_code, dataframe)
                     elif "```python" in content['summary']:
                         clean_code = format_code_block(content['summary'])
                         markdown_code = format_code_backticked_block(content['summary'])
-                        output, json_outputs, matplotlib_outputs = execute_code_from_markdown(clean_code, datasets)
+                        output, json_outputs, matplotlib_outputs = execute_code_from_markdown(clean_code, dataframe)
                 except Exception as e:
                     logger.log_message(f"Error in execute_code_from_markdown: {str(e)}", level=logging.ERROR)
                     markdown_code = f"**Error**: {str(e)}"
@@ -1119,7 +1059,7 @@ def format_response_to_markdown(api_response, agent_name = None, datasets=None):
 
     except Exception as e:
         logger.log_message(f"Error in format_response_to_markdown: {str(e)}", level=logging.ERROR)
-        return f"error formating markdown {str(e)}"
+        return f"{str(e)}"
         
     # logger.log_message(f"Generated markdown content for agent '{agent_name}' at {time.strftime('%Y-%m-%d %H:%M:%S')}: {markdown}, length: {len(markdown)}", level=logging.INFO)
     
@@ -1130,8 +1070,34 @@ def format_response_to_markdown(api_response, agent_name = None, datasets=None):
             f"API Response: {api_response}",
             level=logging.ERROR
         )
-        return ""
+        return " "
         
     return '\n'.join(markdown)
 
 
+# Example usage with dummy data
+if __name__ == "__main__":
+    sample_response = {
+        "code_combiner_agent": {
+            "reasoning": "Sample reasoning for multiple charts.",
+            "refined_complete_code": """
+```python
+import plotly.express as px
+import pandas as pd
+
+# Sample Data
+df = pd.DataFrame({'Category': ['A', 'B', 'C'], 'Values': [10, 20, 30]})
+
+# First Chart
+fig = px.bar(df, x='Category', y='Values', title='Bar Chart')
+fig.show()
+
+# Second Chart
+fig2 = px.pie(df, values='Values', names='Category', title='Pie Chart')
+fig2.show()
+```
+"""
+        }
+    }
+
+    formatted_md = format_response_to_markdown(sample_response)

scripts/generate_test_data.py

@@ -0,0 +1,90 @@
+import sys
+import os
+import random
+from datetime import datetime, timedelta
+import sqlite3
+
+# Add parent directory to path so we can import modules
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from init_db import ModelUsage, session_factory
+
+# Models and providers to use in test data
+MODELS = {
+    "gpt-3.5-turbo": {"provider": "OpenAI", "cost_per_1k": 0.0015},
+    "gpt-4": {"provider": "OpenAI", "cost_per_1k": 0.03},
+    "gpt-4o": {"provider": "OpenAI", "cost_per_1k": 0.01},
+    "gpt-4o-mini": {"provider": "OpenAI", "cost_per_1k": 0.0015},
+    "o1-mini": {"provider": "OpenAI", "cost_per_1k": 0.00015},
+    "claude-3-opus": {"provider": "Anthropic", "cost_per_1k": 0.015},
+    "claude-3-sonnet": {"provider": "Anthropic", "cost_per_1k": 0.008},
+    "claude-3-haiku": {"provider": "Anthropic", "cost_per_1k": 0.003},
+    "llama-3-8b": {"provider": "Groq", "cost_per_1k": 0.0005},
+    "llama-3-70b": {"provider": "Groq", "cost_per_1k": 0.002},
+}
+
+# User IDs to use (can be random if you don't have specific users)
+USER_IDS = [1, 2, 3, 4, 5]
+
+def generate_test_data(num_records=100):
+    """Generate test model usage data"""
+    session = session_factory()
+    
+    try:
+        # Generate records for the past 30 days
+        end_date = datetime.utcnow()
+        start_date = end_date - timedelta(days=30)
+        
+        for _ in range(num_records):
+            # Random timestamp within the date range
+            random_days = random.randint(0, 30)
+            timestamp = end_date - timedelta(days=random_days, 
+                                           hours=random.randint(0, 23),
+                                           minutes=random.randint(0, 59))
+            
+            # Select random model and user
+            model_name = random.choice(list(MODELS.keys()))
+            model_info = MODELS[model_name]
+            user_id = random.choice(USER_IDS)
+            
+            # Generate random token counts
+            prompt_tokens = random.randint(100, 1000)
+            completion_tokens = random.randint(50, 500)
+            total_tokens = prompt_tokens + completion_tokens
+            
+            # Calculate cost
+            cost = (total_tokens / 1000) * model_info["cost_per_1k"]
+            
+            # Create model usage record
+            usage = ModelUsage(
+                user_id=user_id,
+                chat_id=random.randint(1, 50),  # Random chat ID
+                model_name=model_name,
+                provider=model_info["provider"],
+                prompt_tokens=prompt_tokens,
+                completion_tokens=completion_tokens,
+                total_tokens=total_tokens,
+                query_size=prompt_tokens * 4,  # Approximate characters
+                response_size=completion_tokens * 4,  # Approximate characters
+                cost=cost,
+                timestamp=timestamp,
+                is_streaming=random.choice([True, False]),
+                request_time_ms=random.randint(500, 5000)  # Between 0.5 and 5 seconds
+            )
+            session.add(usage)
+        
+        session.commit()
+        print(f"Successfully generated {num_records} test records")
+    
+    except Exception as e:
+        session.rollback()
+        print(f"Error generating test data: {e}")
+    
+    finally:
+        session.close()
+
+if __name__ == "__main__":
+    # Default to 100 records, but allow command line override
+    num_records = int(sys.argv[1]) if len(sys.argv) > 1 else 100
+    generate_test_data(num_records)
+    print("Done! The model_usage table has been populated with test data.") 

scripts/populate_agent_templates.py

@@ -116,10 +116,10 @@ def sync_agents_from_config():
     session = session_factory()
     db_type = get_database_type()
     
-    # if db_type != "sqlite":
-    #     print(f"⚠️  This script is designed for SQLite, but detected {db_type}")
-    #     print("Consider using manage_templates.py for PostgreSQL")
-    #     return
+    if db_type != "sqlite":
+        print(f"⚠️  This script is designed for SQLite, but detected {db_type}")
+        print("Consider using manage_templates.py for PostgreSQL")
+        return
     
     try:
         # Load configuration

scripts/setup_analytics_data.py

@@ -0,0 +1,114 @@
+import sys
+import os
+from datetime import datetime, timedelta
+import sqlite3
+
+# Add parent directory to path so we can import modules
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from src.db.schemas.models import ModelUsage, User
+from src.db.init_db import session_factory, init_db
+from scripts.generate_test_data import generate_test_data
+from scripts.create_test_user import create_test_users
+
+def check_database():
+    """Check if database exists and has the required tables"""
+    db_path = os.path.join(
+        os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 
+        "chat_database.db"
+    )
+    
+    if not os.path.exists(db_path):
+        print(f"Database file not found at {db_path}")
+        print("Creating database...")
+        init_db()
+        return False
+        
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+    
+    # Check if model_usage table exists
+    cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='model_usage'")
+    if not cursor.fetchone():
+        print("model_usage table does not exist.")
+        print("Creating tables...")
+        init_db()
+        return False
+        
+    # Count records in model_usage table
+    cursor.execute("SELECT COUNT(*) FROM model_usage")
+    count = cursor.fetchone()[0]
+    print(f"Found {count} records in model_usage table")
+    
+    conn.close()
+    return count > 0
+
+def setup_analytics():
+    """Set up analytics data for testing"""
+    # Check database status
+    has_data = check_database()
+    
+    # Create test users
+    print("\nCreating test users...")
+    create_test_users()
+    
+    # Generate model usage data if needed
+    if not has_data:
+        print("\nGenerating model usage test data...")
+        generate_test_data(100)
+    else:
+        print("\nDatabase already has model usage data. Skipping generation.")
+        choice = input("Generate additional data anyway? (y/n): ")
+        if choice.lower() == 'y':
+            records = int(input("How many records to generate? [default: 100]: ") or "100")
+            generate_test_data(records)
+    
+    # Verify data was created
+    session = session_factory()
+    try:
+        # Count model usage records
+        usage_count = session.query(ModelUsage).count()
+        print(f"\nTotal model usage records: {usage_count}")
+        
+        # Count users
+        user_count = session.query(User).count()
+        print(f"Total users: {user_count}")
+        
+        # Check recent data
+        yesterday = datetime.utcnow() - timedelta(days=1)
+        recent_count = session.query(ModelUsage).filter(ModelUsage.timestamp >= yesterday).count()
+        print(f"Records from the last 24 hours: {recent_count}")
+        
+        # Get model breakdown
+        models = {}
+        providers = {}
+        
+        for usage in session.query(ModelUsage).all():
+            if usage.model_name not in models:
+                models[usage.model_name] = 0
+            models[usage.model_name] += 1
+            
+            if usage.provider not in providers:
+                providers[usage.provider] = 0
+            providers[usage.provider] += 1
+            
+        print("\nModel breakdown:")
+        for model, count in models.items():
+            print(f"  {model}: {count} records")
+            
+        print("\nProvider breakdown:")
+        for provider, count in providers.items():
+            print(f"  {provider}: {count} records")
+            
+    finally:
+        session.close()
+    
+    print("\nSetup complete!")
+    print("To access the analytics dashboard:")
+    print("1. Make sure the backend server is running")
+    print("2. In your browser, set localStorage.adminApiKey to 'default-admin-key-change-me'")
+    print("   (or the value in your ADMIN_API_KEY environment variable)")
+    print("3. Go to http://localhost:3000/analytics/dashboard")
+
+if __name__ == "__main__":
+    setup_analytics() 

scripts/test_agent_parsing.py

@@ -0,0 +1,42 @@
+
+def parse_agents(agent_string):
+    """
+    Parse a string containing agent names separated by ->, (, ), or commas
+    and return a list of agent names.
+    """
+    if not agent_string or not agent_string.strip():
+        return []
+    
+    # Replace parentheses with spaces to handle cases with parentheses
+    import re
+    cleaned_string = re.sub(r'\(.*?\)', '', agent_string)
+    
+    # Split by -> to get individual agent segments
+    agent_segments = cleaned_string.split('->')
+    
+    # Process each segment to extract agent names
+    agents = []
+    for segment in agent_segments:
+        # Split by comma and strip whitespace
+        segment_agents = [agent.strip() for agent in segment.split(',') if agent.strip()]
+        agents.extend(segment_agents)
+    
+    return agents
+
+sample = "preprocessing_agent -> statistical_analytics_agent"
+agents = parse_agents(sample)
+print(agents)
+
+# Test with different formats
+test_samples = [
+    "preprocessing_agent -> data_viz_agent",
+    "preprocessing_agent(dataset, goal)",
+    "preprocessing_agent -> statistical_analytics_agent, data_viz_agent",
+    "preprocessing_agent, statistical_analytics_agent -> data_viz_agent",
+    "",
+    "preprocessing_agent(dataset, goal) -> data_viz_agent(dataset)"
+]
+
+for test in test_samples:
+    print(f"Input: {test}")
+    print(f"Output: {parse_agents(test)}")

scripts/test_deep_integration.py

@@ -0,0 +1,226 @@
+#!/usr/bin/env python3
+"""
+Test script to verify deep analysis integration with Housing.csv dataset
+"""
+
+import os
+import sys
+import requests
+import json
+from typing import Dict, Any
+
+# Test configuration
+BASE_URL = "http://localhost:8000"
+TEST_SESSION_ID = "test-deep-analysis-session-housing"
+
+def test_basic_functionality():
+    """Test basic server functionality"""
+    print("Testing server health...")
+    try:
+        response = requests.get(f"{BASE_URL}/health")
+        if response.status_code == 200:
+            print("✅ Server is running")
+            return True
+        else:
+            print(f"❌ Server health check failed: {response.status_code}")
+            return False
+    except Exception as e:
+        print(f"❌ Cannot connect to server: {e}")
+        print("Make sure the FastAPI server is running on localhost:8000")
+        return False
+
+
+
+def test_agents_endpoint():
+    """Test that agents endpoint includes deep analysis"""
+    print("\nTesting agents endpoint for deep analysis info...")
+    try:
+        response = requests.get(f"{BASE_URL}/agents")
+        if response.status_code == 200:
+            agents_info = response.json()
+            if "deep_analysis" in agents_info:
+                print("✅ Deep analysis info found in agents endpoint")
+                print(f"   Description: {agents_info['deep_analysis']['description']}")
+                print(f"   Available agents: {len(agents_info['available_agents'])}")
+                print(f"   Planner agents: {len(agents_info['planner_agents'])}")
+                return True
+            else:
+                print("❌ Deep analysis info not found in agents endpoint")
+                return False
+        else:
+            print(f"❌ Failed to get agents info: {response.status_code}")
+            return False
+    except Exception as e:
+        print(f"❌ Error testing agents endpoint: {e}")
+        return False
+
+def test_deep_analysis_with_housing_data():
+    """Test deep analysis with Housing.csv data"""
+    print("\nTesting deep analysis with Housing.csv dataset...")
+    
+    # First, reset session to ensure we have the default dataset
+    print("  Resetting session to default dataset...")
+    try:
+        reset_response = requests.post(
+            f"{BASE_URL}/session/reset_to_default",
+            headers={"X-Session-ID": TEST_SESSION_ID}
+        )
+        if reset_response.status_code == 200:
+            print("  ✅ Session reset to default dataset")
+        else:
+            print(f"  ⚠️ Session reset failed: {reset_response.status_code}")
+            
+        # Verify the dataset is actually loaded
+        print("  Verifying dataset is loaded...")
+        verify_response = requests.get(
+            f"{BASE_URL}/dataset/info",
+            headers={"X-Session-ID": TEST_SESSION_ID}
+        )
+        
+        if verify_response.status_code == 200:
+            dataset_info = verify_response.json()
+            if dataset_info.get("loaded", False):
+                print(f"  ✅ Dataset loaded: {dataset_info.get('rows', 0)} rows, {dataset_info.get('columns', 0)} columns")
+            else:
+                print("  ⚠️ No dataset loaded - attempting to load default dataset")
+                # Try to explicitly load the default dataset
+                load_response = requests.post(
+                    f"{BASE_URL}/dataset/load_default",
+                    headers={"X-Session-ID": TEST_SESSION_ID}
+                )
+                if load_response.status_code == 200:
+                    print("  ✅ Default dataset loaded")
+                else:
+                    print(f"  ❌ Failed to load default dataset: {load_response.status_code}")
+                    return False
+        else:
+            print(f"  ⚠️ Could not verify dataset: {verify_response.status_code}")
+    except Exception as e:
+        print(f"  ⚠️ Error during dataset setup: {e}")
+    
+    # Now test deep analysis
+    test_payload = {
+        "goal": "Analyze housing price patterns and identify key factors affecting prices"
+    }
+    
+    headers = {
+        "X-Session-ID": TEST_SESSION_ID,
+        "Content-Type": "application/json"
+    }
+    
+    try:
+        print("  Starting deep analysis...")
+        response = requests.post(
+            f"{BASE_URL}/deep_analysis_streaming",
+            json=test_payload,
+            headers=headers,
+            timeout=12000  # 5 minute timeout for analysis
+        )
+        
+        if response.status_code == 200:
+            print("✅ Deep analysis completed successfully!")
+            result = response.json()
+            
+            # Check key components
+            analysis = result.get("analysis", {})
+            print(f"   Goal: {analysis.get('goal', 'N/A')[:50]}...")
+            print(f"   Questions generated: {'✅' if analysis.get('deep_questions') else '❌'}")
+            print(f"   Plan created: {'✅' if analysis.get('deep_plan') else '❌'}")
+            print(f"   Code generated: {'✅' if analysis.get('code') else '❌'}")
+            print(f"   Visualizations: {len(analysis.get('plotly_figs', []))} figures")
+            print(f"   Synthesis: {'✅' if analysis.get('synthesis') else '❌'}")
+            print(f"   Conclusion: {'✅' if analysis.get('final_conclusion') else '❌'}")
+            print(f"   Processing time: {result.get('processing_time_seconds', 'unknown')} seconds")
+            
+            return True
+            
+        elif response.status_code == 400:
+            error_detail = response.json().get('detail', 'Unknown error')
+            if "No dataset" in error_detail:
+                print("❌ Dataset not properly loaded")
+                print(f"   Error: {error_detail}")
+            else:
+                print(f"❌ Client error: {error_detail}")
+            return False
+            
+        elif response.status_code == 500:
+            error_detail = response.json().get('detail', 'Unknown error')
+            print(f"❌ Server error during analysis: {error_detail}")
+            return False
+            
+        else:
+            print(f"❌ Unexpected response: {response.status_code}")
+            print(f"   Response: {response.text[:200]}...")
+            return False
+            
+    except requests.Timeout:
+        print("❌ Analysis timed out (this may be normal for complex analysis)")
+        return False
+    except Exception as e:
+        print(f"❌ Error during deep analysis: {e}")
+        return False
+
+def download_html_report():
+    """Test downloading HTML report"""
+    print("\nTesting HTML report download...")
+    try:
+        response = requests.post(
+            f"{BASE_URL}/deep_analysis/download_report",
+            headers={"X-Session-ID": TEST_SESSION_ID}
+        )
+        if response.status_code == 200:
+            print("✅ HTML report downloaded successfully")
+            return True
+        else:
+            print(f"❌ Failed to download HTML report: {response.status_code}")
+            return False
+    except Exception as e:
+        print(f"❌ Error during HTML report download: {e}")
+        return False
+
+
+def main():
+    print("🧪 Testing Deep Analysis Integration with Housing Data")
+    print("=" * 60)
+    
+    # Run tests in sequence
+    tests = [
+        ("Server Health", test_basic_functionality),
+        ("Agents Endpoint", test_agents_endpoint),
+        ("Deep Analysis with Housing Data", test_deep_analysis_with_housing_data),
+        ("HTML Report Download", download_html_report)
+    ]
+    
+    results = []
+    for test_name, test_func in tests:
+        print(f"\n🔍 Running: {test_name}")
+        result = test_func()
+        results.append((test_name, result))
+        
+        if not result:
+            print(f"⚠️ Test failed: {test_name}")
+            break
+    
+    # Summary
+    print("\n" + "=" * 60)
+    print("📊 Test Results Summary:")
+    
+    passed = sum(1 for _, result in results if result)
+    total = len(results)
+    
+    for test_name, result in results:
+        status = "✅ PASS" if result else "❌ FAIL"
+        print(f"   {status}: {test_name}")
+    
+    print(f"\nOverall: {passed}/{total} tests passed")
+    
+    if passed == total:
+        print("🎉 All tests passed! Deep analysis integration is working correctly.")
+    else:
+        print("⚠️ Some tests failed. Check the errors above for details.")
+    
+    return passed == total
+
+if __name__ == "__main__":
+    success = main()
+    sys.exit(0 if success else 1) 

scripts/test_model_usage.py

@@ -0,0 +1,114 @@
+import requests
+import time
+import uuid
+import os
+import sys
+
+# Add the parent directory to the path so we can import modules
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from ai_manager import AI_Manager
+from init_db import session_factory, ModelUsage
+
+def test_ai_manager_directly():
+    """Test the AI Manager directly to see if it records usage"""
+    print("Starting direct AI Manager test...")
+    
+    # Create a test AI Manager
+    ai_manager = AI_Manager()
+    
+    # Generate a unique user ID and chat ID for testing
+    test_user_id = int(time.time()) % 10000
+    test_chat_id = int(uuid.uuid4().int % 100000)
+    test_model = "test-direct-model"
+    
+    print(f"Using test user_id: {test_user_id}, chat_id: {test_chat_id}")
+    
+    # Call save_usage_to_db directly
+    ai_manager.save_usage_to_db(
+        user_id=test_user_id,
+        chat_id=test_chat_id,
+        model_name=test_model,
+        provider="Test Provider",
+        prompt_tokens=150,
+        completion_tokens=250,
+        total_tokens=400,
+        query_size=600,
+        response_size=1200,
+        cost=0.002,
+        request_time_ms=200,
+        is_streaming=False
+    )
+    
+    print("Usage data saved directly via AI Manager")
+    
+    # Check if it was actually saved
+    session = session_factory()
+    try:
+        # Count records for our test user
+        count = session.query(ModelUsage).filter(
+            ModelUsage.user_id == test_user_id,
+            ModelUsage.model_name == test_model
+        ).count()
+        
+        if count > 0:
+            print(f"SUCCESS: Found {count} records for test user {test_user_id}")
+            records = session.query(ModelUsage).filter(
+                ModelUsage.user_id == test_user_id
+            ).all()
+            
+            for record in records:
+                print(f"  - {record.usage_id}: {record.model_name}, {record.total_tokens} tokens, ${record.cost}")
+        else:
+            print(f"FAILURE: No records found for test user {test_user_id}")
+            print("Check the database connection and ModelUsage table")
+    finally:
+        session.close()
+    
+def test_api_endpoint():
+    """Test the API endpoint that creates test usage records"""
+    print("\nTesting API endpoint for creating test usage...")
+    
+    admin_key = os.getenv("ADMIN_API_KEY", "admin-api-key-change-me")
+    base_url = "http://localhost:8000" 
+    
+    # Generate a unique user ID for testing
+    test_user_id = int(time.time()) % 10000
+    test_model = "test-api-model"
+    
+    # Call the API endpoint
+    response = requests.post(
+        f"{base_url}/analytics/debug/create_test_usage",
+        params={
+            "user_id": test_user_id,
+            "model_name": test_model
+        },
+        headers={
+            "X-Admin-API-Key": admin_key
+        }
+    )
+    
+    if response.status_code == 200:
+        data = response.json()
+        print(f"SUCCESS: Created test usage via API endpoint")
+        print(f"Response: {data}")
+        
+        # Verify the record was created
+        verification = requests.get(
+            f"{base_url}/analytics/debug/user_usage/{test_user_id}",
+            headers={
+                "X-Admin-API-Key": admin_key
+            }
+        ).json()
+        
+        print(f"User {test_user_id} has {verification.get('total_records', 0)} records")
+        if verification.get('recent_records'):
+            print(f"Recent records: {verification['recent_records']}")
+    else:
+        print(f"FAILURE: API endpoint returned status {response.status_code}")
+        print(f"Response: {response.text}")
+
+if __name__ == "__main__":
+    # Run both tests
+    test_ai_manager_directly()
+    test_api_endpoint() 

scripts/test_token_counting.py

@@ -0,0 +1,176 @@
+import sys
+from pathlib import Path
+sys.path.append(str(Path(__file__).resolve().parent.parent))
+
+from src.managers.ai_manager import AI_Manager
+
+ai_manager = AI_Manager()
+
+input_prompt = "What are the key insights from this data? "
+
+output = """
+## Analytical Planner
+
+### Reasoning
+To extract key insights from the housing dataset, it is essential to first preprocess the data to ensure its quality and suitability for analysis. Following preprocessing, performing statistical analysis will help identify significant patterns and relationships within the data. Finally, visualizing the results will provide a clear and interpretable representation of the insights derived.
+
+
+## Preprocessing Agent
+
+### Reasoning
+To derive key insights from the Housing Dataset, it is essential to start with data preprocessing and introductory analysis. This involves categorizing the data into numeric and categorical features, handling missing values appropriately, and preparing the data for further exploration. By organizing the data effectively and understanding the relationships between different variables, we can uncover meaningful patterns and trends that inform our analysis objectives.
+
+### Code Implementation
+```python
+import numpy as np
+import pandas as pd
+
+# Define a safe datetime conversion function
+def safe_to_datetime(date):
+    try:
+        return pd.to_datetime(date, errors='coerce', cache=False)
+    except (ValueError, TypeError):
+        return pd.NaT
+
+# Create copies of the original dataframe
+df_copy = df.copy()
+
+# Identify categorical and numeric columns
+categorical_columns = df_copy.select_dtypes(include=['object', 'category']).columns.tolist()
+numeric_columns = df_copy.select_dtypes(include=[np.number]).columns.tolist()
+
+# Handle missing values
+# For numeric columns, fill missing values with the median
+df_copy[numeric_columns] = df_copy[numeric_columns].fillna(df_copy[numeric_columns].median())
+
+# For categorical columns, fill missing values with the mode
+for col in categorical_columns:
+    df_copy[col] = df_copy[col].fillna(df_copy[col].mode()[0])
+
+# Convert string dates to datetime format if any date columns exist
+# Example: If there's a column named 'date', uncomment the following lines
+# if 'date' in df_copy.columns:
+#     df_copy['date'] = df_copy['date'].apply(safe_to_datetime)
+
+# Create a correlation matrix for numeric columns
+correlation_matrix = df_copy[numeric_columns].corr()
+
+# Display the first few rows of the processed dataframe and the correlation matrix
+print(df_copy.head())
+print("\nCorrelation Matrix:\n", correlation_matrix)
+```
+
+### Summary
+1. **Import Libraries**: The necessary libraries, NumPy and Pandas, are imported to handle numerical operations and data manipulation.
+
+2. **Datetime Conversion Function**: A `safe_to_datetime` function is defined to gracefully handle the conversion of string dates to datetime objects, ensuring that any errors during conversion result in `NaT` (Not a Time) values without interrupting the workflow.
+
+3. **Dataframe Copy**: A copy of the original dataframe `df` is created to preserve the original data during preprocessing.
+
+4. **Identify Column Types**:
+   - **Categorical Columns**: Identified using `select_dtypes` to include data types `object` and `category`.
+   - **Numeric Columns**: Identified by selecting columns with numeric data types using `np.number`.
+
+5. **Handle Missing Values**:
+   - **Numeric Columns**: Missing values are filled with the median of each numeric column to maintain the central tendency without being skewed by outliers.
+   - **Categorical Columns**: Missing values are filled with the mode (most frequent value) to preserve the most common category.
+
+6. **Datetime Conversion**: Although the current dataset does not contain date columns, the preparation is made to handle such conversions if needed in the future by checking for date-specific columns and applying the `safe_to_datetime` function.
+
+7. **Correlation Matrix**: A correlation matrix is generated for the numeric columns to identify the strength and direction of relationships between different numerical variables, which is crucial for understanding how features interact with each other.
+
+8. **Display Outputs**: The first few rows of the processed dataframe and the correlation matrix are printed to provide an initial view of the data and its interrelationships, laying the groundwork for deeper exploratory data analysis.
+
+
+## Statistical Analytics Agent
+
+### Reasoning
+To extract key insights from the Housing Dataset, we will perform a multiple linear regression analysis. This will allow us to identify which factors significantly influence housing prices. By analyzing both numerical variables (such as area, bedrooms, bathrooms) and categorical variables (like mainroad, furnishingstatus), we can determine the key drivers of price variations. Proper handling of missing values and categorical data will ensure the robustness of our analysis.
+
+### Code Implementation
+```python
+import pandas as pd
+import numpy as np
+import statsmodels.api as sm
+
+# Create a copy of the dataframe
+df_copy = df.copy()
+
+try:
+    # Define the dependent variable
+    y = df_copy['price']
+    
+    # Define the independent variables
+    X = df_copy.drop('price', axis=1)
+    
+    # Check for missing values and drop rows with any missing values
+    X = X.dropna()
+    y = y.loc[X.index]
+    
+    # Convert categorical variables to 'category' dtype
+    categorical_columns = X.select_dtypes(include=['object', 'category']).columns
+    for col in categorical_columns:
+        X[col] = X[col].astype('category')
+    
+    # Add a constant term to the predictors
+    X = sm.add_constant(X)
+    
+    # Fit the OLS regression model
+    model = sm.OLS(y.astype(float), X.astype(float)).fit()
+    
+    # Print the model summary
+    print(model.summary())
+    
+except Exception as e:
+    print(f"An error occurred: {e}")
+```
+
+### Summary
+The code performs a multiple linear regression analysis to identify the key factors affecting housing prices. It begins by creating a copy of the original dataframe to preserve the data. The dependent variable `price` is separated from the independent variables. Rows with any missing values are removed to ensure data quality. Categorical variables are converted to the 'category' data type, which allows statsmodels to handle them appropriately in the regression. A constant term is added to the independent variables to account for the intercept in the model. The Ordinary Least Squares (OLS) regression model is then fitted using the prepared data. Finally, the summary of the regression model is printed, providing detailed statistics on the significance and impact of each predictor on the housing price.
+
+
+## Data Viz Agent
+
+### Reasoning
+To extract key insights from the housing dataset, it's important to analyze the relationship between house area and price, as well as understand how factors like the number of bedrooms and bathrooms influence pricing. A scatter plot of area versus price can reveal trends and correlations, while color-coding by the number of bedrooms can provide additional depth to the analysis.
+
+### Code Implementation
+```python
+import plotly.express as px
+
+# Check if the dataset has more than 50,000 rows and sample if necessary
+if len(df) > 50000:
+    df_sample = df.sample(n=50000)
+else:
+    df_sample = df.copy()
+
+# Create a scatter plot of area vs price, colored by number of bedrooms
+fig = px.scatter(df_sample, 
+                 x='area', 
+                 y='price', 
+                 color='bedrooms', 
+                 title='Housing Price vs Area Colored by Number of Bedrooms', 
+                 labels={'area': 'Area (sq ft)', 'price': 'Price'},
+                 hover_data=['bathrooms', 'stories'])
+
+# Update layout for clarity and styling
+fig.update_layout(
+    template='plotly_white',
+    xaxis_title='Area (sq ft)',
+    yaxis_title='Price',
+    legend_title='Bedrooms'
+)
+
+fig.to_html(full_html=False)
+```
+
+### Summary
+The scatter plot visualizes the relationship between the area of the houses and their prices. By color-coding the points based on the number of bedrooms, it becomes easier to observe how bedroom count correlates with both area and price. This helps in identifying trends such as whether larger houses with more bedrooms tend to be priced higher.
+"""
+
+
+input_tokens = len(ai_manager.tokenizer.encode(input_prompt))
+output_tokens = len(ai_manager.tokenizer.encode(output))
+
+print(f"Input tokens: {input_tokens}")
+print(f"Output tokens: {output_tokens}")

scripts/verify_session_state.py

@@ -0,0 +1,76 @@
+import requests
+import uuid
+import time
+
+def test_session_workflow():
+    # Base URL
+    base_url = "http://localhost:8000"
+    
+    # Create a unique session ID
+    session_id = str(uuid.uuid4())
+    print(f"Testing with session ID: {session_id}")
+    
+    # Step 1: Create a user and associate with session
+    print("\n1. Creating user...")
+    username = f"test_user_{session_id[:8]}"
+    email = f"{username}@example.com"
+    login_response = requests.post(
+        f"{base_url}/auth/login",
+        json={"username": username, "email": email, "session_id": session_id}
+    ).json()
+    print(f"Login response: {login_response}")
+    
+    # Step 2: Verify session state
+    print("\n2. Verifying session state...")
+    session_state = requests.get(f"{base_url}/debug/session/{session_id}").json()
+    print(f"Session state: {session_state}")
+    
+    # Step 3: Make a chat request with this session
+    print("\n3. Making a test chat request...")
+    chat_response = requests.post(
+        f"{base_url}/chat/data_explorer",
+        json={"query": "Show me a summary of the dataset"},
+        params={"session_id": session_id}
+    ).json()
+    print(f"Chat response received: {len(str(chat_response))} bytes")
+    
+    # Step 4: Verify session state again
+    print("\n4. Verifying session state after chat...")
+    session_state = requests.get(f"{base_url}/debug/session/{session_id}").json()
+    print(f"Updated session state: {session_state}")
+    
+    # Step 5: Check analytics data
+    print("\n5. Checking analytics data...")
+    time.sleep(1)  # Wait a moment for data to be saved
+    admin_key = "default-admin-key-change-me"  # Adjust to your admin key
+
+    # Check general model usage
+    analytics_response = requests.get(
+        f"{base_url}/analytics/debug/model_usage",
+        headers={"X-Admin-API-Key": admin_key}
+    ).json()
+    print(f"Total records in database: {analytics_response.get('total_records', 0)}")
+
+    if 'sample_records' in analytics_response and analytics_response['sample_records']:
+        print(f"Latest usage records: {analytics_response['sample_records']}")
+    else:
+        print("No sample records found in general model usage")
+
+    # Check user-specific usage
+    user_id = login_response.get('user_id')
+    if user_id:
+        user_usage = requests.get(
+            f"{base_url}/analytics/debug/user_usage/{user_id}",
+            headers={"X-Admin-API-Key": admin_key}
+        ).json()
+        print(f"User {user_id} has {user_usage.get('total_records', 0)} usage records")
+        
+        if user_usage.get('recent_records'):
+            print(f"Recent usage: {user_usage['recent_records']}")
+        else:
+            print(f"No usage records found for user {user_id}")
+
+    print("\nTest completed!")
+
+if __name__ == "__main__":
+    test_session_workflow()

src/agents/agents.py

@@ -4,20 +4,14 @@ import asyncio
 from dotenv import load_dotenv
 import logging
 from src.utils.logger import Logger
-from src.utils.model_registry import small_lm, mid_lm
-import json
-
 load_dotenv()
 
-logger = Logger("agents", see_time=True, console_log=True)
-
-
+logger = Logger("agents", see_time=True, console_log=False)
 
 # === CUSTOM AGENT FUNCTIONALITY ===
 def create_custom_agent_signature(agent_name, description, prompt_template, category=None):
     """
     Dynamically creates a dspy.Signature class for custom agents.
-    Has to be tested
     
     Args:
         agent_name: Name of the custom agent (e.g., 'pytorch_agent')
@@ -131,83 +125,73 @@ def load_user_enabled_templates_for_planner_from_db(user_id, db_session):
     Returns:
         Dict of template agent signatures keyed by template name (max 10)
     """
-    # try:
-    from src.db.schemas.models import AgentTemplate, UserTemplatePreference
-    from datetime import datetime, UTC
-    
-    agent_signatures = {}
-    
-
-    
-    # Get list of default planner agent names that should be enabled by default
-    default_planner_agent_names = [
-        "planner_preprocessing_agent",
-        "planner_statistical_analytics_agent", 
-        "planner_sk_learn_agent",
-        "planner_data_viz_agent"
-    ]
-    # if not user_id:
-    #     return agent_signatures
-    
-    # Get all active planner variant templates
-    all_templates = db_session.query(AgentTemplate).filter(
-        AgentTemplate.is_active == True,
-        AgentTemplate.variant_type.in_(['planner', 'both'])
-    ).all()
-    
-    enabled_templates = []
-    # Fetch all preferences for the user in a single query for efficiency
-    preferences = db_session.query(UserTemplatePreference).filter(
-        UserTemplatePreference.user_id == user_id
-    ).all()
-    preference_map = {pref.template_id: pref for pref in preferences}
-
-    for template in all_templates:
-        preference = preference_map.get(template.template_id)
-        is_default_planner_agent = template.template_name in default_planner_agent_names
-        default_enabled = is_default_planner_agent
-        is_enabled = preference.is_enabled if preference else default_enabled
-
-        if is_enabled:
-            enabled_templates.append({
-                'template': template,
-                'preference': preference,
-                'usage_count': preference.usage_count if preference else 0,
-                'last_used_at': preference.last_used_at if preference else None
-            })
-
-    # If user has no enabled templates, fall back to default enabled (default planner agents)
-    if enabled_templates == []:
+    try:
+        from src.db.schemas.models import AgentTemplate, UserTemplatePreference
+        from datetime import datetime, UTC
+        
+        agent_signatures = {}
+        
+        if not user_id:
+            return agent_signatures
+        
+        # Get list of default planner agent names that should be enabled by default
+        default_planner_agent_names = [
+            "planner_preprocessing_agent",
+            "planner_statistical_analytics_agent", 
+            "planner_sk_learn_agent",
+            "planner_data_viz_agent"
+        ]
+        
+        # Get all active planner variant templates
+        all_templates = db_session.query(AgentTemplate).filter(
+            AgentTemplate.is_active == True,
+            AgentTemplate.variant_type.in_(['planner', 'both'])
+        ).all()
+        
+        enabled_templates = []
         for template in all_templates:
-            if template.template_name in default_planner_agent_names:
+            # Check if user has a preference record for this template
+            preference = db_session.query(UserTemplatePreference).filter(
+                UserTemplatePreference.user_id == user_id,
+                UserTemplatePreference.template_id == template.template_id
+            ).first()
+            
+            # Determine if template should be enabled by default
+            is_default_planner_agent = template.template_name in default_planner_agent_names
+            default_enabled = is_default_planner_agent  # Default planner agents enabled by default, others disabled
+            
+            # Template is enabled by default for default agents, disabled for others
+            is_enabled = preference.is_enabled if preference else default_enabled
+            
+            if is_enabled:
                 enabled_templates.append({
                     'template': template,
-                    'preference': None,
-                    'usage_count': 0,
-                    'last_used_at': None
+                    'preference': preference,
+                    'usage_count': preference.usage_count if preference else 0,
+                    'last_used_at': preference.last_used_at if preference else None
                 })
-
-    # Sort by usage (most used first) and limit to 10
-    enabled_templates.sort(key=lambda x: (x['usage_count'], x['last_used_at'] or datetime.min.replace(tzinfo=UTC)), reverse=True)
-    enabled_templates = enabled_templates[:10]
-    
-    for item in enabled_templates:
-        template = item['template']
-        # Create dynamic signature for each enabled template
-        signature = create_custom_agent_signature(
-            template.template_name,
-            template.description,
-            template.prompt_template,
-            template.category  # Pass the category from database
-        )
-        agent_signatures[template.template_name] = signature
-            
-    logger.log_message(f"Loaded {len(agent_signatures)} templates for planner", level=logging.DEBUG)
-    return agent_signatures
         
-    # except Exception as e:
-    #     logger.log_message(f"Error loading planner templates for user {user_id}: {str(e)}", level=logging.ERROR)
-    #     return {}
+        # Sort by usage (most used first) and limit to 10
+        enabled_templates.sort(key=lambda x: (x['usage_count'], x['last_used_at'] or datetime.min.replace(tzinfo=UTC)), reverse=True)
+        enabled_templates = enabled_templates[:10]
+        
+        for item in enabled_templates:
+            template = item['template']
+            # Create dynamic signature for each enabled template
+            signature = create_custom_agent_signature(
+                template.template_name,
+                template.description,
+                template.prompt_template,
+                template.category  # Pass the category from database
+            )
+            agent_signatures[template.template_name] = signature
+                
+        logger.log_message(f"Loaded {len(agent_signatures)} templates for planner", level=logging.DEBUG)
+        return agent_signatures
+        
+    except Exception as e:
+        logger.log_message(f"Error loading planner templates for user {user_id}: {str(e)}", level=logging.ERROR)
+        return {}
 
 def get_all_available_templates(db_session):
     """
@@ -221,34 +205,11 @@ def get_all_available_templates(db_session):
     """
     try:
         from src.db.schemas.models import AgentTemplate
-        import os
-        import json
-
+        
         templates = db_session.query(AgentTemplate).filter(
             AgentTemplate.is_active == True
         ).all()
-
-        if not templates:
-            # Try to load from agents_config.json after the main folder (project root or backend dir)
-            current_dir = os.path.dirname(os.path.abspath(__file__))
-            backend_dir = os.path.dirname(current_dir)
-            project_root = os.path.dirname(backend_dir)
-            possible_paths = [
-                os.path.join(backend_dir, 'agents_config.json'),  # backend directory
-                os.path.join(project_root, 'agents_config.json'),  # project root
-                '/app/agents_config.json'  # container root (for Docker/Spaces)
-            ]
-            config_path = None
-            for path in possible_paths:
-                if os.path.exists(path):
-                    config_path = path
-                    break
-            if config_path:
-                with open(config_path, 'r', encoding='utf-8') as f:
-                    config = json.load(f)
-                templates = config.get('templates', [])
-            else:
-                templates = []
+        
         return templates
         
     except Exception as e:
@@ -390,59 +351,37 @@ class chat_history_name_agent(dspy.Signature):
     name = dspy.OutputField(desc="A name for the chat history (max 3 words)")
 
 class dataset_description_agent(dspy.Signature):
-    """
-
-    Generate a structured dataset context/description like this, you will be given headers for the data & existing description!
-{
-  "exact": "apple_stock_historical_data",
-  "description": "Daily historical stock market data for Apple Inc. including open, close, high, low prices, trading volume, and adjusted close for accurate return calculations.",
-  "columns": {
-    "Date": {
-      "type": "datetime",
-      "format": "YYYY-MM-DD",
-      "description": "Trading date",
-      "preprocessing": "Convert strings using pd.to_datetime(df['Date'], format='%Y-%m-%d')",
-      "missing_values_handling": "Interpolate or forward-fill missing dates for continuity"
-    },
-    "Open": {
-      "type": "float",
-      "description": "Opening stock price in USD",
-      "preprocessing": "Direct float conversion"
-    },
-    "High": {
-      "type": "float",
-      "description": "Highest stock price in USD during the trading day",
-      "preprocessing": "Direct float conversion"
-    },
-    "Low": {
-      "type": "float",
-      "description": "Lowest stock price in USD during the trading day",
-      "preprocessing": "Direct float conversion"
-    },
-    "Close": {
-      "type": "float",
-      "description": "Closing stock price in USD",
-      "preprocessing": "Direct float conversion"
-    },
-    "Adj Close": {
-      "type": "float",
-      "description": "Adjusted closing price accounting for dividends and splits",
-      "preprocessing": "Direct float conversion"
-    },
-    "Volume": {
-      "type": "integer",
-      "description": "Number of shares traded during the day",
-      "preprocessing": "Direct integer conversion"
-    }
-  },
-
-
-  "usage_notes": "Ensure adjusted close prices are used for return calculations. Use consistent timezone if merging with other datasets. Handle missing values carefully to maintain temporal continuity.",
-
+    """You are an AI agent that generates a detailed description of a given dataset for both users and analysis agents.
+Your description should serve two key purposes:
+1. Provide users with context about the dataset's purpose, structure, and key attributes.
+2. Give analysis agents critical data handling instructions to prevent common errors.
+For data handling instructions, you must always include Python data types and address the following:
+- Data type warnings (e.g., numeric columns stored as strings that need conversion).
+- Null value handling recommendations.
+- Format inconsistencies that require preprocessing.
+- Explicit warnings about columns that appear numeric but are stored as strings (e.g., '10' vs 10).
+- Explicit Python data types for each major column (e.g., int, float, str, bool, datetime).
+- Columns with numeric values that should be treated as categorical (e.g., zip codes, IDs).
+- Any date parsing or standardization required (e.g., MM/DD/YYYY to datetime).
+- Any other technical considerations that would affect downstream analysis or modeling.
+- List all columns and their data types with exact case sensitive spelling
+If an existing description is provided, enhance it with both business context and technical guidance for analysis agents, preserving accurate information from the existing description or what the user has written.
+Ensure the description is comprehensive and provides actionable insights for both users and analysis agents.
+Example:
+This housing dataset contains property details including price, square footage, bedrooms, and location data.
+It provides insights into real estate market trends across different neighborhoods and property types.
+TECHNICAL CONSIDERATIONS FOR ANALYSIS:
+- price (str): Appears numeric but is stored as strings with a '$' prefix and commas (e.g., "$350,000"). Requires cleaning with str.replace('$','').replace(',','') and conversion to float.
+- square_footage (str): Contains unit suffix like 'sq ft' (e.g., "1,200 sq ft"). Remove suffix and commas before converting to int.
+- bedrooms (int): Correctly typed but may contain null values (~5% missing) – consider imputation or filtering.
+- zip_code (int): Numeric column but should be treated as str or category to preserve leading zeros and prevent unintended numerical analysis.
+- year_built (float): May contain missing values (~15%) – consider mean/median imputation or exclusion depending on use case.
+- listing_date (str): Dates stored in "MM/DD/YYYY" format – convert to datetime using pd.to_datetime().
+- property_type (str): Categorical column with inconsistent capitalization (e.g., "Condo", "condo", "CONDO") – normalize to lowercase for consistent grouping.
     """
     dataset = dspy.InputField(desc="The dataset to describe, including headers, sample data, null counts, and data types.")
     existing_description = dspy.InputField(desc="An existing description to improve upon (if provided).", default="")
-    description = dspy.OutputField(desc="A comprehensive dataset context with business context and technical guidance for analysis agents.")
+    description = dspy.OutputField(desc="A comprehensive dataset description with business context and technical guidance for analysis agents.")
 
 
 class custom_agent_instruction_generator(dspy.Signature):
@@ -625,14 +564,12 @@ class basic_query_planner(dspy.Signature):
     plan_instructions:{
                     "data_viz_agent": {
                         "create": ["correlation"],
-                        "use": "use": ["original_data"],
+                        "use": ["original_data"],
                         "instruction": "use the original_data to measure correlation of X & Y, using pandas"
                     }
     
     
     Respond in the user's language for all explanations and instructions, but keep all code, variable names, function names, model names, agent names, and library names in English.
-    original_data is placeholder, use exact_python_name: name_of_df for actual dataset name
-
     """
     dataset = dspy.InputField(desc="Available datasets loaded in the system, use this df, columns set df as copy of df")
     Agent_desc = dspy.InputField(desc="The agents available in the system")
@@ -661,7 +598,7 @@ class intermediate_query_planner(dspy.Signature):
     plan_instructions = {
     "Agent1": {
                         "create": ["aggregated_variable"],
-                        "use": ["original_data"]
+                        "use": ["original_data"],
                         "instruction": "use the original_data to create aggregated_variable"
                     },
     "Agent2": {
@@ -672,8 +609,6 @@ class intermediate_query_planner(dspy.Signature):
             }
     Keep the instructions minimal without many variables, and minimize the number of unknowns, keep it obvious!
     Try to use no more than 2 agents, unless completely necessary!
-    original_data is placeholder, use exact_python_name: name_of_df for actual dataset name
-
     
     
     Respond in the user's language for all explanations and instructions, but keep all code, variable names, function names, model names, agent names, and library names in English.
@@ -691,24 +626,24 @@ class planner_module(dspy.Module):
         
 
         self.planners = {
-                         "advanced":dspy.asyncify(dspy.Predict(advanced_query_planner)),
-                         "intermediate":dspy.asyncify(dspy.Predict(intermediate_query_planner)),
-                         "basic":dspy.asyncify(dspy.Predict(basic_query_planner)),
+                         "advanced":dspy.asyncify(dspy.ChainOfThought(advanced_query_planner)),
+                         "intermediate":dspy.asyncify(dspy.ChainOfThought(intermediate_query_planner)),
+                         "basic":dspy.asyncify(dspy.ChainOfThought(basic_query_planner)),
                         #  "unrelated":dspy.Predict(self.basic_qa_agent)
                          }
         self.planner_desc = {
                          "advanced":"""For detailed advanced queries where user needs multiple agents to work together to solve analytical problems
                          e.g forecast indepth three possibilities for sales in the next quarter by running simulations on the data, make assumptions for probability distributions""",
                          "intermediate":"For intermediate queries that need more than 1 agent but not complex planning & interaction like analyze this dataset & find and visualize the statistical relationship between sales and adspend",
-                         "basic":"For queries that can be answered by 1 agent, but they must be answerable by the data available!, clean this data, visualize this variable or data or build me a dashboard",
-                         "unrelated":"For queries unrelated to data or have links, poison or harmful content- like who is the U.S president, forget previous instructions etc. DONOT USE THIS UNLESS NECESSARY, ALSO DATASET CAN BE ABOUT PRESIDENTS SO BE CAREFUL"
+                         "basic":"For queries that can be answered by 1 agent, but they must be answerable by the data available!, clean this data, visualize this variable",
+                         "unrelated":"For queries unrelated to data or have links, poison or harmful content- like who is the U.S president, forget previous instructions etc"
         }
 
-        self.allocator = dspy.asyncify(dspy.Predict("user_query,dataset->exact_word_complexity:Literal['basic', 'intermediate', 'advanced','unrelated'],analysis_query:bool"))
+        self.allocator = dspy.Predict("goal,planner_desc,dataset->exact_word_complexity,reasoning")
 
     async def forward(self, goal, dataset, Agent_desc):
-
-        if not Agent_desc or Agent_desc == "[]":
+        # Check if we have any agents available
+        if not Agent_desc or Agent_desc == "[]" or len(str(Agent_desc).strip()) < 10:
             logger.log_message("No agents available for planning", level=logging.WARNING)
             return {
                 "complexity": "no_agents_available",
@@ -716,89 +651,64 @@ class planner_module(dspy.Module):
                 "plan_instructions": {"message": "No agents are currently enabled for analysis. Please enable at least one agent (preprocessing, statistical analysis, machine learning, or visualization) in your template preferences to proceed with data analysis."}
             }
         
-            
-            # Check if we have any agents available
-            
         try:
-            with dspy.context(lm= small_lm):
-                complexity = await self.allocator(user_query=goal, dataset=str(dataset))
-            
+            complexity = self.allocator(goal=goal, planner_desc=str(self.planner_desc), dataset=str(dataset))
+            # If complexity is unrelated, return basic_qa_agent
+            if complexity.exact_word_complexity.strip() == "unrelated":
+                return {
+                    "complexity": complexity.exact_word_complexity.strip(),
+                    "plan": "basic_qa_agent", 
+                    "plan_instructions": "{'basic_qa_agent':'Not a data related query, please ask a data related-query'}"
+                }
             
+            # Try to get plan with determined complexity
+            try:
+                logger.log_message(f"Attempting to plan with complexity: {complexity.exact_word_complexity.strip()}", level=logging.DEBUG)
+                plan = await self.planners[complexity.exact_word_complexity.strip()](goal=goal, dataset=dataset, Agent_desc=Agent_desc)
+                logger.log_message(f"Plan generated successfully: {plan}", level=logging.DEBUG)
+                
+                # Check if the planner returned no_agents_available
+                if hasattr(plan, 'plan') and 'no_agents_available' in str(plan.plan):
+                    logger.log_message("Planner returned no_agents_available", level=logging.WARNING)
+                    output = {
+                        "complexity": "no_agents_available",
+                        "plan": "no_agents_available",
+                        "plan_instructions": {"message": "No agents are currently enabled for analysis. Please enable at least one agent (preprocessing, statistical analysis, machine learning, or visualization) in your template preferences to proceed with data analysis."}
+                    }
+                else:
+                    output = {
+                        "complexity": complexity.exact_word_complexity.strip(),
+                        "plan": dict(plan)
+                    }
 
-
+            except Exception as e:
+                logger.log_message(f"Error with {complexity.exact_word_complexity.strip()} planner, falling back to intermediate: {str(e)}", level=logging.WARNING)
+                
+                # Fallback to intermediate planner
+                plan = await self.planners["intermediate"](goal=goal, dataset=dataset, Agent_desc=Agent_desc)
+                logger.log_message(f"Fallback plan generated: {plan}", level=logging.DEBUG)
+                
+                # Check if the fallback planner also returned no_agents_available
+                if hasattr(plan, 'plan') and 'no_agents_available' in str(plan.plan):
+                    logger.log_message("Fallback planner also returned no_agents_available", level=logging.WARNING)
+                    output = {
+                        "complexity": "no_agents_available",
+                        "plan": "no_agents_available", 
+                        "plan_instructions": {"message": "No agents are currently enabled for analysis. Please enable at least one agent (preprocessing, statistical analysis, machine learning, or visualization) in your template preferences to proceed with data analysis."}
+                    }
+                else:
+                    output = {
+                        "complexity": "intermediate",
+                        "plan": dict(plan)
+                    }
+                    
         except Exception as e:
             logger.log_message(f"Error in planner forward: {str(e)}", level=logging.ERROR)
             # Return error response
             return {
                 "complexity": "error",
                 "plan": "basic_qa_agent",
-                "plan_instructions": {"error": f"Planning error in agents: {str(e)} "}
-            }
-            # If complexity is unrelated, return basic_qa_agent
-        if complexity.exact_word_complexity.strip() == "unrelated":
-            if complexity.analysis_query==True:
-                plan = await self.planners['basic'](goal=goal, dataset=dataset, Agent_desc=Agent_desc)
-                return {
-                "complexity": 'basic',
-                "plan": plan.plan,
-                "plan_instructions": plan.plan_instructions}
-
-
-            return {
-                "complexity": complexity.exact_word_complexity.strip().lower(),
-                "plan": "basic_qa_agent", 
-                "plan_instructions": "{'basic_qa_agent':'Not a data related query, please ask a data related-query'}"
-            }
-                
-                
-
-                
-                # Try to get plan with determined complexity
-        # try:
-        logger.log_message(f"Attempting to plan with complexity: {complexity.exact_word_complexity.strip().lower()}", level=logging.DEBUG)
-        with dspy.context(lm = mid_lm):
-            plan = await self.planners[complexity.exact_word_complexity.strip()](goal=goal, dataset=dataset, Agent_desc=Agent_desc)
-
-        # if len(str(plan)) == 0:
-        #     output = {
-        #         "complexity": "error",
-        #         "plan": "Something went wrong it is not 0" + str(plan),
-        #         "plan_instructions": {"message": "the plan was not constructed" + str(Agent_desc)}   
-        #         }
-        # else:
-        #     output = {
-        #         "complexity": "error",
-        #         "plan": "Something went wrong it is 0" + str(plan),
-        #         "plan_instructions": {"message": "the plan was not constructed" + str(Agent_desc)}   
-        #         }
-
-
-
-        
-        # If plan or plan.plan is not available, return an error response
-        if not plan or not hasattr(plan, 'plan'):
-            logger.log_message("Planner did not return a valid plan object or 'plan' attribute is missing", level=logging.ERROR)
-            return {
-                "complexity": complexity.exact_word_complexity.strip().lower(),
-                "plan": "planning_error",
-                "plan_instructions": {"error": "Planner did not return a valid plan. Please try again or check agent configuration."}
-            }
-        
-        logger.log_message(f"Plan generated successfully: {plan}", level=logging.DEBUG)
-        
-        # Check if the planner returned no_agents_available
-        if hasattr(plan, 'plan') and 'no_agents_available' in str(plan.plan):
-            logger.log_message("Planner returned no_agents_available", level=logging.WARNING)
-            output = {
-                "complexity": "no_agents_available",
-                "plan": "no_agents_available",
-                "plan_instructions": {"message": "No agents are currently enabled for analysis. Please enable at least one agent (preprocessing, statistical analysis, machine learning, or visualization) in your template preferences to proceed with data analysis."}
-            }
-
-        output = {
-                "complexity": complexity.exact_word_complexity.strip().lower(),
-                "plan": plan.plan,
-                "plan_instructions": plan.plan_instructions
+                "plan_instructions": {"error": f"Planning error: {str(e)}"}
             }
         
         return output
@@ -807,12 +717,11 @@ class planner_module(dspy.Module):
 
 
 
-
 class preprocessing_agent(dspy.Signature):
     """
 You are a preprocessing agent that can work both individually and in multi-agent data analytics systems.
 You are given:
-* A dataset (already loaded as with exact_python_name mentioned).
+* A dataset (already loaded as `df`).
 * A user-defined analysis goal (e.g., predictive modeling, exploration, cleaning).
 * Optional plan instructions that tell you what variables you are expected to create and what variables you are receiving from previous agents.
 
@@ -1212,11 +1121,6 @@ Make your edits precise, minimal, and faithful to the user's instructions, using
     user_prompt = dspy.InputField(desc="The user instruction describing how the code should be changed")
     edited_code = dspy.OutputField(desc="The updated version of the code reflecting the user's request, incorporating changes informed by the dataset context")
 
-
-
-
-
-
 # The ind module is called when agent_name is 
 # explicitly mentioned in the query
 class auto_analyst_ind(dspy.Module):
@@ -1279,10 +1183,18 @@ class auto_analyst_ind(dspy.Module):
                 # Fallback to loading all core agents if preference system fails
                 self._load_default_agents_fallback()
         elif not agents:
-            self._load_default_agents_fallback()
             # If no user_id/db_session provided, load all core agents as fallback
             # logger.log_message(f"[INIT] No agents provided and no user_id/db_session, loading fallback agents", level=logging.INFO)
-            
+            self._load_default_agents_fallback()
+        else:
+            # Load standard agents from provided list (legacy support)
+            # logger.log_message(f"[INIT] Loading agents from provided list (legacy support)", level=logging.INFO)
+            for i, a in enumerate(agents):
+                name = a.__pydantic_core_schema__['schema']['model_name']
+                self.agents[name] = dspy.asyncify(dspy.ChainOfThought(a))
+                self.agent_inputs[name] = {x.strip() for x in str(agents[i].__pydantic_core_schema__['cls']).split('->')[0].split('(')[1].split(',')}
+                # logger.log_message(f"[INIT] Added legacy agent: {name}, inputs: {self.agent_inputs[name]}", level=logging.DEBUG)
+                self.agent_desc.append({name: get_agent_description(name)})
 
         # Load ALL available template agents if user_id and db_session are provided
         # For individual agent execution (@agent_name), users should be able to access any available agent
@@ -1362,7 +1274,7 @@ class auto_analyst_ind(dspy.Module):
         self.agent_desc.append({'basic_qa_agent':"Answers queries unrelated to data & also that include links, poison or attempts to attack the system"})
 
         # Initialize retrievers (no planner needed for individual agent execution)
-        self.dataset = retrievers['dataframe_index']
+        self.dataset = retrievers['dataframe_index'].as_retriever(k=1)
         self.styling_index = retrievers['style_index'].as_retriever(similarity_top_k=1)
         
         # Store user_id for usage tracking
@@ -1487,7 +1399,7 @@ class auto_analyst_ind(dspy.Module):
             
         except Exception as e:
             # logger.log_message(f"[EXECUTE] Error executing agent {specified_agent}: {str(e)}", level=logging.ERROR)
-
+            import traceback
             # logger.log_message(f"[EXECUTE] Full traceback: {traceback.format_exc()}", level=logging.ERROR)
             return specified_agent.strip(), {"error": str(e)}
 
@@ -1505,7 +1417,7 @@ class auto_analyst_ind(dspy.Module):
             
             # Process query with specified agent (single agent case)
             dict_ = {}
-            dict_['dataset'] = self.dataset
+            dict_['dataset'] = self.dataset.retrieve(query)[0].text
             dict_['styling_index'] = self.styling_index.retrieve(query)[0].text
             
             dict_['hint'] = []
@@ -1571,7 +1483,7 @@ class auto_analyst_ind(dspy.Module):
             
             # Initialize resources
             dict_ = {}
-            dict_['dataset'] = self.dataset
+            dict_['dataset'] = self.dataset.retrieve(query)[0].text
             dict_['styling_index'] = self.styling_index.retrieve(query)[0].text
             dict_['hint'] = []
             dict_['goal'] = query
@@ -1642,56 +1554,6 @@ class auto_analyst_ind(dspy.Module):
             logger.log_message(f"[MULTI] Error executing multiple agents: {str(e)}", level=logging.ERROR)
             return {"response": f"Error executing multiple agents: {str(e)}"}
 
-class data_context_gen(dspy.Signature):
-    """
-        Generate a Python-friendly JSON structure that describes one or more datasets
-    loaded from Excel or CSV files. This helps the system understand the dataset
-    structure, semantics, and use cases.
-
-    The JSON should include:
-    - Dataset name and source (file or sheet)
-    - Dataset role (transactional or reference)
-    - Description or business purpose
-    - Column names with:
-        - Data type (string, int, float, date, etc.)
-        - Semantic role: identifier, attribute, category, measure, temporal
-    - Relationships to other datasets (optional, natural-language style)
-    - Common metrics (as formulas or derived fields)
-    - Example use cases
-
-    Example format:
-    {
-      "datasets": {
-        "sales_data": {
-          "source": "Sales_Data.csv",
-          "role": "transactional",
-          "description": "Sales transactions across regions and products.",
-          "columns": {
-            "order_id": {"type": "string", "role": "identifier"},
-            "order_date": {"type": "date", "role": "temporal"},
-            "region": {"type": "string", "role": "category"},
-            "product_id": {"type": "string", "role": "identifier"},
-            "quantity": {"type": "int", "role": "measure"},
-            "unit_price": {"type": "float", "role": "measure"}
-          },
-      "metrics": [
-            "revenue = quantity * unit_price"
-      ],
-      "use_cases": [
-            "Revenue trend analysis",
-            "Regional sales comparison"
-          ]
-        }
-      }
-    }
-
-    Column roles: identifier, attribute, category, measure, temporal
-    Dataset roles: transactional, reference
-
-    """
-    user_description = dspy.InputField(desc="User's description of the data, including relationships")
-    dataset_view = dspy.InputField(desc="Dataset name with sample head(5 rows) view")
-    data_context = dspy.OutputField(desc="Compact JSON describing DuckDB tables, columns, relationships, metrics and use cases")
 
 # This is the auto_analyst with planner
 class auto_analyst(dspy.Module):
@@ -1714,9 +1576,11 @@ class auto_analyst(dspy.Module):
                 for template_name, signature in template_signatures.items():
                     # For planner module, load all planner variants (including core planner agents)
                     # Skip only individual variants, not planner variants
+                    if template_name in ['planner_preprocessing_agent', 'planner_statistical_analytics_agent', 'planner_sk_learn_agent', 'planner_data_viz_agent']:
+                        continue
                         
                     # Add template agent to agents dict
-                    self.agents[template_name] = dspy.asyncify(dspy.Predict(signature))
+                    self.agents[template_name] = dspy.asyncify(dspy.ChainOfThought(signature))
                     
                     # Determine if this is a visualization agent based on database category
                     is_viz_agent = False
@@ -1773,67 +1637,76 @@ class auto_analyst(dspy.Module):
 
         # Load core planner agents based on user preferences (only planner variants for planner module)
         if len(self.agents) == 0 and user_id and db_session:
-            # try:
+            try:
                 # Get user preferences for core planner agents
-            from src.db.schemas.models import AgentTemplate, UserTemplatePreference
-            
-            # For planner module, use planner variants of core agents
-            core_planner_agent_names = ['planner_preprocessing_agent', 'planner_statistical_analytics_agent', 'planner_sk_learn_agent', 'planner_data_viz_agent']
-            
-            for agent_name in core_planner_agent_names:
-                # Check if user has enabled this core agent (check both planner and individual preferences)
-                template = db_session.query(AgentTemplate).filter(
-                    AgentTemplate.template_name == agent_name,
-                    AgentTemplate.is_active == True
-                ).first()
-                
-                if not template:
-                    logger.log_message(f"Core planner agent template '{agent_name}' not found in database", level=logging.WARNING)
-                    continue
-                
-                # Check user preference for this planner agent
-                preference = db_session.query(UserTemplatePreference).filter(
-                    UserTemplatePreference.user_id == user_id,
-                    UserTemplatePreference.template_id == template.template_id
-                ).first()
-                
-                # Core planner agents are enabled by default unless explicitly disabled
-                is_enabled = preference.is_enabled if preference else True
-                
-                if not is_enabled:
-                    continue
-                
-                # Skip if already loaded from template_signatures
-                if agent_name in self.agents:
-                    continue
-                
-                # Create dynamic signature for planner agent
-                signature = create_custom_agent_signature(
-                    template.template_name,
-                    template.description,
-                    template.prompt_template,
-                    template.category
-                )
-                
-                # Add to agents dict
-                self.agents[agent_name] = dspy.asyncify(dspy.Predict(signature))
+                from src.db.schemas.models import AgentTemplate, UserTemplatePreference
                 
-                # Set input fields based on signature (all planner agents need plan_instructions)
-                if 'data_viz' in agent_name.lower() or template.category == 'Data Visualization':
-                    self.agent_inputs[agent_name] = {'goal', 'dataset', 'styling_index', 'plan_instructions'}
-                else:
-                    self.agent_inputs[agent_name] = {'goal', 'dataset', 'plan_instructions'}
+                # For planner module, use planner variants of core agents
+                core_planner_agent_names = ['planner_preprocessing_agent', 'planner_statistical_analytics_agent', 'planner_sk_learn_agent', 'planner_data_viz_agent']
                 
-                # Get description from database
-                description = f"Planner: {template.description}"
-                self.agent_desc.append({agent_name: description})
-                logger.log_message(f"Loaded core planner agent: {agent_name}", level=logging.DEBUG)
+                for agent_name in core_planner_agent_names:
+                    # Check if user has enabled this core agent (check both planner and individual preferences)
+                    template = db_session.query(AgentTemplate).filter(
+                        AgentTemplate.template_name == agent_name,
+                        AgentTemplate.is_active == True
+                    ).first()
+                    
+                    if not template:
+                        logger.log_message(f"Core planner agent template '{agent_name}' not found in database", level=logging.WARNING)
+                        continue
+                    
+                    # Check user preference for this planner agent
+                    preference = db_session.query(UserTemplatePreference).filter(
+                        UserTemplatePreference.user_id == user_id,
+                        UserTemplatePreference.template_id == template.template_id
+                    ).first()
+                    
+                    # Core planner agents are enabled by default unless explicitly disabled
+                    is_enabled = preference.is_enabled if preference else True
+                    
+                    if not is_enabled:
+                        continue
+                    
+                    # Skip if already loaded from template_signatures
+                    if agent_name in self.agents:
+                        continue
                     
+                    # Create dynamic signature for planner agent
+                    signature = create_custom_agent_signature(
+                        template.template_name,
+                        template.description,
+                        template.prompt_template,
+                        template.category
+                    )
+                    
+                    # Add to agents dict
+                    self.agents[agent_name] = dspy.asyncify(dspy.ChainOfThought(signature))
+                    
+                    # Set input fields based on signature (all planner agents need plan_instructions)
+                    if 'data_viz' in agent_name.lower() or template.category == 'Data Visualization':
+                        self.agent_inputs[agent_name] = {'goal', 'dataset', 'styling_index', 'plan_instructions'}
+                    else:
+                        self.agent_inputs[agent_name] = {'goal', 'dataset', 'plan_instructions'}
+                    
+                    # Get description from database
+                    description = f"Planner: {template.description}"
+                    self.agent_desc.append({agent_name: description})
+                    logger.log_message(f"Loaded core planner agent: {agent_name}", level=logging.DEBUG)
+                    
+            except Exception as e:
+                logger.log_message(f"Error loading core planner agents based on preferences: {str(e)}", level=logging.ERROR)
                 # Don't fallback - user must explicitly enable agents
+        elif len(self.agents) == 0:
+            # If no user_id/db_session provided and no agents loaded, this indicates a configuration issue
+            logger.log_message("No agents loaded and no user preferences available - check configuration", level=logging.ERROR)
         else:
-            self._load_default_planner_agents_fallback()
             # Load standard agents from provided list (legacy support)
-
+            for i, a in enumerate(agents):
+                name = a.__pydantic_core_schema__['schema']['model_name']
+                self.agents[name] = dspy.asyncify(dspy.ChainOfThought(a))
+                self.agent_inputs[name] = {x.strip() for x in str(agents[i].__pydantic_core_schema__['cls']).split('->')[0].split('(')[1].split(',')}
+                logger.log_message(f"Added agent: {name}, inputs: {self.agent_inputs[name]}", level=logging.DEBUG)
+                self.agent_desc.append({name: get_agent_description(name)})
 
         self.agents['basic_qa_agent'] = dspy.asyncify(dspy.Predict("goal->answer")) 
         self.agent_inputs['basic_qa_agent'] = {"goal"}
@@ -1844,7 +1717,7 @@ class auto_analyst(dspy.Module):
         # self.memory_summarize_agent = dspy.ChainOfThought(m.memory_summarize_agent)
                 
         # Initialize retrievers
-        self.dataset = retrievers['dataframe_index']
+        self.dataset = retrievers['dataframe_index'].as_retriever(k=1)
         self.styling_index = retrievers['style_index'].as_retriever(similarity_top_k=1)
         
         # Store user_id for usage tracking
@@ -1870,7 +1743,7 @@ class auto_analyst(dspy.Module):
                 agent_signature = data_viz_agent
             
             # Add to agents dict
-            self.agents[agent_name] = dspy.asyncify(dspy.Predict(agent_signature))
+            self.agents[agent_name] = dspy.asyncify(dspy.ChainOfThought(agent_signature))
             
             # Set input fields based on signature
             if agent_name == 'data_viz_agent':
@@ -1988,82 +1861,86 @@ class auto_analyst(dspy.Module):
         except Exception as e:
             logger.log_message(f"Error in _track_agent_usage for {agent_name}: {str(e)}", level=logging.ERROR)
 
-
+    async def execute_agent(self, agent_name, inputs):
+        """Execute a single agent with given inputs"""
+        
+        try:
+            result = await self.agents[agent_name.strip()](**inputs)
+            
+            # Track usage for custom agents and templates
+            await self._track_agent_usage(agent_name.strip())
+            
+            logger.log_message(f"Agent {agent_name} execution completed", level=logging.DEBUG)
+            return agent_name.strip(), dict(result)
+        except Exception as e:
+            logger.log_message(f"Error in execute_agent for {agent_name}: {str(e)}", level=logging.ERROR)
+            return agent_name.strip(), {"error": str(e)}
 
     async def get_plan(self, query):
         """Get the analysis plan"""
         dict_ = {}
-        dict_['dataset'] = self.dataset
+        dict_['dataset'] = self.dataset.retrieve(query)[0].text
         dict_['styling_index'] = self.styling_index.retrieve(query)[0].text
         dict_['goal'] = query
         dict_['Agent_desc'] = str(self.agent_desc)
         
         
-        module_return = await self.planner(
-            goal=dict_['goal'], 
-            dataset=dict_['dataset'], 
-            Agent_desc=dict_['Agent_desc']
-        )
-
-        logger.log_message(f"Module return: {module_return}", level=logging.INFO)
-        
-        # Add None check before accessing dictionary keys
-        if module_return is None:
-            logger.log_message("Planner returned None, returning error response", level=logging.ERROR)
-            return {
-                "plan": "There was an error" + str(dict_) +'\n'+ str(dspy.inspect_history()) +'\n agent_desc_len'+ str(len(self.agent_desc)) + '\n agents_len'+ str(len(self.agents)),
-                "plan_instructions": {},
-                "complexity": "unknown",
-                "error": "Planner failed to generate a plan"
-            }
-
-        # Handle different plan formats
-        plan = module_return['plan']
-        logger.log_message(f"Plan from module_return: {plan}, type: {type(plan)}", level=logging.INFO)
-        
-        # If plan is a string (agent name), convert to proper format
-        if isinstance(plan, str):
-            if 'complexity' in module_return:
-                complexity = module_return['complexity']
-            else:
-                complexity = 'basic'
-                
-            plan_dict = {
-                'plan': plan,
-                'complexity': complexity
-            }
+        try:
+            module_return = await self.planner(
+                goal=dict_['goal'], 
+                dataset=dict_['dataset'], 
+                Agent_desc=dict_['Agent_desc']
+            )
+            logger.log_message(f"Module return: {module_return}", level=logging.INFO)
             
-            # Add plan_instructions if available
-            if 'plan_instructions' in module_return:
-                plan_dict['plan_instructions'] = module_return['plan_instructions']
-            else:
-                plan_dict['plan_instructions'] = {}
-        else:
-            # If plan is already a dict, use it directly
-            plan_dict = dict(plan) if not isinstance(plan, dict) else plan
-            if 'complexity' in module_return:
-                complexity = module_return['complexity']
-            else:
-                complexity = 'basic'
-            plan_dict['complexity'] = complexity
+            # Handle different plan formats
+            plan = module_return['plan']
+            logger.log_message(f"Plan from module_return: {plan}, type: {type(plan)}", level=logging.INFO)
             
-        logger.log_message(f"Final plan dict: {plan_dict}", level=logging.INFO)
+            # If plan is a string (agent name), convert to proper format
+            if isinstance(plan, str):
+                if 'complexity' in module_return:
+                    complexity = module_return['complexity']
+                else:
+                    complexity = 'basic'
+                    
+                plan_dict = {
+                    'plan': plan,
+                    'complexity': complexity
+                }
+                
+                # Add plan_instructions if available
+                if 'plan_instructions' in module_return:
+                    plan_dict['plan_instructions'] = module_return['plan_instructions']
+                else:
+                    plan_dict['plan_instructions'] = {}
+            else:
+                # If plan is already a dict, use it directly
+                plan_dict = dict(plan) if not isinstance(plan, dict) else plan
+                if 'complexity' in module_return:
+                    complexity = module_return['complexity']
+                else:
+                    complexity = 'basic'
+                plan_dict['complexity'] = complexity
+                
+            logger.log_message(f"Final plan dict: {plan_dict}", level=logging.INFO)
 
-        return plan_dict
+            return plan_dict
             
-        # except Exception as e:
-        #     logger.log_message(f"Error in get_plan: {str(e)}", level=logging.ERROR)
-        #     raise
+        except Exception as e:
+            logger.log_message(f"Error in get_plan: {str(e)}", level=logging.ERROR)
+            raise
 
     async def execute_plan(self, query, plan):
         """Execute the plan and yield results as they complete"""
         
         dict_ = {}
-        dict_['dataset'] = self.dataset
+        dict_['dataset'] = self.dataset.retrieve(query)[0].text
         dict_['styling_index'] = self.styling_index.retrieve(query)[0].text
         dict_['hint'] = []
         dict_['goal'] = query
         
+        import json
 
         # Clean and split the plan string into agent names
         plan_text = plan.get("plan", "").lower().replace("plan:", "").strip()
@@ -2071,20 +1948,11 @@ class auto_analyst(dspy.Module):
         
         if "basic_qa_agent" in plan_text:
             inputs = dict(goal=query)
-
-            response = await self.agents['basic_qa_agent'](**inputs)
-            yield 'basic_qa_agent', inputs, response 
+            agent_name, response = await self.execute_agent('basic_qa_agent', inputs)
+            yield agent_name, inputs, response
             return 
-            
-
-        plan_list = []
-        for agent in [a.strip() for a in plan_text.split("->") if a.strip()]:
-            if not agent.startswith("planner_"):
-                agent = "planner_" + agent
-            plan_list.append(agent)
-
-        
 
+        plan_list = [agent.strip() for agent in plan_text.split("->") if agent.strip()]
         logger.log_message(f"Plan list: {plan_list}", level=logging.INFO)
         # Parse the attached plan_instructions into a dict
         raw_instr = plan.get("plan_instructions", {})
@@ -2101,14 +1969,15 @@ class auto_analyst(dspy.Module):
 
 
         # Check if we have no valid agents to execute
-        if not plan_list:
-            if len(plan_text) != 0: 
-                yield "plan_not_formatted_correctly", str(plan_text), {'error': "There was a error in the formatting"}
-
+        if not plan_list or all(agent not in self.agents for agent in plan_list):
+            yield "plan_not_found", None, {"error": "No valid agents found in plan"}
             return
-
+        
         # Execute agents in sequence
         for agent_name in plan_list:
+            if agent_name not in self.agents:
+                yield agent_name, {}, {"error": f"Agent '{agent_name}' not available"}
+                continue
                         
             try:
                 # Prepare inputs for the agent
@@ -2116,25 +1985,18 @@ class auto_analyst(dspy.Module):
                 
                 # Add plan instructions if available for this agent
                 if agent_name in plan_instructions:
-                    inputs['plan_instructions'] = plan_instructions[agent_name]
+                    inputs['plan_instructions'] = json.dumps(plan_instructions[agent_name])
                 else:
                     inputs['plan_instructions'] = ""
                 
                 # logger.log_message(f"Agent inputs for {agent_name}: {inputs}", level=logging.INFO)
-            
-
-                result = await self.agents[agent_name.strip()](**inputs)
                 
-                # Track usage for custom agents and templates
-                await self._track_agent_usage(agent_name.strip())
                 # Execute the agent
-            
+                agent_result_name, response = await self.execute_agent(agent_name, inputs)
                 
-                yield agent_name, inputs, result
+                yield agent_result_name, inputs, response
                     
             except Exception as e:
-                logger.log_message(f"Error executing agent {agent_name}: {str(e)}", level=logging.ERROR)
-                yield agent_name, {}, {"error": f"Error executing {agent_name}: {str(e)}"}
-        return
-        
+                    logger.log_message(f"Error executing agent {agent_name}: {str(e)}", level=logging.ERROR)
+                    yield agent_name, {}, {"error": f"Error executing {agent_name}: {str(e)}"}
 

src/agents/deep_agents.py

@@ -105,30 +105,38 @@ def configure_plotly_no_display():
 # Call the configuration function immediately
 configure_plotly_no_display()
 
-logger = Logger("deep_agents", see_time=True, console_log=True, level=logging.DEBUG)
+logger = Logger("deep_agents", see_time=True, console_log=False)
 load_dotenv()
 
 class deep_questions(dspy.Signature):
     """
 You are a data analysis assistant.
+
 Your role is to take a user's high-level analytical goal and generate a set of deep, targeted follow-up questions. These questions should guide an analyst toward a more thorough understanding of the goal by encouraging exploration, segmentation, and causal reasoning.
+
 Instructions:
 - Generate up to 5 insightful, data-relevant questions.
 - Use the dataset structure to tailor your questions (e.g., look at the available columns, data types, and what kind of information they can reveal).
 - The questions should help the user decompose their analytic goal and explore it from multiple angles (e.g., time trends, customer segments, usage behavior, external factors, feedback).
 - Each question should be specific enough to guide actionable analysis or investigation.
 - Use a clear and concise style, but maintain depth.
+
 Inputs:
 - goal: The user's analytical goal or main question they want to explore
 - dataset_info: A description of the dataset the user is querying, including:
     - What the dataset represents
     - Key columns and their data types
+
 Output:
 - deep_questions: A list of up to 5 specific, data-driven questions that support the analytic goal
+
 ---
+
 Example:
+
 Analytical Goal:
 Understand why churn has been rising
+
 Dataset Info:
 Customer Retention Dataset tracking subscription activity over time.  
 Columns:
@@ -142,38 +150,46 @@ Columns:
 - avg_weekly_logins (float)
 - support_tickets_last_30d (int)
 - satisfaction_score (float, 0–10 scale)
+
 Decomposed Questions:
 1. How has the churn rate changed month-over-month, and during which periods was the increase most pronounced?
 2. Are specific plan types or regions showing a higher churn rate relative to others?
 3. What is the average satisfaction score and support ticket count among churned users compared to retained users?
 4. Do churned users exhibit different login behavior (e.g., avg_weekly_logins) in the weeks leading up to their churn date?
 5. What is the tenure distribution (time from join_date to churn_date) among churned customers, and are short-tenure users more likely to churn?
+
     """
     goal = dspy.InputField(desc="User analytical goal — what main insight or question they want to answer")
     dataset_info = dspy.InputField(desc="A description of the dataset: what it represents, and the main columns with data types")
-    deep_questions:str = dspy.OutputField(desc="A list of up to five questions that help deeply explore the analytical goal using the dataset")
+    deep_questions = dspy.OutputField(desc="A list of up to five questions that help deeply explore the analytical goal using the dataset")
 
 class deep_synthesizer(dspy.Signature):
     """
 You are a data analysis synthesis expert.
+
 Your job is to take the outputs from a multi-agent data analytics system - including the original user query, the code summaries from each agent, and the actual printed results from running those code blocks - and synthesize them into a comprehensive, well-structured final report.
+
 This report should:
 - Explain what steps were taken and why (based on the query)
 - Summarize the code logic used by each agent, without including raw code
 - Highlight key findings and results from the code outputs
 - Offer clear, actionable insights tied back to the user's original question
 - Be structured, readable, and suitable for decision-makers or analysts
+
 Instructions:
 - Begin with a brief restatement of the original query and what it aimed to solve
 - Organize your report step-by-step or by analytical theme (e.g., segmentation, trend analysis, etc.)
 - For each part, summarize what was analyzed, how (based on code summaries), and what the result was (based on printed output)
 - End with a final set of synthesized conclusions and potential next steps or recommendations
+
 Inputs:
 - query: The user's original analytical question or goal
 - summaries: A list of natural language descriptions of what each agent's code did
 - print_outputs: A list of printed outputs (results) from running each agent's code
+
 Output:
 - synthesized_report: A structured and readable report that ties all parts together, grounded in the code logic and results
+
 Example use:
 You are not just summarizing outputs - you're telling a story that answers the user's query using real data.
     """
@@ -183,13 +199,13 @@ You are not just summarizing outputs - you're telling a story that answers the u
     print_outputs = dspy.InputField(desc="List of print outputs - the actual data insights generated by the code")
     synthesized_report = dspy.OutputField(desc="The final, structured report that synthesizes all the information into clear insights")
 
-def clean_and_store_code(code, session_datasets=None):
+def clean_and_store_code(code, session_df=None):
     """
     Cleans and stores code execution results in a standardized format.
     
     Args:
         code (str): Raw code text to execute
-        session_datasets (dict): Dictionary of datasets from session state
+        session_df (DataFrame): Optional session DataFrame
         
     Returns:
         dict: Execution results containing printed_output, plotly_figs, and error info
@@ -202,10 +218,9 @@ def clean_and_store_code(code, session_datasets=None):
     from plotly.subplots import make_subplots
     import plotly.io as pio
     
-    # Make all session datasets available globally if provided
-    if session_datasets is not None:
-        for dataset_name, dataset_df in session_datasets.items():
-            globals()[dataset_name] = dataset_df
+    # Make session DataFrame available globally if provided
+    if session_df is not None:
+        globals()['df'] = session_df
     
     # Initialize output containers
     output_dict = {
@@ -287,9 +302,10 @@ def clean_and_store_code(code, session_datasets=None):
         }
         
         # Add session DataFrame if available
-        if session_datasets is not None:
-            for dataset_name, dataset_df in session_datasets.items():
-                exec_globals[dataset_name] = dataset_df
+        if session_df is not None:
+            exec_globals['df'] = session_df
+        elif 'df' in globals():
+            exec_globals['df'] = globals()['df']
         
         # Add other common libraries that might be needed
         try:
@@ -337,7 +353,7 @@ def clean_and_store_code(code, session_datasets=None):
         
     return output_dict
 
-def score_code(args, code, datasets=None):
+def score_code(args, code):
     """
     Cleans and stores code execution results in a standardized format.
     Safely handles execution errors and returns clean output even if execution fails.
@@ -346,7 +362,6 @@ def score_code(args, code, datasets=None):
     Args:
         args: Arguments (unused but required for dspy.Refine)
         code: Code object with combined_code attribute
-        datasets: Dictionary of datasets from session state (optional)
         
     Returns:
         int: Score (0=error, 1=success, 2=success with plots)
@@ -384,34 +399,16 @@ def score_code(args, code, datasets=None):
         cleaned_code = re.sub(r'\w+_fig\w*\.show\(\s*[^)]*\s*\)', '', cleaned_code)  # *_fig*.show(any_args)
             
         cleaned_code = remove_main_block(cleaned_code)
-        
         # Capture stdout using StringIO
         from io import StringIO
         import sys
         import plotly.graph_objects as go
-        import pandas as pd
-        import numpy as np
-        
         stdout_capture = StringIO()
         original_stdout = sys.stdout
         sys.stdout = stdout_capture
         
-        # Execute code in a new namespace with datasets available
+        # Execute code in a new namespace to avoid polluting globals
         local_vars = {}
-        
-        # Add datasets to the execution context if provided
-        if datasets:
-            local_vars.update(datasets)
-        
-        # Add common imports to the execution context
-        local_vars.update({
-            'pd': pd,
-            'np': np,
-            'go': go,
-            'plt': __import__('matplotlib.pyplot'),
-            'sns': __import__('seaborn'),
-        })
-        
         exec(cleaned_code, globals(), local_vars)
         
         # Capture any plotly figures from local namespace
@@ -455,6 +452,7 @@ class deep_planner(dspy.Signature):
     """
     You are an advanced multi-question planning agent. Your task is to generate the most optimized and minimal plan
     to answer up to 5 analytical questions using available agents.
+
     Your responsibilities:
     1. Feasibility: Verify that the goal is achievable using the provided datasets and agent descriptions.
     2. Optimization: 
@@ -466,10 +464,12 @@ class deep_planner(dspy.Signature):
        - Define clear variable usage (create/use).
        - Specify concise step-by-step instructions per agent.
        - Use dependency arrows (->) to indicate required agent outputs used by others.
+
     Inputs:
     - deep_questions: A list of up to 5 deep analytical questions (e.g., ["q1", "q2", ..., "q5"])
     - dataset: The available dataset(s) in memory or context
     - agents_desc: Dictionary containing each agent's name and its capabilities or descriptions
+
     Outputs:
     - plan_instructions: Detailed per-agent variable flow and functionality in the format:
         {
@@ -484,6 +484,7 @@ class deep_planner(dspy.Signature):
                 "instruction": "Perform correlation analysis to identify strong predictors."
             }
         }
+
     Output Goal:
     Generate a small, clean, optimized execution plan using minimal agent calls, reusable outputs, and well-structured dependencies.
     USE THE EXACT NAME OF THE AGENTS IN THE INSTRUCTIONS
@@ -498,6 +499,7 @@ class deep_plan_fixer(dspy.Signature):
     """
     You are a plan instruction fixer agent. Your task is to take potentially malformed plan instructions
     and convert them into a properly structured dictionary format that can be safely evaluated.
+
     Your responsibilities:
     1. Parse and validate the input plan instructions
     2. Convert the instructions into a proper dictionary format
@@ -511,8 +513,10 @@ class deep_plan_fixer(dspy.Signature):
        }
     4. Handle any malformed or missing components
     5. Return a properly formatted dictionary string that can be safely evaluated
+
     Inputs:
     - plan_instructions: The potentially malformed plan instructions to fix
+
     Outputs:
     - fixed_plan: A properly formatted dictionary string that can be safely evaluated
     """
@@ -523,32 +527,43 @@ class deep_plan_fixer(dspy.Signature):
 class final_conclusion(dspy.Signature):
     """
 You are a high-level analytics reasoning engine.
+
 Your task is to take multiple synthesized analytical results (each answering part of the original query) and produce a cohesive final conclusion that directly addresses the user's original question.
+
 This is not just a summary — it's a judgment. Use evidence from the synthesized findings to:
 - Answer the original question with clarity
 - Highlight the most important insights
 - Offer any causal reasoning or patterns discovered
 - Suggest next steps or strategic recommendations where appropriate
+
 Instructions:
 - Focus on relevance to the original query
 - Do not just repeat what the synthesized sections say — instead, infer, interpret, and connect dots
 - Prioritize clarity and insight over detail
 - End with a brief "Next Steps" section if applicable
+
 Inputs:
 - query: The original user question or goal
 - synthesized_sections: A list of synthesized result sections from the deep_synthesizer step (each covering part of the analysis)
+
 Output:
 - final_summary: A cohesive final conclusion that addresses the query, draws insight, and offers high-level guidance
+
 ---
+
 Example Output Structure:
+
 **Conclusion**  
 Summarize the overall answer to the user's question, using the most compelling evidence across the synthesized sections.
+
 **Key Takeaways**  
 - Bullet 1  
 - Bullet 2  
 - Bullet 3  
+
 **Recommended Next Steps**  
 (Optional based on context)
+
     """
 
     query = dspy.InputField(desc="The user's original query or analytical goal")
@@ -561,6 +576,7 @@ Summarize the overall answer to the user's question, using the most compelling e
 class deep_code_synthesizer(dspy.Signature):
     """
 You are a code synthesis and optimization engine that combines and fixes code from multiple analytical agents.
+
 Your task is to take code outputs from preprocessing, statistical analysis, machine learning, and visualization agents, then:
 - Combine them into a single, coherent analysis pipeline
 - Fix any errors or inconsistencies between agent outputs
@@ -575,6 +591,7 @@ Your task is to take code outputs from preprocessing, statistical analysis, mach
 - Ensure all visualizations use Plotly exclusively
 - Create comprehensive visualizations that show all important variables and relationships
 - Store all Plotly figures in a list for later use in the report
+
 Instructions:
 - Review each agent's code for correctness and completeness
 - Ensure variables are properly passed between steps with consistent types
@@ -593,13 +610,17 @@ Instructions:
 - Use consistent styling across all Plotly visualizations
 - DONOT COMMENT OUT ANYTHING AS THE CODE SHOULD RUN & SHOW OUTPUTS
 - THE DATASET IS ALREADY LOADED, DON'T CREATE FAKE DATA. 'df' is always loaded
+
 Inputs:
 - deep_questions- The five deep questions this system is answering
 - dataset_info - Information about the dataset structure and types
 - planner_instructions - the plan according to the planner, ensure that the final code makes everything coherent
 - code - List of all agent code
+
+
 Output:
 - combined_code: - A single, optimized Python script that combines all analysis steps with proper type handling and Plotly visualizations
+
 """
     deep_questions = dspy.InputField(desc="The five deep questions this system is answering")
     dataset_info = dspy.InputField(desc="Information about the dataset")
@@ -644,6 +665,7 @@ class deep_code_fix(dspy.Signature):
 
 chart_instructions = """
 Chart Styling Guidelines:
+
 1. General Styling:
    - Use a clean, professional color palette (e.g., Tableau, ColorBrewer)
    - Include clear titles and axis labels
@@ -651,6 +673,7 @@ Chart Styling Guidelines:
    - Use consistent font sizes and styles
    - Include grid lines where helpful
    - Add hover information for interactive plots
+
 2. Specific Chart Types:
    - Bar Charts:
      * Use horizontal bars for many categories
@@ -675,6 +698,7 @@ Chart Styling Guidelines:
      * Include value annotations
      * Sort rows/columns by similarity
      * Add clear color scale legend
+
 3. Data Visualization Best Practices:
    - Start axes at zero when appropriate
    - Use log scales for wide-ranging data
@@ -684,12 +708,14 @@ Chart Styling Guidelines:
    - Use consistent color encoding
    - Include data source and timestamp
    - Add clear figure captions
+
 4. Interactive Features:
    - Enable zooming and panning
    - Add tooltips with detailed information
    - Include download options
    - Allow toggling of data series
    - Enable cross-filtering between charts
+
 5. Accessibility:
    - Use colorblind-friendly palettes
    - Include alt text for all visualizations
@@ -700,46 +726,34 @@ Chart Styling Guidelines:
 
 
 
-
 class deep_analysis_module(dspy.Module):
     def __init__(self,agents, agents_desc):
-        # logger.log_message(f"Initializing deep_analysis_module with {agents} agents: {list(agents.keys())}", level=logging.INFO)
+        logger.log_message(f"Initializing deep_analysis_module with {agents} agents: {list(agents.keys())}", level=logging.INFO)
         
-        # self.agent = agents
-        self.agents = {}
-        for key in agents.keys():
-            self.agents[key] = dspy.asyncify(dspy.Predict(agents[key]))
-
-        # for sig in self.agen
+        self.agents = agents
         # Make all dspy operations async using asyncify
         self.deep_questions = dspy.asyncify(dspy.Predict(deep_questions))
-        self.deep_planner = dspy.asyncify(dspy.Predict(deep_planner))
-        self.deep_synthesizer = dspy.asyncify(dspy.Predict(deep_synthesizer))
+        self.deep_planner = dspy.asyncify(dspy.ChainOfThought(deep_planner))
+        self.deep_synthesizer = dspy.asyncify(dspy.ChainOfThought(deep_synthesizer))
         # Keep both asyncified and non-asyncified versions for code synthesizer
         self.deep_code_synthesizer_sync = dspy.Predict(deep_code_synthesizer)  # For dspy.Refine
         self.deep_code_synthesizer = dspy.asyncify(dspy.Predict(deep_code_synthesizer))  # For async use
-        self.deep_plan_fixer = dspy.asyncify(dspy.Predict(deep_plan_fixer))
-        self.deep_code_fixer = dspy.asyncify(dspy.Predict(deep_code_fix))
+        self.deep_plan_fixer = dspy.asyncify(dspy.ChainOfThought(deep_plan_fixer))
+        self.deep_code_fixer = dspy.asyncify(dspy.ChainOfThought(deep_code_fix))
         self.styling_instructions = chart_instructions
         self.agents_desc = agents_desc
         self.final_conclusion = dspy.asyncify(dspy.ChainOfThought(final_conclusion))
         
-        # logger.log_message(f"Deep analysis module initialized successfully with agents: {list(self.agents.keys())}", level=logging.INFO)
+        logger.log_message(f"Deep analysis module initialized successfully with agents: {list(self.agents.keys())}", level=logging.INFO)
 
-    async def execute_deep_analysis_streaming(self, goal, dataset_info, session_datasets=None):
+    async def execute_deep_analysis_streaming(self, goal, dataset_info, session_df=None):
         """
         Execute deep analysis with streaming progress updates.
         This is an async generator that yields progress updates incrementally.
         """
-        logger.log_message(f"🔍 DEEP ANALYSIS STREAMING START - goal: {goal[:100]}...", level=logging.DEBUG)
-        logger.log_message(f"🔍 DEEP ANALYSIS STREAMING START - dataset_info type: {type(dataset_info)}, length: {len(dataset_info) if isinstance(dataset_info, str) else 'N/A'}", level=logging.DEBUG)
-        logger.log_message(f"🔍 DEEP ANALYSIS STREAMING START - session_datasets type: {type(session_datasets)}, keys: {list(session_datasets.keys()) if session_datasets else 'None'}", level=logging.DEBUG)
-        
-        # Make all session datasets available globally for code execution
-        if session_datasets is not None:
-            for dataset_name, dataset_df in session_datasets.items():
-                globals()[dataset_name] = dataset_df
-                logger.log_message(f"🔍 MADE DATASET AVAILABLE GLOBALLY - {dataset_name}: shape {dataset_df.shape}", level=logging.DEBUG)
+        # Make the session DataFrame available globally for code execution
+        if session_df is not None:
+            globals()['df'] = session_df
         
         try:
             # Step 1: Generate deep questions (20% progress)
@@ -750,7 +764,6 @@ class deep_analysis_module(dspy.Module):
                 "progress": 10
             }
             
-            logger.log_message(f"🔍 CALLING DEEP_QUESTIONS - dataset_info type: {type(dataset_info)}, length: {len(dataset_info) if isinstance(dataset_info, str) else 'N/A'}", level=logging.DEBUG)
             questions = await self.deep_questions(goal=goal, dataset_info=dataset_info)
             logger.log_message("Questions generated")
             
@@ -770,7 +783,6 @@ class deep_analysis_module(dspy.Module):
             }
             
             question_list = [q.strip() for q in questions.deep_questions.split('\n') if q.strip()]
-            logger.log_message(f" CALLING DEEP_PLANNER - dataset_info type: {type(dataset_info)}, length: {len(dataset_info) if isinstance(dataset_info, str) else 'N/A'}", level=logging.DEBUG)
             deep_plan = await self.deep_planner(
                 deep_questions=questions.deep_questions, 
                 dataset=dataset_info, 
@@ -796,6 +808,8 @@ class deep_analysis_module(dspy.Module):
                     if not isinstance(plan_instructions, dict):
                         plan_instructions = json.loads(deep_plan.fixed_plan)
                     keys = [key for key in plan_instructions.keys()]
+                    logger.log_message(f"Plan instructions fixed: {plan_instructions}", logging.INFO)
+                    logger.log_message(f"Keys: {keys}", logging.INFO)
                 except (ValueError, SyntaxError, json.JSONDecodeError) as e:
                     logger.log_message(f"Error parsing plan instructions: {e}", logging.ERROR)
                     raise e
@@ -817,42 +831,22 @@ class deep_analysis_module(dspy.Module):
                 "progress": 45
             }
             
-            logger.log_message(f"🔍 CREATING DSPY EXAMPLES - dataset_info type: {type(dataset_info)}, length: {len(dataset_info) if isinstance(dataset_info, str) else 'N/A'}", level=logging.DEBUG)
-
-
-            # tasks = [self.agents[key](**q) for q, key in zip(queries, keys)]
-            tasks = []
-            for key in  keys:
-                logger.log_message(f"Preparing agent task for key: {key}", level=logging.DEBUG)
-                logger.log_message(f"Plan instructions for {key}: {plan_instructions[key]}", level=logging.DEBUG)
-                if "data_viz" in key or "viz" in key.lower() or "visual" in key.lower() or "plot" in key.lower() or "chart" in key.lower():
-                    logger.log_message(f"Agent {key} identified as visualization agent. Adding styling_index.", level=logging.DEBUG)
-                    task = self.agents[key](
-                        goal=questions.deep_questions,
-                        dataset=dataset_info,
-                        plan_instructions=str(plan_instructions[key]),
-                        styling_index=self.styling_instructions
-                    )
-                    logger.log_message(f"Visualization agent task for {key} created.", level=logging.DEBUG)
-                    tasks.append(task)
-                else:
-                    logger.log_message(f"Agent {key} identified as non-visualization agent.", level=logging.DEBUG)
-                    task = self.agents[key](
-                        goal=questions.deep_questions,
-                        dataset=dataset_info,
-                        plan_instructions=str(plan_instructions[key])
-                    )
-                    logger.log_message(f"Non-visualization agent task for {key} created.", level=logging.DEBUG)
-                    tasks.append(task)
-
-
-
-
-
-
-            logger.log_message("Passed that stage")
-            # DEBUG: Log what parameters each agent will receive
-                  
+            queries = [
+                dspy.Example(
+                    goal=questions.deep_questions,
+                    dataset=dataset_info,
+                    plan_instructions=str(plan_instructions[key]),
+                    **({"styling_index": "Sample styling guidelines"} if "data_viz" in key or "viz" in key.lower() or "visual" in key.lower() or "plot" in key.lower() or "chart" in key.lower() else {})
+                ).with_inputs(
+                    "goal",
+                    "dataset", 
+                    "plan_instructions",
+                    *(["styling_index"] if "data_viz" in key or "viz" in key.lower() or "visual" in key.lower() or "plot" in key.lower() or "chart" in key.lower() else [])
+                )
+                for key in keys
+            ]
+            tasks = [self.agents[key](**q) for q, key in zip(queries, keys)]
+                        
             # Await all tasks to complete
             summaries = []
             codes = []
@@ -909,25 +903,7 @@ class deep_analysis_module(dspy.Module):
                     code.append(c.replace('try\n','try:\n'))
             
             # Create deep coder without asyncify to avoid source inspection issues
-            def create_score_code_with_datasets(datasets):
-                """
-                Creates a score function that includes the session datasets.
-                
-                Args:
-                    datasets: Dictionary of datasets from session_state['datasets']
-                    
-                Returns:
-                    A reward function compatible with dspy.Refine
-                """
-                def score_code_with_datasets(args, pred):
-                    return score_code(args, pred, datasets)  # FIXED: Use positional argument
-                
-                return score_code_with_datasets
-
-            # Then in your deep analysis method:
-            # Create score function with datasets
-            score_fn = create_score_code_with_datasets(self.datasets)
-            deep_coder = dspy.Refine(module=self.deep_code_synthesizer_sync, N=5, reward_fn=score_fn, threshold=1.0, fail_count=10)
+            deep_coder = dspy.Refine(module=self.deep_code_synthesizer_sync, N=5, reward_fn=score_code, threshold=1.0, fail_count=10)
             
             # Check if we have valid API key
             anthropic_key = os.environ.get('ANTHROPIC_API_KEY')
@@ -936,7 +912,7 @@ class deep_analysis_module(dspy.Module):
             
             try:
                 # Create the LM instance that will be used
-                thread_lm = dspy.LM("anthropic/claude-sonnet-4-6", api_key=anthropic_key, max_tokens=17000)
+                thread_lm = dspy.LM("anthropic/claude-sonnet-4-20250514", api_key=anthropic_key, max_tokens=17000)
                 
                 logger.log_message("Starting code generation...")
                 start_time = datetime.datetime.now()
@@ -986,7 +962,7 @@ class deep_analysis_module(dspy.Module):
                 # Run code execution in thread pool to avoid blocking
                 import concurrent.futures
                 with concurrent.futures.ThreadPoolExecutor() as executor:
-                    future = executor.submit(clean_and_store_code, code, session_datasets)
+                    future = executor.submit(clean_and_store_code, code, session_df)
                     output = future.result(timeout=300)  # 5 minute timeout
                 
                 logger.log_message(f"Deep Code executed")

src/agents/retrievers/retrievers.py

@@ -52,8 +52,9 @@ def correct_num(df,c):
 # does most of the pre-processing
 def make_data(df, desc):
     dict_ = {}
-    dict_['dataset_python_name'] = "The data is loaded as df"       
+    dict_['df_name'] = "The data is loaded as df"
     dict_['Description'] = desc
+    dict_['dataframe_head_view'] = df.head(2).to_markdown()
     # dict_['all_column_names'] = str(list(df.columns[:20]))
 
     # for c in df.columns:
@@ -65,3 +66,88 @@ def make_data(df, desc):
     return dict_
 
 
+
+# These are stored styling instructions for data_viz_agent, helps generate good graphs
+styling_instructions =[
+    """
+        Dont ignore any of these instructions.
+        For a line chart always use plotly_white template, reduce x axes & y axes line to 0.2 & x & y grid width to 1. 
+        Always give a title and make bold using html tag axis label and try to use multiple colors if more than one line
+        Annotate the min and max of the line
+        Display numbers in thousand(K) or Million(M) if larger than 1000/100000 
+        Show percentages in 2 decimal points with '%' sign
+        Default size of chart should be height =1200 and width =1000
+        
+        """
+        
+   , """
+        Dont ignore any of these instructions.
+        For a bar chart always use plotly_white template, reduce x axes & y axes line to 0.2 & x & y grid width to 1. 
+        Always give a title and make bold using html tag axis label 
+        Always display numbers in thousand(K) or Million(M) if larger than 1000/100000. 
+        Annotate the values of the bar chart
+        If variable is a percentage show in 2 decimal points with '%' sign.
+        Default size of chart should be height =1200 and width =1000
+        """
+        ,
+
+          """
+        For a histogram chart choose a bin_size of 50
+        Do not ignore any of these instructions
+        always use plotly_white template, reduce x & y axes line to 0.2 & x & y grid width to 1. 
+        Always give a title and make bold using html tag axis label 
+        Always display numbers in thousand(K) or Million(M) if larger than 1000/100000. Add annotations x values
+        If variable is a percentage show in 2 decimal points with '%'
+        Default size of chart should be height =1200 and width =1000
+        """,
+
+
+          """
+        For a pie chart only show top 10 categories, bundle rest as others
+        Do not ignore any of these instructions
+        always use plotly_white template, reduce x & y axes line to 0.2 & x & y grid width to 1. 
+        Always give a title and make bold using html tag axis label 
+        Always display numbers in thousand(K) or Million(M) if larger than 1000/100000. Add annotations x values
+        If variable is a percentage show in 2 decimal points with '%'
+        Default size of chart should be height =1200 and width =1000
+        """,
+
+          """
+        Do not ignore any of these instructions
+        always use plotly_white template, reduce x & y axes line to 0.2 & x & y grid width to 1. 
+        Always give a title and make bold using html tag axis label 
+        Always display numbers in thousand(K) or Million(M) if larger than 1000/100000. Add annotations x values
+        Don't add K/M if number already in , or value is not a number
+        If variable is a percentage show in 2 decimal points with '%'
+        Default size of chart should be height =1200 and width =1000
+        """,
+"""
+    For a heat map
+    Use the 'plotly_white' template for a clean, white background. 
+    Set a chart title 
+    Style the X-axis with a black line color, 0.2 line width, 1 grid width, format 1000/1000000 as K/M
+    Do not format non-numerical numbers 
+    .style the Y-axis with a black line color, 0.2 line width, 1 grid width format 1000/1000000 as K/M
+    Do not format non-numerical numbers 
+
+    . Set the figure dimensions to a height of 1200 pixels and a width of 1000 pixels.
+""",
+"""
+    For a Histogram, used for returns/distribution plotting
+    Use the 'plotly_white' template for a clean, white background. 
+    Set a chart title 
+    Style the X-axis  1 grid width, format 1000/1000000 as K/M
+    Do not format non-numerical numbers 
+    .style the Y-axis, 1 grid width format 1000/1000000 as K/M
+    Do not format non-numerical numbers 
+    
+    Use an opacity of 0.75
+
+     Set the figure dimensions to a height of 1200 pixels and a width of 1000 pixels.
+"""
+       
+         ]
+
+
+
+

src/db/init_default_agents.py

@@ -0,0 +1,281 @@
+"""
+Initialize default agents in the database.
+This module should be run during application startup to ensure
+default agents are available in the database.
+"""
+
+import logging
+from datetime import datetime, UTC
+from src.utils.logger import Logger
+
+# Initialize logger
+logger = Logger("init_default_agents", see_time=True, console_log=False)
+
+def load_default_agents_to_db(db_session, force_update=False):
+    """
+    Load the default agents into the AgentTemplate table.
+    
+    Args:
+        db_session: Database session
+        force_update: If True, update existing agents. If False, skip existing ones.
+    
+    Returns:
+        Tuple (success: bool, message: str)
+    """
+    try:
+        from src.db.schemas.models import AgentTemplate
+        
+        # Define default agents with their signatures and metadata
+        default_agents = {
+            "preprocessing_agent": {
+                "display_name": "Data Preprocessing Agent",
+                "description": "Cleans and prepares a DataFrame using Pandas and NumPy—handles missing values, detects column types, and converts date strings to datetime.",
+                "prompt_template": """You are a AI data-preprocessing agent. Generate clean and efficient Python code using NumPy and Pandas to perform introductory data preprocessing on a pre-loaded DataFrame df, based on the user's analysis goals.
+Preprocessing Requirements:
+1. Identify Column Types
+- Separate columns into numeric and categorical using:
+    categorical_columns = df.select_dtypes(include=[object, 'category']).columns.tolist()
+    numeric_columns = df.select_dtypes(include=[np.number]).columns.tolist()
+2. Handle Missing Values
+- Numeric columns: Impute missing values using the mean of each column
+- Categorical columns: Impute missing values using the mode of each column
+3. Convert Date Strings to Datetime
+- For any column suspected to represent dates (in string format), convert it to datetime using:
+    def safe_to_datetime(date):
+        try:
+            return pd.to_datetime(date, errors='coerce', cache=False)
+        except (ValueError, TypeError):
+            return pd.NaT
+    df['datetime_column'] = df['datetime_column'].apply(safe_to_datetime)
+- Replace 'datetime_column' with the actual column names containing date-like strings
+Important Notes:
+- Do NOT create a correlation matrix — correlation analysis is outside the scope of preprocessing
+- Do NOT generate any plots or visualizations
+Output Instructions:
+1. Include the full preprocessing Python code
+2. Provide a brief bullet-point summary of the steps performed. Example:
+• Identified 5 numeric and 4 categorical columns
+• Filled missing numeric values with column means
+• Filled missing categorical values with column modes
+• Converted 1 date column to datetime format
+ Respond in the user's language for all summary and reasoning but keep the code in english""",
+                "category": "Data Manipulation",
+                "icon_url": "https://cdn.jsdelivr.net/gh/devicons/devicon/icons/pandas/pandas-original.svg"
+            },
+            "statistical_analytics_agent": {
+                "display_name": "Statistical Analytics Agent",
+                "description": "Performs statistical analysis (e.g., regression, seasonal decomposition) using statsmodels, with proper handling of categorical data and missing values.",
+                "prompt_template": """ 
+You are a statistical analytics agent. Your task is to take a dataset and a user-defined goal and output Python code that performs the appropriate statistical analysis to achieve that goal. Follow these guidelines:
+IMPORTANT: You may be provided with previous interaction history. The section marked "### Current Query:" contains the user's current request. Any text in "### Previous Interaction History:" is for context only and is NOT part of the current request.
+Data Handling:
+Always handle strings as categorical variables in a regression using statsmodels C(string_column).
+Do not change the index of the DataFrame.
+Convert X and y into float when fitting a model.
+Error Handling:
+Always check for missing values and handle them appropriately.
+Ensure that categorical variables are correctly processed.
+Provide clear error messages if the model fitting fails.
+Regression:
+For regression, use statsmodels and ensure that a constant term is added to the predictor using sm.add_constant(X).
+Handle categorical variables using C(column_name) in the model formula.
+Fit the model with model = sm.OLS(y.astype(float), X.astype(float)).fit().
+Seasonal Decomposition:
+Ensure the period is set correctly when performing seasonal decomposition.
+Verify the number of observations works for the decomposition.
+Output:
+Ensure the code is executable and as intended.
+Also choose the correct type of model for the problem
+Avoid adding data visualization code.
+Use code like this to prevent failing:
+import pandas as pd
+import numpy as np
+import statsmodels.api as sm
+def statistical_model(X, y, goal, period=None):
+    try:
+        # Check for missing values and handle them
+        X = X.dropna()
+        y = y.loc[X.index].dropna()
+        # Ensure X and y are aligned
+        X = X.loc[y.index]
+        # Convert categorical variables
+        for col in X.select_dtypes(include=['object', 'category']).columns:
+            X[col] = X[col].astype('category')
+        # Add a constant term to the predictor
+        X = sm.add_constant(X)
+        # Fit the model
+        if goal == 'regression':
+            # Handle categorical variables in the model formula
+            formula = 'y ~ ' + ' + '.join([f'C({col})' if X[col].dtype.name == 'category' else col for col in X.columns])
+            model = sm.OLS(y.astype(float), X.astype(float)).fit()
+            return model.summary()
+        elif goal == 'seasonal_decompose':
+            if period is None:
+                raise ValueError("Period must be specified for seasonal decomposition")
+            decomposition = sm.tsa.seasonal_decompose(y, period=period)
+            return decomposition
+        else:
+            raise ValueError("Unknown goal specified. Please provide a valid goal.")
+    except Exception as e:
+        return f"An error occurred: {e}"
+# Example usage:
+result = statistical_analysis(X, y, goal='regression')
+print(result)
+If visualizing use plotly
+Provide a concise bullet-point summary of the statistical analysis performed.
+
+Example Summary:
+• Applied linear regression with OLS to predict house prices based on 5 features
+• Model achieved R-squared of 0.78
+• Significant predictors include square footage (p<0.001) and number of bathrooms (p<0.01)
+• Detected strong seasonal pattern with 12-month periodicity
+• Forecast shows 15% growth trend over next quarter
+Respond in the user's language for all summary and reasoning but keep the code in english""",
+                "category": "Statistical Analysis",
+                "icon_url": "https://cdn.jsdelivr.net/gh/devicons/devicon/icons/statsmodels/statsmodels-original.svg"
+            },
+            "sk_learn_agent": {
+                "display_name": "Machine Learning Agent",
+                "description": "Trains and evaluates machine learning models using scikit-learn, including classification, regression, and clustering with feature importance insights.",
+                "prompt_template": """You are a machine learning agent. 
+Your task is to take a dataset and a user-defined goal, and output Python code that performs the appropriate machine learning analysis to achieve that goal. 
+You should use the scikit-learn library.
+IMPORTANT: You may be provided with previous interaction history. The section marked "### Current Query:" contains the user's current request. Any text in "### Previous Interaction History:" is for context only and is NOT part of the current request.
+Make sure your output is as intended!
+Provide a concise bullet-point summary of the machine learning operations performed.
+
+Example Summary:
+• Trained a Random Forest classifier on customer churn data with 80/20 train-test split
+• Model achieved 92% accuracy and 88% F1-score
+• Feature importance analysis revealed that contract length and monthly charges are the strongest predictors of churn
+• Implemented K-means clustering (k=4) on customer shopping behaviors
+• Identified distinct segments: high-value frequent shoppers (22%), occasional big spenders (35%), budget-conscious regulars (28%), and rare visitors (15%)
+Respond in the user's language for all summary and reasoning but keep the code in english""",
+                "category": "Modelling",
+                "icon_url": "https://cdn.jsdelivr.net/gh/devicons/devicon/icons/scikit-learn/scikit-learn-original.svg"
+            },
+            "data_viz_agent": {
+                "display_name": "Data Visualization Agent",
+                "description": "Generates interactive visualizations with Plotly, selecting the best chart type to reveal trends, comparisons, and insights based on the analysis goal.",
+                "prompt_template": """    
+You are an AI agent responsible for generating interactive data visualizations using Plotly.
+IMPORTANT Instructions:
+- The section marked "### Current Query:" contains the user's request. Any text in "### Previous Interaction History:" is for context only and should NOT be treated as part of the current request.
+- You must only use the tools provided to you. This agent handles visualization only.
+- If len(df) > 50000, always sample the dataset before visualization using:  
+if len(df) > 50000:  
+    df = df.sample(50000, random_state=1)
+- Each visualization must be generated as a **separate figure** using go.Figure().  
+Do NOT use subplots under any circumstances.
+- Each figure must be returned individually using:  
+fig.to_html(full_html=False)
+- Use update_layout with xaxis and yaxis **only once per figure**.
+- Enhance readability and clarity by:  
+• Using low opacity (0.4-0.7) where appropriate  
+• Applying visually distinct colors for different elements or categories  
+- Make sure the visual **answers the user's specific goal**:  
+• Identify what insight or comparison the user is trying to achieve  
+• Choose the visualization type and features (e.g., color, size, grouping) to emphasize that goal  
+• For example, if the user asks for "trends in revenue," use a time series line chart; if they ask for "top-performing categories," use a bar chart sorted by value  
+• Prioritize highlighting patterns, outliers, or comparisons relevant to the question
+- Never include the dataset or styling index in the output.
+- If there are no relevant columns for the requested visualization, respond with:  
+"No relevant columns found to generate this visualization."
+- Use only one number format consistently: either 'K', 'M', or comma-separated values like 1,000/1,000,000. Do not mix formats.
+- Only include trendlines in scatter plots if the user explicitly asks for them.
+- Output only the code and a concise bullet-point summary of what the visualization reveals.
+- Always end each visualization with:  
+fig.to_html(full_html=False)
+Respond in the user's language for all summary and reasoning but keep the code in english
+Example Summary:  
+• Created an interactive scatter plot of sales vs. marketing spend with color-coded product categories  
+• Included a trend line showing positive correlation (r=0.72)  
+• Highlighted outliers where high marketing spend resulted in low sales  
+• Generated a time series chart of monthly revenue from 2020-2023  
+• Added annotations for key business events  
+• Visualization reveals 35% YoY growth with seasonal peaks in Q4""",
+                "category": "Visualization",
+                "icon_url": "https://cdn.jsdelivr.net/gh/devicons/devicon/icons/plotly/plotly-original.svg"
+            }
+        }
+        
+        created_count = 0
+        updated_count = 0
+        
+        for template_name, agent_data in default_agents.items():
+            # Check if agent already exists
+            existing_agent = db_session.query(AgentTemplate).filter(
+                AgentTemplate.template_name == template_name
+            ).first()
+            
+            if existing_agent:
+                if force_update:
+                    # Update existing agent
+                    existing_agent.display_name = agent_data["display_name"]
+                    existing_agent.description = agent_data["description"]
+                    existing_agent.prompt_template = agent_data["prompt_template"]
+                    existing_agent.category = agent_data["category"]
+                    existing_agent.icon_url = agent_data["icon_url"]
+                    existing_agent.is_premium_only = False
+                    existing_agent.is_active = True
+                    existing_agent.updated_at = datetime.now(UTC)
+                    updated_count += 1
+                else:
+                    logger.log_message(f"Agent '{template_name}' already exists, skipping", level=logging.INFO)
+                    continue
+            else:
+                # Create new agent
+                new_agent = AgentTemplate(
+                    template_name=template_name,
+                    display_name=agent_data["display_name"],
+                    description=agent_data["description"],
+                    prompt_template=agent_data["prompt_template"],
+                    category=agent_data["category"],
+                    icon_url=agent_data["icon_url"],
+                    is_premium_only=False,
+                    is_active=True,
+                    created_at=datetime.now(UTC),
+                    updated_at=datetime.now(UTC)
+                )
+                db_session.add(new_agent)
+                created_count += 1
+        
+        db_session.commit()
+        
+        message = f"Successfully loaded default agents. Created: {created_count}, Updated: {updated_count}"
+        logger.log_message(message, level=logging.INFO)
+        return True, message
+        
+    except Exception as e:
+        db_session.rollback()
+        error_msg = f"Error loading default agents: {str(e)}"
+        logger.log_message(error_msg, level=logging.ERROR)
+        return False, error_msg
+
+def initialize_default_agents(force_update=False):
+    """
+    Initialize default agents during application startup.
+    
+    Args:
+        force_update: If True, update existing agents. If False, skip existing ones.
+    
+    Returns:
+        bool: True if successful, False otherwise
+    """
+    try:
+        from src.db.init_db import session_factory
+        
+        session = session_factory()
+        try:
+            success, message = load_default_agents_to_db(session, force_update=force_update)
+            logger.log_message(f"Default agents initialization: {message}", level=logging.INFO)
+            return success
+        finally:
+            session.close()
+            
+    except Exception as e:
+        logger.log_message(f"Failed to initialize default agents: {str(e)}", level=logging.ERROR)
+        return False 
+    
+if __name__ == "__main__":
+    initialize_default_agents(force_update=True)

src/db/schemas/models.py

@@ -11,7 +11,7 @@ class User(Base):
     __tablename__ = 'users'
     
     user_id = Column(Integer, primary_key=True, autoincrement=True)
-    username = Column(String, nullable=False)  # Removed unique=True
+    username = Column(String, unique=True, nullable=False)
     email = Column(String, unique=True, nullable=False)
     created_at = Column(DateTime, default=lambda: datetime.now(UTC))
     # Add relationship for cascade options

src/managers/ai_manager.py

@@ -1,8 +1,10 @@
 import logging
 from typing import Optional, Dict, Any
+import time
 from src.db.schemas.models import ModelUsage
 from src.db.init_db import session_factory
 from datetime import datetime, UTC
+import tiktoken
 from src.routes.analytics_routes import handle_new_model_usage
 import asyncio
 

src/managers/app_manager.py

@@ -1,127 +0,0 @@
-import logging
-import dspy
-from src.managers.session_manager import SessionManager
-from src.managers.ai_manager import AI_Manager
-from src.utils.logger import Logger
-
-logger = Logger("app_manager", see_time=True, console_log=False)
-
-class AppState:
-    def __init__(self, styling_instructions, chat_history_name_agent, default_model_config):
-        self._session_manager = SessionManager(styling_instructions, {})  # Empty dict, agents loaded from DB
-        self.model_config = default_model_config.copy()
-        
-        # Update the SessionManager with the current model_config
-        self._session_manager._app_model_config = self.model_config
-        
-        self.ai_manager = AI_Manager()
-        self.chat_name_agent = chat_history_name_agent
-        
-        # Initialize deep analysis module
-        self.deep_analyzer = None
-
-    def get_session_state(self, session_id: str):
-        """Get or create session-specific state using the SessionManager"""
-        return self._session_manager.get_session_state(session_id)
-
-    def clear_session_state(self, session_id: str):
-        """Clear session-specific state using the SessionManager"""
-        self._session_manager.clear_session_state(session_id)
-
-    def update_session_dataset(self, session_id: str, datasets, names, desc, pre_generated=False):
-        """Update dataset for a specific session using the SessionManager"""
-        self._session_manager.update_session_dataset(session_id, datasets, names, desc, pre_generated=pre_generated)
-
-    def reset_session_to_default(self, session_id: str):
-        """Reset a session to use the default dataset using the SessionManager"""
-        self._session_manager.reset_session_to_default(session_id)
-
-    def set_session_user(self, session_id: str, user_id: int, chat_id: int = None):
-        """Associate a user with a session using the SessionManager"""
-        return self._session_manager.set_session_user(session_id, user_id, chat_id)
-
-    def get_ai_manager(self):
-        """Get the AI Manager instance"""
-        return self.ai_manager
-
-    def get_provider_for_model(self, model_name):
-        return self.ai_manager.get_provider_for_model(model_name)
-
-    def calculate_cost(self, model_name, input_tokens, output_tokens):
-        return self.ai_manager.calculate_cost(model_name, input_tokens, output_tokens)
-
-    def save_usage_to_db(self, user_id, chat_id, model_name, provider, prompt_tokens, completion_tokens, total_tokens, query_size, response_size, cost, request_time_ms, is_streaming=False):
-        return self.ai_manager.save_usage_to_db(user_id, chat_id, model_name, provider, prompt_tokens, completion_tokens, total_tokens, query_size, response_size, round(cost, 7), request_time_ms, is_streaming)
-
-    def get_tokenizer(self):
-        return self.ai_manager.tokenizer
-
-    def get_chat_history_name_agent(self):
-        return dspy.Predict(self.chat_name_agent)
-
-    def get_deep_analyzer(self, session_id: str):
-        """Get or create deep analysis module for a session"""
-        session_state = self.get_session_state(session_id)
-        user_id = session_state.get("user_id")
-        
-        # Check if we need to recreate the deep analyzer (user changed or doesn't exist)
-        current_analyzer = session_state.get('deep_analyzer')
-        analyzer_user_id = session_state.get('deep_analyzer_user_id')
-        
-        logger.log_message(f"Deep analyzer check - session: {session_id}, current_user: {user_id}, analyzer_user: {analyzer_user_id}, has_analyzer: {current_analyzer is not None}", level=logging.INFO)
-        
-        if (not current_analyzer or 
-            analyzer_user_id != user_id or 
-            not hasattr(session_state, 'deep_analyzer')):
-            
-            logger.log_message(f"Creating/recreating deep analyzer for session {session_id}, user_id: {user_id} (reason: analyzer_exists={current_analyzer is not None}, user_match={analyzer_user_id == user_id})", level=logging.INFO)
-            
-            # Load user-enabled agents from database using preference system
-            from src.db.init_db import session_factory
-            from src.agents.agents import load_user_enabled_templates_for_planner_from_db
-            
-            db_session = session_factory()
-            try:
-                # Load user-enabled agents for planner (respects preferences)
-                if user_id:
-                    enabled_agents_dict = load_user_enabled_templates_for_planner_from_db(user_id, db_session)
-                else:
-                    enabled_agents_dict = {}
-                
-                if not enabled_agents_dict:
-                    # Fallback to default agents if no user preferences
-                    enabled_agents_dict = {
-                        "preprocessing_agent": "preprocessing_agent",
-                        "statistical_analytics_agent": "statistical_analytics_agent", 
-                        "sk_learn_agent": "sk_learn_agent",
-                        "data_viz_agent": "data_viz_agent"
-                    }
-                
-                # Import deep analysis module
-                from src.agents.deep_agents import deep_analysis_module
-                from src.agents.agents import get_agent_description
-                
-                deep_agents = {}
-                deep_agents_desc = {}
-                
-                for agent_name, signature in enabled_agents_dict.items():
-                    deep_agents[agent_name] = signature
-                    deep_agents_desc[agent_name] = get_agent_description(agent_name)
-                
-                logger.log_message(f"Deep analyzer initialized with {len(deep_agents)} agents: {list(deep_agents.keys())}", level=logging.INFO)
-                
-            finally:
-                db_session.close()
-            
-            session_state['deep_analyzer'] = deep_analysis_module(
-                agents=deep_agents, 
-                agents_desc=deep_agents_desc
-            )
-            # Set datasets separately or pass them when needed
-            session_state['deep_analyzer'].datasets = session_state.get("datasets")
-            session_state['deep_analyzer_user_id'] = user_id  # Track which user this analyzer was created for
-
-        else:
-            logger.log_message(f"Using existing deep analyzer for session {session_id}, user_id: {user_id}", level=logging.INFO)
-        
-        return session_state['deep_analyzer']

src/managers/chat_manager.py

@@ -1,12 +1,15 @@
-from sqlalchemy import create_engine, func, exists
+from sqlalchemy import create_engine, desc, func, exists
 from sqlalchemy.orm import sessionmaker, scoped_session
 from sqlalchemy.exc import SQLAlchemyError
 from src.db.schemas.models import Base, User, Chat, Message, ModelUsage, MessageFeedback
 import logging
-from typing import List, Dict, Optional, Any
+import requests
+import json
+from typing import List, Dict, Optional, Tuple, Any
 from datetime import datetime, UTC
+import time
+import tiktoken
 from src.utils.logger import Logger
-from src.utils.model_registry import MODEL_COSTS
 import re
 
 logger = Logger("chat_manager", see_time=True, console_log=False)
@@ -30,7 +33,39 @@ class ChatManager:
         self.Session = scoped_session(sessionmaker(bind=self.engine))
         
         # Add price mappings for different models
-        self.model_costs = MODEL_COSTS
+        self.model_costs = {
+            "openai": {
+                "gpt-4": {"input": 0.03, "output": 0.06},  
+                "gpt-4o": {"input": 0.0025, "output": 0.01},  
+                "gpt-4.5-preview": {"input": 0.075, "output": 0.15},
+                "gpt-4o-mini": {"input": 0.00015, "output": 0.0006},  
+                "gpt-3.5-turbo": {"input": 0.0005, "output": 0.0015},  
+                "o1": {"input": 0.015, "output": 0.06},  
+                "o1-mini": {"input": 0.00011, "output": 0.00044},  
+                "o3-mini": {"input": 0.00011, "output": 0.00044}  
+            },
+            "anthropic": {
+                "claude-3-opus-latest": {"input": 0.015, "output": 0.075},  
+                "claude-3-7-sonnet-latest": {"input": 0.003, "output": 0.015},   
+                "claude-3-5-sonnet-latest": {"input": 0.003, "output": 0.015}, 
+                "claude-3-5-haiku-latest": {"input": 0.0008, "output": 0.0004},
+            },
+            "groq": {
+                "deepseek-r1-distill-llama-70b": {"input": 0.00075, "output": 0.00099},
+                "llama-3.3-70b-versatile": {"input": 0.00059, "output": 0.00079},
+                "llama3-8b-8192": {"input": 0.00005, "output": 0.00008},
+                "llama3-70b-8192": {"input": 0.00059, "output": 0.00079},
+                "llama-3.1-8b-instant": {"input": 0.00005, "output": 0.00008},
+                "mistral-saba-24b": {"input": 0.00079, "output": 0.00079},
+                "gemma2-9b-it": {"input": 0.0002, "output": 0.0002},
+                "qwen-qwq-32b": {"input": 0.00029, "output": 0.00039},
+                "meta-llama/llama-4-maverick-17b-128e-instruct": {"input": 0.0002, "output": 0.0006},
+                "meta-llama/llama-4-scout-17b-16e-instruct": {"input": 0.00011, "output": 0.00034},
+            },
+            "gemini": {
+                "gemini-2.5-pro-preview-03-25": {"input": 0.00015, "output": 0.001}
+            }
+        }
                 
         
         # Add model providers mapping

src/managers/session_manager.py

@@ -4,31 +4,21 @@ import time
 import uuid
 import logging
 import pandas as pd
-from typing import Dict, Any, List
+from typing import Dict, Any, List, Optional
 
-from fastapi import HTTPException
-from src.utils.simple_retriever import Document, SimpleRetriever
+from llama_index.core import Document, VectorStoreIndex
 from src.utils.logger import Logger
-from src.managers.user_manager import get_current_user
-from src.agents.agents import auto_analyst, dataset_description_agent, data_context_gen
+from src.managers.user_manager import create_user, get_current_user, get_user_by_email
+from src.agents.agents import auto_analyst, auto_analyst_ind
 from src.agents.retrievers.retrievers import make_data
 from src.managers.chat_manager import ChatManager
-from src.utils.model_registry import mid_lm
 from dotenv import load_dotenv
-import duckdb
-import dspy
-from src.utils.dataset_description_generator import generate_dataset_description
-from fastapi import Request
 
 load_dotenv()
 
 # Initialize logger
 logger = Logger("session_manager", see_time=False, console_log=False)
 
-# Helper to clamp temperature to valid range
-def _get_clamped_temperature():
-    return min(1.0, max(0.0, float(os.getenv("TEMPERATURE", "1.0"))))
-
 class SessionManager:
     """
     Manages session-specific state, including datasets, retrievers, and AI systems.
@@ -49,11 +39,9 @@ class SessionManager:
         self._default_retrievers = None
         self._default_ai_system = None
         self._make_data = None
-
         # Initialize chat manager
-
+        self._dataset_description = "Housing Dataset"
         self._default_name = "Housing.csv"
-
         
         self._dataset_description = """This dataset contains residential property information with details about pricing, physical characteristics, and amenities. The data can be used for real estate market analysis, property valuation, and understanding the relationship between house features and prices.
 
@@ -94,6 +82,7 @@ Data Handling Recommendations:
 
 This dataset appears clean with consistent formatting and no missing values, making it suitable for immediate analysis with appropriate categorical encoding.
         """
+        self.styling_instructions = styling_instructions
         self.available_agents = available_agents
         self.chat_manager = ChatManager(db_url=os.getenv("DATABASE_URL"))
         
@@ -103,7 +92,7 @@ This dataset appears clean with consistent formatting and no missing values, mak
         """Initialize the default dataset and store it"""
         try:
             self._default_df = pd.read_csv("Housing.csv")
-            self._make_data = {'dataset_python_name':"this dataset is loaded as `df`","description":self._dataset_description}
+            self._make_data = make_data(self._default_df, self._dataset_description)
             self._default_retrievers = self.initialize_retrievers(self.styling_instructions, [str(self._make_data)])
             # Create default AI system - agents will be loaded from database
             self._default_ai_system = auto_analyst(agents=[], retrievers=self._default_retrievers)
@@ -111,11 +100,21 @@ This dataset appears clean with consistent formatting and no missing values, mak
             logger.log_message(f"Error initializing default dataset: {str(e)}", level=logging.ERROR)
             raise e
     
-    def initialize_retrievers(self,styling_instructions: List[str], doc: List[str]):
-        try:
-            style_index = SimpleRetriever.from_documents([Document(text=x) for x in styling_instructions])
+    def initialize_retrievers(self, styling_instructions: List[str], doc: List[str]):
+        """
+        Initialize retrievers for styling and data
+        
+        Args:
+            styling_instructions: List of styling instructions
+            doc: List of document strings
             
-            return {"style_index": style_index, "dataframe_index": doc}
+        Returns:
+            Dictionary containing style_index and dataframe_index
+        """
+        try:
+            style_index = VectorStoreIndex.from_documents([Document(text=x) for x in styling_instructions])
+            data_index = VectorStoreIndex.from_documents([Document(text=x) for x in doc])
+            return {"style_index": style_index, "dataframe_index": data_index}
         except Exception as e:
             logger.log_message(f"Error initializing retrievers: {str(e)}", level=logging.ERROR)
             raise e
@@ -136,10 +135,10 @@ This dataset appears clean with consistent formatting and no missing values, mak
             default_model_config = self._app_model_config
         else:
             default_model_config = {
-                "provider": os.getenv("MODEL_PROVIDER", "anthropic"),
-                "model": os.getenv("MODEL_NAME", "claude-sonnet-4-6"),
-                "api_key": os.getenv("ANTHROPIC_API_KEY"),
-                "temperature": _get_clamped_temperature(),
+                "provider": os.getenv("MODEL_PROVIDER", "openai"),
+                "model": os.getenv("MODEL_NAME", "gpt-4o-mini"),
+                "api_key": os.getenv("OPENAI_API_KEY"),
+                "temperature": float(os.getenv("TEMPERATURE", 1.0)),
                 "max_tokens": int(os.getenv("MAX_TOKENS", 6000))
             }
         
@@ -147,21 +146,16 @@ This dataset appears clean with consistent formatting and no missing values, mak
             # Check if we need to create a brand new session
             logger.log_message(f"Creating new session state for session_id: {session_id}", level=logging.INFO)
             
-            # Initialize DuckDB connection for this session
-
-            
             # Initialize with default state
             self._sessions[session_id] = {
-                "datasets": {"df":self._default_df.copy() if self._default_df is not None else None},
-                "dataset_names": ["df"],
+                "current_df": self._default_df.copy() if self._default_df is not None else None,
                 "retrievers": self._default_retrievers,
                 "ai_system": self._default_ai_system,
                 "make_data": self._make_data,
                 "description": self._dataset_description,
                 "name": self._default_name,
                 "model_config": default_model_config,
-                "creation_time": time.time(),
-                "duckdb_conn": None,
+                "creation_time": time.time()
             }
         else:
             # Verify dataset integrity in existing session
@@ -171,9 +165,9 @@ This dataset appears clean with consistent formatting and no missing values, mak
             session["model_config"] = default_model_config
             
             # If dataset is somehow missing, restore it
-            if "datasets" not in session or session["datasets"] is None:
+            if "current_df" not in session or session["current_df"] is None:
                 logger.log_message(f"Restoring missing dataset for session {session_id}", level=logging.WARNING)
-                session["datasets"] = {"df":self._default_df.copy() if self._default_df is not None else None}
+                session["current_df"] = self._default_df.copy() if self._default_df is not None else None
                 session["retrievers"] = self._default_retrievers
                 session["ai_system"] = self._default_ai_system
                 session["description"] = self._dataset_description
@@ -190,41 +184,29 @@ This dataset appears clean with consistent formatting and no missing values, mak
             
         return self._sessions[session_id]
 
-   
+    def clear_session_state(self, session_id: str):
+        """
+        Clear session-specific state
+        
+        Args:
+            session_id: The session identifier
+        """
+        if session_id in self._sessions:
+            del self._sessions[session_id]
 
 
-    def update_session_dataset(self, session_id: str, datasets, names, desc: str, pre_generated=False):
+    def update_session_dataset(self, session_id: str, df, name: str, desc: str):
         """
-        Update session with new dataset and optionally auto-generate description
+        Update dataset for a specific session
+
+        Args:
+            session_id: The session identifier
+            df: Pandas DataFrame containing the dataset
+            name: Name of the dataset
+            desc: Description of the dataset
         """
         try:
-            # Get default model config for new sessions
-            default_model_config = {
-                "provider": os.getenv("MODEL_PROVIDER", "anthropic"),
-                "model": os.getenv("MODEL_NAME", "claude-sonnet-4-6"),
-                "api_key": os.getenv("ANTHROPIC_API_KEY"),
-                "temperature": _get_clamped_temperature(),
-                "max_tokens": int(os.getenv("MAX_TOKENS", 6000))
-            }
-            
-            # Get or create DuckDB connection for this session
-            
-            # Register the new dataset in DuckDB
-            
-            # Auto-generate description if we have datasets
-            if datasets and pre_generated==False:
-                try:
-                    generated_desc = generate_dataset_description(datasets, desc, names)
-                    desc = generated_desc  # No need to format again since it's already formatted
-                    logger.log_message(f"Auto-generated description for session {session_id}", level=logging.INFO)
-                except Exception as e:
-                    logger.log_message(f"Failed to auto-generate description: {str(e)}", level=logging.WARNING)
-                    # Keep the original description if generation fails
-                    pass
-            
-            # Initialize retrievers and AI system BEFORE creating session_state
-            # Update make_data with the description
-            self._make_data = {'description': desc}
+            self._make_data = make_data(df, desc)
             retrievers = self.initialize_retrievers(self.styling_instructions, [str(self._make_data)])
             
             # Check if session has a user_id to create user-specific AI system
@@ -234,17 +216,25 @@ This dataset appears clean with consistent formatting and no missing values, mak
             
             ai_system = self.create_ai_system_for_user(retrievers, current_user_id)
             
+            # Get default model config for new sessions
+            default_model_config = {
+                "provider": os.getenv("MODEL_PROVIDER", "openai"),
+                "model": os.getenv("MODEL_NAME", "gpt-4o-mini"),
+                "api_key": os.getenv("OPENAI_API_KEY"),
+                "temperature": float(os.getenv("TEMPERATURE", 1.0)),
+                "max_tokens": int(os.getenv("MAX_TOKENS", 6000))
+            }
+            
             # Create a completely fresh session state for the new dataset
+            # This ensures no remnants of the previous dataset remain
             session_state = {
-                "datasets": datasets,
-                "dataset_names": names,
-                "retrievers": retrievers,  # Now retrievers is defined
-                "ai_system": ai_system,    # Now ai_system is defined
+                "current_df": df,
+                "retrievers": retrievers,
+                "ai_system": ai_system,
                 "make_data": self._make_data,
                 "description": desc,
-                "name": names[0],
-                "duckdb_conn": None,
-                "model_config": default_model_config,
+                "name": name,
+                "model_config": default_model_config,  # Initialize with default
             }
             
             # Preserve user_id, chat_id, and model_config if they exist in the current session
@@ -254,12 +244,13 @@ This dataset appears clean with consistent formatting and no missing values, mak
                 if "chat_id" in self._sessions[session_id]:
                     session_state["chat_id"] = self._sessions[session_id]["chat_id"]
                 if "model_config" in self._sessions[session_id]:
+                    # Preserve the user's model configuration
                     session_state["model_config"] = self._sessions[session_id]["model_config"]
             
             # Replace the entire session with the new state
             self._sessions[session_id] = session_state
             
-            logger.log_message(f"Updated session {session_id} with completely fresh dataset state: {str(names)}", level=logging.INFO)
+            logger.log_message(f"Updated session {session_id} with completely fresh dataset state: {name}", level=logging.INFO)
         except Exception as e:
             logger.log_message(f"Error updating dataset for session {session_id}: {str(e)}", level=logging.ERROR)
             raise e
@@ -274,10 +265,10 @@ This dataset appears clean with consistent formatting and no missing values, mak
         try:
             # Get default model config from environment
             default_model_config = {
-                "provider": os.getenv("MODEL_PROVIDER", "anthropic"),
-                "model": os.getenv("MODEL_NAME", "claude-sonnet-4-6"),
-                "api_key": os.getenv("ANTHROPIC_API_KEY"),
-                "temperature": _get_clamped_temperature(),
+                "provider": os.getenv("MODEL_PROVIDER", "openai"),
+                "model": os.getenv("MODEL_NAME", "gpt-4o-mini"),
+                "api_key": os.getenv("OPENAI_API_KEY"),
+                "temperature": float(os.getenv("TEMPERATURE", 1.0)),
                 "max_tokens": int(os.getenv("MAX_TOKENS", 6000))
             }
             
@@ -286,19 +277,15 @@ This dataset appears clean with consistent formatting and no missing values, mak
                 del self._sessions[session_id]
                 logger.log_message(f"Cleared existing state for session {session_id} before reset.", level=logging.INFO)
 
-            # Create new DuckDB connection for default session
-
             # Initialize with default state
             self._sessions[session_id] = {
-                "datasets": {'df':self._default_df.copy()},
-                "dataset_names": ["df"], # Use a copy
+                "current_df": self._default_df.copy(), # Use a copy
                 "retrievers": self._default_retrievers,
                 "ai_system": self._default_ai_system,
                 "description": self._dataset_description,
                 "name": self._default_name, # Explicitly set the default name
                 "make_data": None, # Clear any custom make_data
-                "model_config": default_model_config, # Initialize with default model config
-                "duckdb_conn": None, # Create new DuckDB connection
+                "model_config": default_model_config # Initialize with default model config
             }
             logger.log_message(f"Reset session {session_id} to default dataset: {self._default_name}", level=logging.INFO)
         except Exception as e:
@@ -344,77 +331,6 @@ This dataset appears clean with consistent formatting and no missing values, mak
             # Fallback to standard AI system
             return auto_analyst(agents=[], retrievers=retrievers)
 
-    def set_default_lm_for_user(self, session_id: str, user_id: int = None):
-        """
-        Set the default language model for a user upon signin using MODEL_OBJECTS.
-        
-        Args:
-            session_id: The session identifier
-            user_id: The authenticated user ID (optional)
-            
-        Returns:
-            Dictionary containing the default model configuration
-        """
-        try:
-            # Import MODEL_OBJECTS directly
-            from src.utils.model_registry import MODEL_OBJECTS
-            
-            # Set Claude Sonnet 4.6 as default model
-            default_model_name = "claude-sonnet-4-6"
-            
-            # Ensure the model exists in MODEL_OBJECTS
-            if default_model_name not in MODEL_OBJECTS:
-                logger.log_message(f"Default model '{default_model_name}' not found in MODEL_OBJECTS, using gpt-5-mini", level=logging.WARNING)
-                default_model_name = "gpt-5-mini"
-            
-            # Get the model object directly from MODEL_OBJECTS
-            model_object = MODEL_OBJECTS[default_model_name]
-            
-            # Determine provider from model name
-            provider = "anthropic"  # Claude models use Anthropic
-            
-            # Create default model configuration
-            default_model_config = {
-                "provider": provider,
-                "model": default_model_name,
-                "api_key": os.getenv(f"{provider.upper()}_API_KEY"),
-                "temperature": getattr(model_object, 'kwargs', {}).get('temperature', 0.7),
-                "max_tokens": getattr(model_object, 'kwargs', {}).get('max_tokens', 4000)
-            }
-            
-            # Ensure we have a session state for this session ID
-            if session_id not in self._sessions:
-                self.get_session_state(session_id)
-            
-            # Set the default model configuration in session state
-            self._sessions[session_id]["model_config"] = default_model_config
-            
-            # Also update the app-level model config if available
-            if hasattr(self, '_app_model_config'):
-                self._app_model_config.update(default_model_config)
-            
-            logger.log_message(f"Set default LM '{default_model_name}' for session {session_id} (user: {user_id})", level=logging.INFO)
-            
-            return {
-                "status": "success",
-                "model_config": default_model_config,
-                "message": f"Default model '{default_model_name}' set successfully"
-            }
-            
-        except Exception as e:
-            logger.log_message(f"Error setting default LM for user {user_id}: {str(e)}", level=logging.ERROR)
-            # Return fallback configuration
-            return {
-                "status": "error",
-                "model_config": {
-                    "provider": "anthropic",
-                    "model": "claude-sonnet-4-6",
-                    "temperature": 0.7,
-                    "max_tokens": 4000
-                },
-                "message": f"Failed to set default model, using fallback: {str(e)}"
-            }
-
     def set_session_user(self, session_id: str, user_id: int, chat_id: int = None):
         """
         Associate a user with a session
@@ -434,9 +350,6 @@ This dataset appears clean with consistent formatting and no missing values, mak
         # Store user ID
         self._sessions[session_id]["user_id"] = user_id
         
-        # Set default LM for user upon signin
-        self.set_default_lm_for_user(session_id, user_id)
-        
         # Generate or use chat ID
         if chat_id:
             chat_id_to_use = chat_id
@@ -463,35 +376,32 @@ This dataset appears clean with consistent formatting and no missing values, mak
             # Continue with existing AI system if update fails
         
         # Make sure this data gets saved
-        logger.log_message(f"Associated session {session_id} with user {user_id}, chat_id: {chat_id_to_use}", level=logging.INFO)
+        logger.log_message(f"Associated session {session_id} with user_id={user_id}, chat_id={chat_id_to_use}", level=logging.INFO)
         
         # Return the updated session data
         return self._sessions[session_id]
 
-async def get_session_id(request: Request, session_manager):
+async def get_session_id(request, session_manager):
     """
-    Get or create a session ID from the request
-    """
-    # Debug: Log all headers
-    logger.log_message(f"🔍 ALL REQUEST HEADERS: {dict(request.headers)}", level=logging.DEBUG)
+    Get the session ID from the request, create/associate a user if needed
     
-    # Try to get session ID from headers FIRST (primary method)
-    session_id = request.headers.get("X-Session-ID")
-    logger.log_message(f"🔍 Session ID from X-Session-ID header: {session_id}", level=logging.DEBUG)
+    Args:
+        request: FastAPI Request object
+        session_manager: SessionManager instance
+        
+    Returns:
+        Session ID string
+    """
+    # First try to get from query params
+    session_id = request.query_params.get("session_id")
     
-    # If not in headers, try query parameters (fallback for backward compatibility)
+    # If not in query params, try to get from headers
     if not session_id:
-        session_id = request.query_params.get("session_id")
-        logger.log_message(f"🔍 Session ID from query params: {session_id}", level=logging.DEBUG)
-    
-    logger.log_message(f"🔍 Final session_id before validation: '{session_id}' (type: {type(session_id)})", level=logging.DEBUG)
+        session_id = request.headers.get("X-Session-ID")
     
-    # STOP auto-generating sessions
+    # If still not found, generate a new one
     if not session_id:
-        logger.log_message(f"❌ No session ID found in request", level=logging.ERROR)
-        raise HTTPException(status_code=400, detail="Session ID required")
-    else:
-        logger.log_message(f"✅ Using existing session ID: {session_id}", level=logging.INFO)
+        session_id = str(uuid.uuid4())
     
     # Get or create the session state
     session_state = session_manager.get_session_state(session_id)