Add useable demo to test functionality

.gitignore

@@ -1,4 +0,0 @@
-__pycache__/
-.gradio/
-.env
-temp/

README.md

@@ -1,13 +1,13 @@
 ---
-title: Virtual Data Analyst
+title: Virtual Data Analyst Demo
 emoji: 📈
 colorFrom: pink
 colorTo: blue
 sdk: gradio
-sdk_version: 5.29.0
+sdk_version: 5.14.0
 app_file: app.py
-pinned: true
-short_description: Queries, visualizations, stat analysis on your data
+pinned: false
+short_description: demo of virtual data analyst project
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,192 +1,13 @@
-from utils import TEMP_DIR, message_dict, api_key_store, model_store
-import gradio as gr
-import templates.data_file as data_file, templates.sql_db as sql_db, templates.doc_db as doc_db, templates.graphql as graphql
-
-import os
-from dotenv import load_dotenv
-
-load_dotenv()
-
-def delete_db(req: gr.Request):
-    import shutil
-    dir_path = TEMP_DIR / str(req.session_hash)
-    if os.path.exists(dir_path):
-        shutil.rmtree(dir_path)
-        message_dict[req.session_hash] = {}
-    api_key_store.pop(req.session_hash, None)
-    model_store.pop(req.session_hash, None)
-
-def set_api_key(api_key, model, request: gr.Request):
-    api_key = api_key.strip()
-    if not api_key:
-        return (
-            gr.update(visible=True),
-            gr.update(visible=True, value="<p style='color:#b91c1c;text-align:center;margin:6px 0;font-size:14px;'>Please enter your API key.</p>"),
-            gr.update(visible=False),
-        )
-    api_key_store[request.session_hash] = api_key
-    model_store[request.session_hash] = model
-    provider = "Anthropic" if api_key.startswith("sk-ant-") else "OpenAI"
-    provider_icon = "fa-a" if provider == "Anthropic" else "fa-o"
-    badge_html = f"""
-    <div style="display:flex;flex-direction:column;align-items:center;gap:6px;padding:10px 0 4px;">
-        <div style="display:inline-flex;align-items:center;gap:10px;background:#f0fdf4;border:1px solid #86efac;
-                    padding:8px 20px;border-radius:9999px;font-size:13px;font-weight:500;color:#15803d;
-                    box-shadow:0 1px 3px rgba(0,0,0,0.06);">
-            <i class="fas fa-circle-check" style="font-size:14px;"></i>
-            <span>{provider}</span>
-            <span style="color:#86efac;">·</span>
-            <span style="font-weight:600;">{model}</span>
-        </div>
-        <p style="margin:0;font-size:11px;color:#9ca3af;letter-spacing:0.02em;">
-            Session active — use the button below to change
-        </p>
-    </div>
-    """
-    return gr.update(visible=False), gr.update(visible=True, value=badge_html), gr.update(visible=True)
-
-def show_api_form():
-    return gr.update(visible=True), gr.update(visible=False, value=""), gr.update(visible=False)
-
-css = ".file_marker .large{min-height:50px !important;} .padding{padding:0;} .description_component{overflow:visible !important;}"
-head = """<meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Virtual Data Analyst</title>
-    <!-- Tailwind CSS -->
-    <script src="https://cdn.tailwindcss.com"></script>
-    <!-- Google Fonts -->
-    <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap" rel="stylesheet">
-    <!-- Font Awesome -->
-    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0-beta3/css/all.min.css">
-    <!-- Custom Styles -->
-    <link rel="stylesheet" href="/gradio_api/file=assets/styles.css">
-    """
-
-theme = gr.themes.Base(primary_hue="sky", secondary_hue="slate", font=[gr.themes.GoogleFont("Inter"), "Inter", "sans-serif"]).set(
-    button_primary_background_fill="#3B82F6",
-    button_secondary_background_fill="#6B7280",
-)
-
-from pathlib import Path
-gr.set_static_paths(paths=[Path.cwd().absolute() / "assets"])
-
-_env_api_key = os.getenv("OPENAI_API_KEY", "")
-
-OPENAI_MODELS = [
-    "gpt-4.1", "gpt-4.1-mini", "gpt-4.1-nano",
-    "gpt-4o", "gpt-4o-mini",
-    "o3-mini", "o4-mini",
-    "gpt-5.4-mini", "gpt-5.4", "gpt-5.5",
-]
-ANTHROPIC_MODELS = [
-    "claude-sonnet-4-6",
-    "claude-opus-4-8",
-    "claude-haiku-4-5-20251001",
-]
-
-def update_models(api_key):
-    if api_key.strip().startswith("sk-ant-"):
-        return gr.update(choices=ANTHROPIC_MODELS, value=ANTHROPIC_MODELS[0])
-    return gr.update(choices=OPENAI_MODELS, value=OPENAI_MODELS[0])
-
-with gr.Blocks(theme=theme, css=css, head=head, delete_cache=(3600, 3600)) as demo:
-
-    with gr.Column(visible=True) as api_key_section:
-        gr.HTML("""
-            <div style="max-width:640px;margin:28px auto 12px;padding:22px 28px;
-                        background:linear-gradient(135deg,#eff6ff 0%,#e0f2fe 100%);
-                        border:1px solid #bfdbfe;border-radius:14px;
-                        box-shadow:0 2px 8px rgba(59,130,246,0.08);">
-                <div style="display:flex;align-items:flex-start;gap:16px;">
-                    <div style="width:42px;height:42px;flex-shrink:0;background:#3B82F6;
-                                border-radius:10px;display:flex;align-items:center;
-                                justify-content:center;box-shadow:0 2px 6px rgba(59,130,246,0.35);">
-                        <i class="fas fa-key" style="color:white;font-size:16px;"></i>
-                    </div>
-                    <div>
-                        <h3 style="color:#1e40af;margin:0 0 6px;font-size:16px;font-weight:700;letter-spacing:-0.01em;">
-                            Get Started
-                        </h3>
-                        <p style="color:#3730a3;font-size:13.5px;margin:0;line-height:1.6;">
-                            Enter your <strong>OpenAI</strong>
-                            (<code style="background:rgba(255,255,255,0.7);padding:1px 6px;border-radius:4px;font-size:12px;">sk-...</code>)
-                            or <strong>Anthropic</strong>
-                            (<code style="background:rgba(255,255,255,0.7);padding:1px 6px;border-radius:4px;font-size:12px;">sk-ant-...</code>)
-                            API key. The model list updates automatically. Your key is held in memory only
-                            and cleared when you leave — never saved or shared.
-                        </p>
-                    </div>
-                </div>
-            </div>
-        """)
-        with gr.Row(equal_height=True):
-            api_key_input = gr.Textbox(
-                label="API Key",
-                placeholder="sk-proj-...  or  sk-ant-api03-...",
-                type="password",
-                value=_env_api_key,
-                scale=4,
-            )
-            model_dropdown = gr.Dropdown(
-                label="Model",
-                choices=OPENAI_MODELS,
-                value=OPENAI_MODELS[0],
-                scale=2,
-            )
-            api_key_btn = gr.Button("Set API Key", variant="primary", scale=1, min_width=120)
-
-    api_key_status = gr.HTML("", visible=False)
-    change_key_btn = gr.Button("🔑  Change Key / Model", variant="secondary", visible=False, size="sm")
-
-    api_key_input.change(fn=update_models, inputs=api_key_input, outputs=model_dropdown)
-    api_key_btn.click(
-        fn=set_api_key,
-        inputs=[api_key_input, model_dropdown],
-        outputs=[api_key_section, api_key_status, change_key_btn],
-    )
-    change_key_btn.click(fn=show_api_form, outputs=[api_key_section, api_key_status, change_key_btn])
-
-    header = gr.HTML("""
-        <header class="max-w-4xl mx-auto mb-12 text-center">
-            <h1 class="text-4xl font-bold text-gray-900 mb-4">Virtual Data Analyst</h1>
-            <p class="text-lg text-gray-600 mb-6">
-                A powerful tool for data analysis, visualizations, and insights
-            </p>
-        </header>
-        <main class="max-w-4xl mx-auto">
-            <div class="mt-12 grid md:grid-cols-3 gap-6" style="margin-bottom:3px !important;">
-                <div class="feature-card bg-white p-6 rounded-lg shadow-md">
-                    <i class="feature-icon fas fa-chart-line text-primary text-2xl mb-4"></i>
-                    <h3 class="font-semibold text-gray-800 mb-2">Advanced Analytics</h3>
-                    <p class="text-gray-600 text-sm">Run SQL queries, perform regressions, and analyze results with ease</p>
-                </div>
-                <div class="feature-card bg-white p-6 rounded-lg shadow-md">
-                    <i class="feature-icon fas fa-chart-pie text-primary text-2xl mb-4"></i>
-                    <h3 class="font-semibold text-gray-800 mb-2">Rich Visualizations</h3>
-                    <p class="text-gray-600 text-sm">Create scatter plots, line charts, pie charts, and more</p>
-                </div>
-                <div class="feature-card bg-white p-6 rounded-lg shadow-md">
-                    <i class="feature-icon fas fa-magic text-primary text-2xl mb-4"></i>
-                    <h3 class="font-semibold text-gray-800 mb-2">Automated Insights</h3>
-                    <p class="text-gray-600 text-sm">Get instant insights and recommendations for your data</p>
-                </div>
-            </div>
-        </main>""")
-
-    with gr.Tab("📄  Data File"):
-        data_file.demo.render()
-    with gr.Tab("🗄  SQL Database"):
-        sql_db.demo.render()
-    with gr.Tab("🍃  MongoDB"):
-        doc_db.demo.render()
-    with gr.Tab("⚡  GraphQL API"):
-        graphql.demo.render()
-
-    footer = gr.HTML("""
-        <footer class="max-w-4xl mx-auto mt-12 text-center text-gray-500 text-sm">
-            <p>This application is under active development. For bugs or feedback, please open a discussion in the community tab.</p>
-        </footer>""")
-
-    demo.unload(delete_db)
-
-demo.launch(debug=True, allowed_paths=["temp/", "assets/"])
+from functions import demo
+
+import os
+from getpass import getpass
+from dotenv import load_dotenv
+
+load_dotenv()
+
+if "OPENAI_API_KEY" not in os.environ:
+    os.environ["OPENAI_API_KEY"] = getpass("Enter OpenAI API key:")
+
+## Uncomment the line below to launch the chat app with UI
+demo.launch(debug=True)

assets/styles.css

@@ -1,198 +0,0 @@
-/* Loading Animation */
-.loading-spinner {
-    display: none;
-    width: 50px;
-    height: 50px;
-    border: 5px solid #f3f3f3;
-    border-top: 5px solid #3B82F6;
-    border-radius: 50%;
-    animation: spin 1s linear infinite;
-    margin: 0 auto;
-}
-
-@keyframes spin {
-    0% { transform: rotate(0deg); }
-    100% { transform: rotate(360deg); }
-}
-
-/* File Upload Progress */
-.progress-bar {
-    width: 100%;
-    height: 6px;
-    background-color: #e5e7eb;
-    border-radius: 3px;
-    overflow: hidden;
-    display: none;
-    margin: 1rem auto;
-    max-width: 300px;
-}
-
-.progress-bar-fill {
-    height: 100%;
-    background-color: #3B82F6;
-    width: 0%;
-    transition: width 0.3s ease;
-}
-
-/* Tooltip */
-.tooltip {
-    position: relative;
-    display: inline-block;
-}
-
-.tooltip .tooltip-text {
-    visibility: hidden;
-    background-color: #1f2937;
-    color: white;
-    text-align: center;
-    padding: 8px 12px;
-    border-radius: 6px;
-    position: absolute;
-    z-index: 1;
-    bottom: 125%;
-    left: 50%;
-    transform: translateX(-50%);
-    opacity: 0;
-    transition: opacity 0.3s;
-    font-size: 0.875rem;
-    white-space: nowrap;
-    box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1);
-}
-
-.tooltip:hover .tooltip-text {
-    visibility: visible;
-    opacity: 1;
-}
-
-/* File Type Icons */
-.file-type-icon {
-    font-size: 1.5rem;
-    margin-right: 0.5rem;
-    color: #3B82F6;
-}
-
-/* Success Animation */
-@keyframes checkmark {
-    0% { transform: scale(0); opacity: 0; }
-    50% { transform: scale(1.2); opacity: 0.8; }
-    100% { transform: scale(1); opacity: 1; }
-}
-
-.success-checkmark {
-    display: none;
-    color: #10B981;
-    animation: checkmark 0.5s ease-in-out forwards;
-}
-
-/* Sample Data Cards */
-.sample-btn {
-    transition: all 0.3s ease;
-    position: relative;
-    overflow: hidden;
-    background: linear-gradient(135deg, #3B82F6, #0ea5e9) !important;
-}
-
-.sample-btn::after {
-    content: '';
-    position: absolute;
-    top: 0;
-    left: 0;
-    width: 100%;
-    height: 100%;
-    background: linear-gradient(rgba(255,255,255,0.12), rgba(255,255,255,0));
-    transform: translateY(-100%);
-    transition: transform 0.3s ease;
-}
-
-.sample-btn:hover::after {
-    transform: translateY(0);
-}
-
-.sample-btn:hover {
-    transform: translateY(-2px);
-    box-shadow: 0 8px 20px rgba(59,130,246,0.3);
-}
-
-/* Status badge fade-in */
-@keyframes fadeSlideIn {
-    from { opacity: 0; transform: translateY(-6px); }
-    to   { opacity: 1; transform: translateY(0); }
-}
-
-.api-status-badge {
-    animation: fadeSlideIn 0.35s ease forwards;
-}
-
-/* Drop Zone Enhancements */
-.drop-zone {
-    transition: all 0.3s ease;
-    position: relative;
-    overflow: hidden;
-}
-
-.drop-zone::before {
-    position: absolute;
-    top: 0;
-    left: 0;
-    right: 0;
-    bottom: 0;
-    border-radius: 8px;
-    border: 2px dashed #3B82F6;
-    opacity: 0;
-    transition: opacity 0.3s ease;
-}
-
-.drop-zone:hover::before {
-    opacity: 1;
-}
-
-/* File Info Card */
-#fileInfo {
-    background: linear-gradient(to right, #f8fafc, #f1f5f9);
-    border: 1px solid #e2e8f0;
-    transition: all 0.3s ease;
-}
-
-#fileInfo:hover {
-    transform: translateY(-2px);
-    box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1);
-}
-
-/* Features Section */
-.feature-card {
-    transition: all 0.3s ease;
-}
-
-.feature-card:hover {
-    transform: translateY(-2px);
-    box-shadow: 0 8px 15px rgba(0,0,0,0.1);
-}
-
-.feature-icon {
-    transition: all 0.3s ease;
-}
-
-.feature-card:hover .feature-icon {
-    transform: scale(1.1);
-    color: #2563eb;
-}
-
-@media only screen and (max-width: 600px) {
-    .feature-card p {grid-column: 1/3;}
-    .feature-card i, .feature-card h3 {text-align: center;}
-    .feature-card {
-        display: grid;
-        grid-template-columns: 1fr 2fr;
-        align-items: baseline;
-    }
-  }
-  
-dialog {
-  margin: 10% auto;
-  width: 80%;
-  max-width: 350px;
-  background-color: #fff;
-  padding: 34px;
-  border: 0;
-  border-radius: 5px;
-}  

data_sources/__init__.py

@@ -1,6 +1,3 @@
 from .upload_file import process_data_upload
-from .connect_sql_db import connect_sql_db
-from .connect_doc_db import connect_doc_db
-from .connect_graphql import connect_graphql
 
-__all__ = ["process_data_upload","connect_sql_db","connect_doc_db","connect_graphql"]
+__all__ = ["process_data_upload"]

data_sources/connect_doc_db.py

@@ -1,36 +0,0 @@
-from pymongo import MongoClient
-import os
-from utils import TEMP_DIR
-from pymongo_schema.extract import extract_pymongo_client_schema
-
-def connect_doc_db(connection_string, nosql_db_name, session_hash):
-    try:
-        # Create a MongoClient object
-        client = MongoClient(connection_string)
-        print("Connected to NoSQL Mongo DB")
-    
-        # Access a database
-        db = client[nosql_db_name]
-
-        collection_names = db.list_collection_names()
-
-        print(collection_names)
-
-        schema = extract_pymongo_client_schema(client)
-    
-        # Close the connection
-        if 'client' in locals() and client:
-            client.close()
-            print("MongoDB Connection closed.")
-
-        session_path = 'doc_db'
-
-        dir_path = TEMP_DIR / str(session_hash) / str(session_path)
-        os.makedirs(dir_path, exist_ok=True)
-
-        return ["success","<p style='color:green;text-align:center;font-size:18px;'>Document database connected successful</p>", collection_names, schema]
-    except Exception as e:
-        print("DocDB CONNECTION ERROR")
-        print(e)
-        return ["error",f"<p style='color:red;text-align:center;font-size:18px;font-weight:bold;'>ERROR: {e}</p>"]
-        

data_sources/connect_graphql.py

@@ -1,148 +0,0 @@
-import requests
-import certifi
-import os
-import json
-from utils import TEMP_DIR
-
-def connect_graphql(graphql_url, api_token, graphql_token_header, session_hash):
-    try:
-        # Create the GraphQL Introspection Query
-        query = """
-                query IntrospectionQuery {
-                    __schema {
-                    queryType { name }
-                    mutationType { name }
-                    subscriptionType { name }
-                    types {
-                        ...FullType
-                    }
-                    directives {
-                        name
-                        description
-                        locations
-                        args {
-                        ...InputValue
-                        }
-                    }
-                    }
-                }
-                fragment FullType on __Type {
-                    kind
-                    name
-                    description
-                    fields(includeDeprecated: true) {
-                    name
-                    description
-                    args {
-                        ...InputValue
-                    }
-                    type {
-                        ...TypeRef
-                    }
-                    isDeprecated
-                    deprecationReason
-                    }
-                    inputFields {
-                    ...InputValue
-                    }
-                    interfaces {
-                    ...TypeRef
-                    }
-                    enumValues(includeDeprecated: true) {
-                    name
-                    description
-                    isDeprecated
-                    deprecationReason
-                    }
-                    possibleTypes {
-                    ...TypeRef
-                    }
-                }
-                fragment InputValue on __InputValue {
-                    name
-                    description
-                    type { ...TypeRef }
-                    defaultValue
-                }
-                fragment TypeRef on __Type {
-                    kind
-                    name
-                    ofType {
-                    kind
-                    name
-                    ofType {
-                        kind
-                        name
-                        ofType {
-                        kind
-                        name
-                        ofType {
-                            kind
-                            name
-                            ofType {
-                            kind
-                            name
-                            ofType {
-                                kind
-                                name
-                                ofType {
-                                kind
-                                name
-                                }
-                            }
-                            }
-                        }
-                        }
-                    }
-                    }
-                }
-                """
-        print("Connecting to GraphQL Endpoint")
-    
-        # Access a database
-        headers = {"Content-Type": "application/json"}
-        if graphql_token_header and api_token:
-            headers[graphql_token_header] = api_token
-        response = requests.post(graphql_url, headers=headers, json={"query": query},
-                    verify=certifi.where())
-        response.raise_for_status()
-
-        introspection_result = response.json()
-
-        client_schema = introspection_result["data"]["__schema"]
-
-        #Generate the list of types
-        type_names_query = """
-                query IntrospectionQuery {
-                    __schema {
-                        types {
-                            name
-                        }
-                    }
-                }
-            """
-        types_response = requests.post(graphql_url, headers=headers, json={"query": type_names_query},
-                                       verify=certifi.where())
-
-        types_response_results =types_response.json()
-
-        types_names = types_response_results["data"]
-
-        type_names = []
-        for name in types_names["__schema"]["types"]:
-            type_names.append(name["name"])
-
-        session_path = 'graphql'
-
-        dir_path = TEMP_DIR / str(session_hash) / str(session_path)
-        os.makedirs(dir_path, exist_ok=True)
-
-        with open(f'{dir_path}/schema.json', 'w') as fp:
-            json.dump(client_schema, fp, indent=2)
-
-        return ["success","<p style='color:green;text-align:center;font-size:18px;'>GraphQL API connected successful</p>", type_names]
-    except Exception as e:
-        print("GraphQL CONNECTION ERROR")
-        print(e)
-        return ["error",f"<p style='color:red;text-align:center;font-size:18px;font-weight:bold;'>ERROR: {e}</p>"]
-        

data_sources/connect_sql_db.py

@@ -1,42 +0,0 @@
-import psycopg2
-import os
-from utils import TEMP_DIR
-
-def connect_sql_db(url, sql_user, sql_port, sql_pass, sql_db_name, session_hash):
-    try:
-        conn = psycopg2.connect(
-            database=sql_db_name,
-            user=sql_user,
-            password=sql_pass,
-            host=url,  # e.g., "localhost" or an IP address
-            port=sql_port  # default is 5432
-        )
-        print("Connected to PostgreSQL")
-    
-        # Create a cursor object to execute SQL queries
-        cur = conn.cursor()
-        # Example: Execute a query
-        cur.execute("""SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'""")
-        table_tuples = cur.fetchall()
-        table_names = []
-        for table in table_tuples:
-            table_names.append(table[0])
-
-        print(table_names)
-    
-        # Close the cursor and connection
-        cur.close()
-        conn.close()
-        print("Connection closed.")
-
-        session_path = 'sql'
-
-        dir_path = TEMP_DIR / str(session_hash) / str(session_path)
-        os.makedirs(dir_path, exist_ok=True)
-
-        return ["success","<p style='color:green;text-align:center;font-size:18px;'>SQL database connected successful</p>", table_names]
-    except Exception as e:
-        print("SQL DB CONNECTION ERROR")
-        print(e)
-        return ["error",f"<p style='color:red;text-align:center;font-size:18px;font-weight:bold;'>ERROR: {e}</p>"]
-        

data_sources/upload_file.py

@@ -1,167 +1,21 @@
 import pandas as pd
 import sqlite3
-import csv
-import json
-import time
-import os
-import re
-from utils import TEMP_DIR
-
-def is_file_done_saving(file_path):
-    try:
-        with open(file_path, 'r') as f:
-            contents = f
-
-        if contents:
-            return True
-        else:
-            return False
-    except PermissionError:
-        return False
-
-def get_delimiter(file_path, bytes = 4096):
-    sniffer = csv.Sniffer()
-    data = open(file_path, "r").read(bytes)
-    delimiter = sniffer.sniff(data).delimiter
-    return delimiter
-
-def read_file(file):
-    if file.endswith(('.csv', '.tsv', '.txt')) :
-        df = pd.read_csv(file, sep=get_delimiter(file))
-    elif file.endswith('.json'):
-        with open(file, 'r') as f:
-            contents = json.load(f)
-        df = pd.json_normalize(contents)
-    elif file.endswith('.ndjson'):
-        with open(file, 'r') as f:
-            contents = f.read()
-        data = [json.loads(str(item)) for item in contents.strip().split('\n')]
-        df = pd.json_normalize(data)
-    elif file.endswith('.xml'):
-        df = pd.read_xml(file)
-    elif file.endswith(('.xls','xlsx')):
-        df = pd.read_excel(file)
-    else:
-        raise ValueError(f'Unsupported filetype: {file}')
-    return df
 
 def process_data_upload(data_file, session_hash):
-    try:
-        total_time = 0
-        while not is_file_done_saving(data_file):
-            total_time += .5
-            time.sleep(.5)
-            if total_time > 10:
-                break
-            
-        df = read_file(data_file)
-
-        # Read each sheet and store data in a DataFrame
-        #data = df.parse(sheet_name)
-        # Process the data as needed
-        # ...
-        df.columns = df.columns.str.replace(' ', '_')
-        df.columns = df.columns.str.replace('/', '_')
-
-        for column in df.columns:
-            if type(column) is str:
-                if "date" in column.lower() or "time" in column.lower():
-                    try: 
-                        df[column] = pd.to_datetime(df[column])
-                    except:
-                        pass
-                if 'year' in column.lower():
-                    try: 
-                        df[column] = pd.to_datetime(df[column], format='%Y')
-                    except:
-                        pass
-            if df[column].dtype == 'object' and isinstance(df[column].iloc[0], list):
-                df[column] = df[column].explode()
-
-        session_path = 'file_upload'
-
-        dir_path = TEMP_DIR / str(session_hash) / str(session_path)
-        os.makedirs(dir_path, exist_ok=True)
-
-        connection = sqlite3.connect(f'{dir_path}/data_source.db')
-        print("Opened database successfully")
-
-        df.to_sql('data_source', connection, if_exists='replace', index = False)
-
-        cur=connection.execute('select * from data_source')
-        columns = [i[0] for i in cur.description]
-        print(columns)
-        
-        connection.commit()
-        connection.close()
-
-        missing_per_col = {col: int(df[col].isnull().sum()) for col in df.columns}
-        total_missing = sum(missing_per_col.values())
-
-        def _simplify_dtype(d):
-            s = str(d)
-            if 'int' in s: return 'Integer'
-            if 'float' in s: return 'Float'
-            if 'datetime' in s: return 'DateTime'
-            if 'bool' in s: return 'Boolean'
-            return 'Text'
-
-        dtypes = {col: _simplify_dtype(df[col].dtype) for col in df.columns}
-
-        preview = []
-        for _, row in df.head(5).iterrows():
-            row_vals = []
-            for v in row:
-                try:
-                    row_vals.append('' if pd.isna(v) else str(v)[:60])
-                except Exception:
-                    row_vals.append(str(v)[:60])
-            preview.append(row_vals)
-
-        duplicate_rows = int(df.duplicated().sum())
-        unique_counts = {col: int(df[col].nunique()) for col in df.columns}
-
-        col_stats = {}
-        for col in df.columns:
-            dtype_str = str(df[col].dtype)
-            try:
-                if 'int' in dtype_str or 'float' in dtype_str:
-                    col_stats[col] = {
-                        'type': 'numeric',
-                        'min': float(df[col].min()),
-                        'max': float(df[col].max()),
-                        'mean': float(df[col].mean()),
-                    }
-                elif 'datetime' in dtype_str:
-                    col_stats[col] = {
-                        'type': 'datetime',
-                        'min': str(df[col].min())[:10],
-                        'max': str(df[col].max())[:10],
-                    }
-            except Exception:
-                pass
-
-        try:
-            file_size_bytes = os.path.getsize(data_file)
-        except Exception:
-            file_size_bytes = 0
-
-        stats = {
-            'num_rows': len(df),
-            'num_cols': len(df.columns),
-            'total_missing': total_missing,
-            'missing_per_col': missing_per_col,
-            'dtypes': dtypes,
-            'preview_cols': list(df.columns),
-            'preview': preview,
-            'duplicate_rows': duplicate_rows,
-            'unique_counts': unique_counts,
-            'col_stats': col_stats,
-            'file_size_bytes': file_size_bytes,
-        }
-
-        return ["success","<p style='color:green;text-align:center;font-size:18px;'>Data upload successful</p>", columns, stats]
-    except Exception as e:
-        print("UPLOAD ERROR")
-        print(e)
-        return ["error",f"<p style='color:red;text-align:center;font-size:18px;font-weight:bold;'>ERROR: {e}</p>"]
+    df = pd.read_csv(data_file, sep=";")
+
+    # Read each sheet and store data in a DataFrame
+    #data = df.parse(sheet_name)
+    # Process the data as needed
+    # ...
+    df.columns = df.columns.str.replace(' ', '_')
+    df.columns = df.columns.str.replace('/', '_')
+    
+    connection = sqlite3.connect(f'data_source_{session_hash}.db')
+    print("Opened database successfully");
+    print(df.columns)
+
+    df.to_sql('data_source', connection, if_exists='replace', index = False)
+    
+    connection.commit()
+    connection.close()

functions/__init__.py

@@ -1,17 +1,4 @@
-from .query_functions import graphql_schema_query, graphql_csv_query, query_func
-from .chart_functions import table_generation_func, scatter_chart_generation_func, \
-    line_chart_generation_func, bar_chart_generation_func, pie_chart_generation_func, \
-    histogram_generation_func, box_chart_generation_func, correlation_heatmap_func, \
-    scatter_chart_fig, rolling_stats_func
-from .chat_functions import example_question_generator, chatbot_func
-from .stat_functions import regression_func, descriptive_stats_func, \
-    kmeans_clustering_func, hypothesis_test_func
+from .sqlite_functions import SQLiteQuery, sqlite_query_func
+from .chat_functions import demo
 
-__all__ = [
-    "query_func", "graphql_schema_query", "graphql_csv_query",
-    "table_generation_func", "scatter_chart_generation_func", "line_chart_generation_func",
-    "bar_chart_generation_func", "pie_chart_generation_func", "histogram_generation_func",
-    "box_chart_generation_func", "correlation_heatmap_func", "rolling_stats_func",
-    "regression_func", "descriptive_stats_func", "kmeans_clustering_func", "hypothesis_test_func",
-    "scatter_chart_fig", "example_question_generator", "chatbot_func",
-]
+__all__ = ["SQLiteQuery","sqlite_query_func","demo"]

functions/chart_functions.py

@@ -1,526 +0,0 @@
-from typing import List
-import plotly.io as pio
-import plotly.express as px
-import pandas as pd
-from utils import TEMP_DIR
-import os
-import ast
-from dotenv import load_dotenv
-
-load_dotenv()
-
-root_url = os.getenv("ROOT_URL", "")
-
-
-def _write_chart(fig, chart_path, chart_url):
-    """Write a Plotly figure to disk and return a responsive iframe HTML string."""
-    pio.write_html(fig, chart_path, full_html=False, config={"responsive": True})
-    return (
-        'Please display this iframe: '
-        '<div style="width:100%;overflow-x:auto;">'
-        '<iframe style="width:100%;min-width:400px;" height="500" '
-        f'src="{chart_url}" frameborder="0" allowfullscreen>'
-        '</iframe></div>'
-    )
-
-
-def llm_chart_data_scrub(data, layout):
-   #Processing data to account for variation from LLM
-   data_list = []
-   layout_dict = {}
-
-   if isinstance(data, list):
-      data_list = data
-   else:
-      data_list.append(data) 
-
-   false_replace = [':false', ': false']
-   false_value = ':False'   
-   true_replace = [':true', ': true']
-   true_value = ':True' 
-
-   data_dict = {}   
-   for data_obj in data_list:   
-      if isinstance(data_obj, str):
-         data_obj = data_obj.replace("\n", "")
-         for replace in false_replace:
-            data_obj = data_obj.replace(replace, false_value)
-         for replace in true_replace:
-            data_obj = data_obj.replace(replace, true_value)
-         print(data_obj)
-         data_dict = ast.literal_eval(data_obj)
-      else:
-         data_dict = data_obj
-
-   if layout and isinstance(layout, list):
-      layout_obj = layout[0]
-   else:
-      layout_obj = layout
-
-   if layout_obj and isinstance(layout_obj, str):
-      for replace in false_replace:
-         layout_obj = layout_obj.replace(replace, false_value)
-      for replace in true_replace:
-         layout_obj = layout_obj.replace(replace, true_value)
-      print(layout_obj)
-      layout_dict = ast.literal_eval(layout_obj)
-   else:
-      layout_dict = layout_obj
-
-   return data_dict, layout_dict
-
-def scatter_chart_fig(df, x_column: List[str], y_column: str, category: str="", trendline: str="", 
-                      trendline_options: List[dict]=[{}], marginal_x: str="", marginal_y: str="",
-                      size: str=""):
-   
-   function_args = {"data_frame":df, "x":x_column, "y":y_column}
-
-   if category:
-      function_args["color"] = category
-   if trendline:
-      function_args["trendline"] = trendline
-   if marginal_x:
-      function_args["marginal_x"] = marginal_x
-   if marginal_y:
-      function_args["marginal_y"] = marginal_y
-   if size:
-      df.loc[df[size] < 0, size] = 0
-      function_args["size"] = size   
-   if trendline_options:
-      trendline_options_dict = {}
-      if trendline_options and isinstance(trendline_options, list):
-         trendline_options_obj = trendline_options[0]
-      else:
-         trendline_options_obj = trendline_options
-
-      if trendline_options_obj and isinstance(trendline_options_obj, str):
-         trendline_options_dict = ast.literal_eval(trendline_options_obj)
-      else:
-         trendline_options_dict = trendline_options_obj
-      function_args["trendline_options"] = trendline_options_dict
-
-   fig = px.scatter(**function_args)  
-
-   return fig
-
-def scatter_chart_generation_func(x_column: List[str], y_column: str, session_hash, session_folder, data: List[dict]=[{}], layout: List[dict]=[{}], 
-                                  category: str="", trendline: str="", trendline_options: List[dict]=[{}], marginal_x: str="", marginal_y: str="",
-                                  size: str="", **kwargs):
-   try:
-      dir_path = TEMP_DIR / str(session_hash) / str(session_folder)
-      chart_path = f'{dir_path}/chart.html'
-      csv_query_path = f'{dir_path}/query.csv'
-
-      df = pd.read_csv(csv_query_path)
-
-      initial_graph = scatter_chart_fig(df, x_column=x_column, y_column=y_column, 
-                              category=category, trendline=trendline, trendline_options=trendline_options,
-                              marginal_x=marginal_x, marginal_y=marginal_y, size=size)
-      
-      fig = initial_graph.to_dict()
-
-      print(data)
-      print(layout)
-
-      data_dict,layout_dict = llm_chart_data_scrub(data,layout)
-
-      #Applying stylings and settings generated from LLM
-      if layout_dict:
-         fig["layout"] = layout_dict
-
-      data_ignore = ["x","y","type"]
-
-      if size:
-         data_ignore.append("marker")
-
-      for key, value in data_dict.items():
-         if key not in data_ignore:
-            for data_item in fig["data"]:
-               data_item[key] = value
-   
-      chart_url = f'{root_url}/gradio_api/file/temp/{session_hash}/{session_folder}/chart.html'
-      return {"reply": _write_chart(fig, chart_path, chart_url)}
-
-   except Exception as e:
-      print("SCATTER PLOT ERROR")
-      print(e)
-      reply = f"""There was an error generating the Plotly Scatter Plot from {x_column}, {y_column}, {data}, and {layout}
-            The error is {e},
-            You should probably try again.
-            """
-      return {"reply": reply}
-   
-def line_chart_generation_func(x_column: str, y_column: str, session_hash, session_folder, data: List[dict]=[{}], layout: List[dict]=[{}], 
-                                  category: str="", **kwargs):
-   try:
-      dir_path = TEMP_DIR / str(session_hash) / str(session_folder)
-      chart_path = f'{dir_path}/chart.html'
-      csv_query_path = f'{dir_path}/query.csv'
-
-      df = pd.read_csv(csv_query_path)
-
-      function_args = {"data_frame":df, "x":x_column, "y":y_column}
-
-      if category:
-         function_args["color"] = category
-
-      initial_graph = px.line(**function_args)  
-
-      fig = initial_graph.to_dict()
-
-      data_dict,layout_dict = llm_chart_data_scrub(data,layout)
-
-      print(data_dict)
-      print(layout_dict)
-
-      #Applying stylings and settings generated from LLM
-      if layout_dict:
-         fig["layout"] = layout_dict
-
-      for key, value in data_dict.items():
-         if key not in ["x","y","type"]:
-            for data_item in fig["data"]:
-               data_item[key] = value
-
-      print(fig)
-
-      chart_url = f'{root_url}/gradio_api/file/temp/{session_hash}/{session_folder}/chart.html'
-      return {"reply": _write_chart(fig, chart_path, chart_url)}
-
-   except Exception as e:
-      print("LINE CHART ERROR")
-      print(e)
-      reply = f"""There was an error generating the Plotly Line Chart from {x_column}, {y_column}, {data}, and {layout}
-            The error is {e},
-            You should probably try again.
-            """
-      return {"reply": reply}
-
-def bar_chart_generation_func(x_column: str, y_column: str, session_hash, session_folder, data: List[dict]=[{}], layout: List[dict]=[{}], 
-                                  category: str="", facet_row: str="", facet_col: str="", **kwargs):
-   try:
-      dir_path = TEMP_DIR / str(session_hash) / str(session_folder)
-      chart_path = f'{dir_path}/chart.html'
-      csv_query_path = f'{dir_path}/query.csv'
-
-      df = pd.read_csv(csv_query_path)
-
-      function_args = {"data_frame":df, "x":x_column, "y":y_column}
-
-      if category:
-         function_args["color"] = category
-      if facet_row:
-         function_args["facet_row"] = facet_row
-      if facet_col:
-         function_args["facet_col"] = facet_col
-
-      initial_graph = px.bar(**function_args)  
-
-      fig = initial_graph.to_dict()
-
-      data_dict,layout_dict = llm_chart_data_scrub(data,layout)
-
-      print(data_dict)
-      print(layout_dict)
-
-      #Applying stylings and settings generated from LLM
-      if layout_dict:
-         fig["layout"] = layout_dict
-
-      for key, value in data_dict.items():
-         if key not in ["x","y","type"]:
-            for data_item in fig["data"]:
-               data_item[key] = value
-
-      print(fig)
-
-      chart_url = f'{root_url}/gradio_api/file/temp/{session_hash}/{session_folder}/chart.html'
-      return {"reply": _write_chart(fig, chart_path, chart_url)}
-
-   except Exception as e:
-      print("BAR CHART ERROR")
-      print(e)
-      reply = f"""There was an error generating the Plotly Bar Chart from {x_column}, {y_column}, {data}, and {layout}
-            The error is {e},
-            You should probably try again.
-            """
-      return {"reply": reply}
-
-def pie_chart_generation_func(values: str, names: str, session_hash, session_folder, data: List[dict]=[{}], layout: List[dict]=[{}], **kwargs):
-   try:
-      dir_path = TEMP_DIR / str(session_hash) / str(session_folder)
-      chart_path = f'{dir_path}/chart.html'
-      csv_query_path = f'{dir_path}/query.csv'
-
-      df = pd.read_csv(csv_query_path)
-
-      function_args = {"data_frame":df, "values":values, "names":names}
-
-      initial_graph = px.pie(**function_args)  
-
-      fig = initial_graph.to_dict()
-
-      data_dict,layout_dict = llm_chart_data_scrub(data,layout)
-
-      print(data_dict)
-      print(layout_dict)
-
-      #Applying stylings and settings generated from LLM
-      if layout_dict:
-         fig["layout"] = layout_dict
-
-      for key, value in data_dict.items():
-         if key not in ["x","y","type"]:
-            for data_item in fig["data"]:
-               data_item[key] = value
-
-      print(fig)
-
-      chart_url = f'{root_url}/gradio_api/file/temp/{session_hash}/{session_folder}/chart.html'
-      return {"reply": _write_chart(fig, chart_path, chart_url)}
-
-   except Exception as e:
-      print("PIE CHART ERROR")
-      print(e)
-      reply = f"""There was an error generating the Plotly Pie Chart from {values}, {names}, {data}, and {layout}
-            The error is {e},
-            You should probably try again.
-            """
-      return {"reply": reply}
-   
-def histogram_generation_func(x_column: str, session_hash, session_folder, y_column: str="", data: List[dict]=[{}], layout: List[dict]=[{}], histnorm: str="", category: str="",
-                              histfunc: str="", **kwargs):   
-   try:
-      dir_path = TEMP_DIR / str(session_hash) / str(session_folder)
-      chart_path = f'{dir_path}/chart.html'
-      csv_query_path = f'{dir_path}/query.csv'
-
-      df = pd.read_csv(csv_query_path)
-
-      print(x_column)
-
-      function_args = {"data_frame":df, "x":x_column}
-
-      if y_column:
-         function_args["y"] = y_column
-      if histnorm:
-         function_args["histnorm"] = histnorm
-      if category:
-         function_args["color"] = category
-      if histfunc:
-         function_args["histfunc"] = histfunc
-
-      initial_graph = px.histogram(**function_args)  
-
-      fig = initial_graph.to_dict()
-
-      data_dict,layout_dict = llm_chart_data_scrub(data,layout)
-
-      print(data_dict)
-      print(layout_dict)
-
-      #Applying stylings and settings generated from LLM
-      if layout_dict:
-         fig["layout"] = layout_dict
-
-      for key, value in data_dict.items():
-         if key not in ["x","y","type"]:
-            for data_item in fig["data"]:
-               data_item[key] = value
-
-      print(fig)
-
-      chart_url = f'{root_url}/gradio_api/file/temp/{session_hash}/{session_folder}/chart.html'
-      return {"reply": _write_chart(fig, chart_path, chart_url)}
-
-   except Exception as e:
-      print("HISTOGRAM ERROR")
-      print(e)
-      reply = f"""There was an error generating the Plotly Histogram from {x_column}.
-            The error is {e},
-            You should probably try again.
-            """
-      return {"reply": reply}
-
-def box_chart_generation_func(y_column: str, session_hash, session_folder,
-                              x_column: str="", category: str="",
-                              layout: List[dict]=[{}], **kwargs):
-    try:
-        dir_path = TEMP_DIR / str(session_hash) / str(session_folder)
-        chart_path = f'{dir_path}/chart.html'
-        csv_query_path = f'{dir_path}/query.csv'
-
-        df = pd.read_csv(csv_query_path)
-
-        function_args = {"data_frame": df, "y": y_column}
-        if x_column:
-            function_args["x"] = x_column
-        if category:
-            function_args["color"] = category
-
-        initial_graph = px.box(**function_args)
-        fig = initial_graph.to_dict()
-
-        _, layout_dict = llm_chart_data_scrub({}, layout)
-        if layout_dict:
-            fig["layout"] = layout_dict
-
-        chart_url = f'{root_url}/gradio_api/file/temp/{session_hash}/{session_folder}/chart.html'
-        return {"reply": _write_chart(fig, chart_path, chart_url)}
-
-    except Exception as e:
-        print("BOX CHART ERROR")
-        print(e)
-        return {"reply": f"There was an error generating the box plot. Error: {e}. You should probably try again."}
-
-
-def correlation_heatmap_func(session_hash, session_folder, columns: List[str]=[], **kwargs):
-    try:
-        dir_path = TEMP_DIR / str(session_hash) / str(session_folder)
-        chart_path = f'{dir_path}/chart.html'
-        csv_query_path = f'{dir_path}/query.csv'
-
-        df = pd.read_csv(csv_query_path)
-
-        numeric_df = df[columns].select_dtypes(include='number') if columns else df.select_dtypes(include='number')
-
-        if numeric_df.shape[1] < 2:
-            return {"reply": "At least two numeric columns are needed for a correlation matrix. Please refine your query to include more numeric columns."}
-
-        corr = numeric_df.corr().round(3)
-
-        fig = px.imshow(
-            corr,
-            text_auto='.2f',
-            color_continuous_scale='RdBu_r',
-            zmin=-1,
-            zmax=1,
-            title='Correlation Matrix',
-            aspect='auto',
-        )
-        fig.update_layout(font=dict(family='Inter, system-ui, sans-serif'))
-
-        chart_url = f'{root_url}/gradio_api/file/temp/{session_hash}/{session_folder}/chart.html'
-        return {"reply": _write_chart(fig, chart_path, chart_url)}
-
-    except Exception as e:
-        print("CORRELATION HEATMAP ERROR")
-        print(e)
-        return {"reply": f"There was an error generating the correlation heatmap. Error: {e}. You should probably try again."}
-
-
-def rolling_stats_func(x_column: str, y_column: str, session_hash, session_folder,
-                       window: int = 7, stats: List[str] = ["mean"],
-                       layout: List[dict] = [{}], category: str = "", **kwargs):
-    try:
-        import plotly.graph_objects as go
-
-        dir_path = TEMP_DIR / str(session_hash) / str(session_folder)
-        chart_path = f'{dir_path}/chart.html'
-        csv_query_path = f'{dir_path}/query.csv'
-
-        df = pd.read_csv(csv_query_path)
-
-        try:
-            df[x_column] = pd.to_datetime(df[x_column])
-        except Exception:
-            pass
-        df = df.sort_values(x_column)
-
-        valid_stats = {"mean", "std", "min", "max"}
-        selected_stats = [s for s in stats if s in valid_stats] or ["mean"]
-
-        fig = go.Figure()
-
-        groups = df[category].unique().tolist() if category and category in df.columns else [None]
-
-        for group in groups:
-            group_df = df[df[category] == group] if group is not None else df
-            prefix = f"{group} — " if group is not None else ""
-
-            fig.add_trace(go.Scatter(
-                x=group_df[x_column].values, y=group_df[y_column].values,
-                mode="lines", name=f"{prefix}{y_column} (raw)",
-                opacity=0.35, line=dict(width=1)
-            ))
-
-            rolling_obj = group_df[y_column].rolling(window)
-            for stat in selected_stats:
-                rolled = getattr(rolling_obj, stat)()
-                fig.add_trace(go.Scatter(
-                    x=group_df[x_column].values, y=rolled.values,
-                    mode="lines", name=f"{prefix}Rolling {stat.capitalize()} (w={window})",
-                    line=dict(width=2.5)
-                ))
-
-        fig.update_layout(
-            title=f"Rolling Statistics (window={window}) — {y_column}",
-            xaxis_title=x_column,
-            yaxis_title=y_column,
-            font=dict(family="Inter, system-ui, sans-serif"),
-            legend=dict(orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1),
-        )
-
-        _, layout_dict = llm_chart_data_scrub({}, layout)
-        if layout_dict:
-            fig.update_layout(**layout_dict)
-
-        chart_url = f'{root_url}/gradio_api/file/temp/{session_hash}/{session_folder}/chart.html'
-        return {"reply": _write_chart(fig, chart_path, chart_url)}
-
-    except Exception as e:
-        print("ROLLING STATS ERROR")
-        print(e)
-        return {"reply": f"There was an error generating the rolling statistics chart. Error: {e}. You should probably try again."}
-
-
-def table_generation_func(session_hash, session_folder, **kwargs):
-    print("TABLE GENERATION")
-    try:
-        from html import escape
-
-        dir_path = TEMP_DIR / str(session_hash) / str(session_folder)
-        csv_query_path = f'{dir_path}/query.csv'
-
-        df = pd.read_csv(csv_query_path)
-
-        total_rows = len(df)
-        max_rows = 200
-        if total_rows > max_rows:
-            df = df.head(max_rows)
-            note = (f'<p class="vda-table-note">Showing first {max_rows} of {total_rows} rows'
-                    ' — refine your query to see more specific results.</p>')
-        else:
-            note = ''
-
-        header_cells = ''.join(f'<th>{escape(str(col))}</th>' for col in df.columns)
-        row_html = [
-            '<tr>' + ''.join(f'<td>{escape(str(val))}</td>' for val in row) + '</tr>'
-            for _, row in df.iterrows()
-        ]
-
-        style = (
-            '<style>'
-            '.vda-table-wrap{overflow-x:auto;margin:8px 0;border-radius:8px;border:1px solid #e5e7eb;}'
-            '.vda-table{width:100%;border-collapse:collapse;font-size:13px;font-family:Inter,system-ui,sans-serif;}'
-            '.vda-table thead th{background:#3B82F6;color:#fff;padding:9px 14px;text-align:left;white-space:nowrap;font-weight:600;}'
-            '.vda-table tbody td{padding:7px 14px;border-bottom:1px solid #f1f5f9;white-space:nowrap;}'
-            '.vda-table tbody tr:nth-child(even){background:#f8fafc;}'
-            '.vda-table tbody tr:last-child td{border-bottom:none;}'
-            '.vda-table-note{font-size:12px;color:#6b7280;margin:4px 0 0;text-align:right;}'
-            '</style>'
-        )
-
-        table = (
-            '<div class="vda-table-wrap"><table class="vda-table">'
-            f'<thead><tr>{header_cells}</tr></thead>'
-            '<tbody>' + '\n'.join(row_html) + '</tbody>'
-            '</table></div>'
-        )
-
-        return {"reply": style + table + note}
-
-    except Exception as e:
-        print("TABLE ERROR")
-        print(e)
-        return {"reply": f"There was an error generating the table. Error: {e}. You should probably try again."}

functions/chat_functions.py

@@ -1,184 +1,111 @@
-from utils import message_dict, api_key_store, model_store
+from data_sources import process_data_upload
+
+import gradio as gr
 
 from haystack.dataclasses import ChatMessage
 from haystack.components.generators.chat import OpenAIChatGenerator
-from haystack.utils import Secret
-
-def _get_generator(session_hash):
-    api_key = api_key_store.get(session_hash)
-    if not api_key:
-        raise ValueError("No API key found for this session. Please enter your API key at the top of the page.")
-    model = model_store.get(session_hash, "gpt-4o")
-    if api_key.startswith("sk-ant-"):
-        from haystack_integrations.components.generators.chat import AnthropicChatGenerator
-        return AnthropicChatGenerator(model=model, api_key=Secret.from_token(api_key))
-    return OpenAIChatGenerator(model=model, api_key=Secret.from_token(api_key))
 
-response = None
+import os
+from getpass import getpass
+from dotenv import load_dotenv
 
-def example_question_message(data_source, name, titles, schema):
-
-    example_message_dict = {
-        'file_upload' : ["You are a helpful and knowledgeable agent who has access to an SQLite database which has a table called 'data_source'.",
-                         f"""We have a SQLite database with the following {titles}. 
-                        We also have an AI agent with access to the same database that will be performing data analysis.
-                        Please return an array of seven strings, each one being a question for our data analysis agent
-                        that we can suggest that you believe will be insightful or helpful to a data analyst looking for
-                        data insights. Return nothing more than the array of questions because I need that specific data structure
-                        to process your response. No other response type or data structure will work."""],
-
-        'sql' : [f"You are a helpful and knowledgeable agent who has access to a PostgreSQL database called {name}.",
-                 f"""We have a PostgreSQL database with the following tables: {titles}. 
-                        We also have an AI agent with access to the same database that will be performing data analysis.
-                        Please return an array of seven strings, each one being a question for our data analysis agent
-                        that we can suggest that you believe will be insightful or helpful to a data analyst looking for
-                        data insights. Return nothing more than the array of questions because I need that specific data structure
-                        to process your response. No other response type or data structure will work."""],
-
-        'doc_db' : [f"You are a helpful and knowledgeable agent who has access to an MongoDB NoSQL document database called {name}.",
-                    f"""We have a MongoDB NoSQL document database with the following collections: {titles}.
-                        The schema of these collections is: {schema}. 
-                        We also have an AI agent with access to the same database that will be performing data analysis.
-                        Please return an array of seven strings, each one being a question for our data analysis agent
-                        that we can suggest that you believe will be insightful or helpful to a data analyst looking for
-                        data insights. Return nothing more than the array of questions because I need that specific data structure
-                        to process your response. No other response type or data structure will work."""],
-
-        'graphql' : [f"You are a helpful and knowledgeable agent who has access to an GraphQL API endpoint called {name}.",
-                     f"""We have a GraphQL API endpoint with the following types: {titles}.
-                        We also have an AI agent with access to the same GraphQL API endpoint that will be performing data analysis.
-                        Please return an array of seven strings, each one being a question for our data analysis agent
-                        that we can suggest that you believe will be insightful or helpful to a data analyst looking for
-                        data insights. Return nothing more than the array of questions because I need that specific data structure
-                        to process your response. No other response type or data structure will work."""]         
-
-    }
-
-    return example_message_dict[data_source]
-
-def example_question_generator(session_hash, data_source, name, titles, schema):
-    example_response = None
-    example_message_list = example_question_message(data_source, name, titles, schema)
-    example_messages = [
-        ChatMessage.from_system(
-            example_message_list[0]
-        )
-    ]
-
-    example_messages.append(ChatMessage.from_user(text=example_message_list[1]))
-
-    example_response = _get_generator(session_hash).run(messages=example_messages)
-
-    response_text = example_response["replies"][0].text
-    start = response_text.index("[") + 1
-    end = response_text.index("]")
-    response_content = response_text[start:end]
-    response_list = '[' + response_content + ']'
-    print(response_list)
-
-    return response_list
-
-def system_message(data_source, titles, schema=""):
-    print("TITLES")
-    print(titles)
-
-    tools_desc = (
-        " You have access to tools for querying the data source, generating charts and visualisations,"
-        " and performing statistical analyses — use them proactively whenever they would help answer the user's question."
-        " Always display any charts, tables, and visualisations inline in your responses by outputting the returned HTML verbatim."
-    )
+load_dotenv()
+
+if "OPENAI_API_KEY" not in os.environ:
+    os.environ["OPENAI_API_KEY"] = getpass("Enter OpenAI API key:")
 
-    system_message_dict = {
-        'file_upload': (
-            f"You are a helpful and knowledgeable agent who has access to an SQLite database which has a table called 'data_source' that contains the following columns: {titles}."
-            + tools_desc
-        ),
-        'sql': (
-            f"You are a helpful and knowledgeable agent who has access to a PostgreSQL database which has a series of tables called {titles}."
-            + tools_desc
-        ),
-        'doc_db': (
-            f"You are a helpful and knowledgeable agent who has access to a NoSQL MongoDB Document database which has a series of collections called {titles}. "
-            f"The schema of these collections is: {schema}."
-            + tools_desc
-        ),
-        'graphql': (
-            f"You are a helpful and knowledgeable agent who has access to a GraphQL API which has the following types: {titles}. "
-            "We have also saved a schema.json file that contains the entire introspection query that we can use to find out more about each type before making a query."
-            + tools_desc
-        ),
-    }
-
-    return system_message_dict[data_source]
-
-def chatbot_func(message, history, session_hash, data_source, titles, schema, *args):
-    try:
-        chat_generator = _get_generator(session_hash)
-    except ValueError as e:
-        return str(e)
-
-    from functions import (
-        table_generation_func, regression_func, descriptive_stats_func,
-        scatter_chart_generation_func, line_chart_generation_func, bar_chart_generation_func,
-        pie_chart_generation_func, histogram_generation_func,
-        box_chart_generation_func, correlation_heatmap_func, rolling_stats_func,
-        query_func, graphql_schema_query, graphql_csv_query,
-        kmeans_clustering_func, hypothesis_test_func,
+chat_generator = OpenAIChatGenerator(model="gpt-4o")
+response = None
+messages = [
+    ChatMessage.from_system(
+        "You are a helpful and knowledgeable agent who has access to an SQL database which has a table called 'data_source'"
     )
-    import tools.tools as tools
-
-    available_functions = {
-        "query_func": query_func,
-        "graphql_schema_query": graphql_schema_query,
-        "graphql_csv_query": graphql_csv_query,
-        "table_generation_func": table_generation_func,
-        "scatter_chart_generation_func": scatter_chart_generation_func,
-        "line_chart_generation_func": line_chart_generation_func,
-        "bar_chart_generation_func": bar_chart_generation_func,
-        "pie_chart_generation_func": pie_chart_generation_func,
-        "histogram_generation_func": histogram_generation_func,
-        "box_chart_generation_func": box_chart_generation_func,
-        "correlation_heatmap_func": correlation_heatmap_func,
-        "rolling_stats_func": rolling_stats_func,
-        "regression_func": regression_func,
-        "descriptive_stats_func": descriptive_stats_func,
-        "kmeans_clustering_func": kmeans_clustering_func,
-        "hypothesis_test_func": hypothesis_test_func,
-    }
-    
-    if message_dict[session_hash][data_source] != None:
-        message_dict[session_hash][data_source].append(ChatMessage.from_user(message))
-    else:
-        messages = [
-            ChatMessage.from_system(system_message(data_source, titles, schema))
-        ]
-        messages.append(ChatMessage.from_user(message))
-        message_dict[session_hash][data_source] = messages
-    
-    active_tools = tools.tools_call(session_hash, data_source, titles)
-    response = chat_generator.run(messages=message_dict[session_hash][data_source], tools=active_tools)
+]
+
+def chatbot_with_fc(message, history, session_hash):
+    from functions import sqlite_query_func
+    from pipelines import rag_pipeline_func
+    import tools
+
+    available_functions = {"sql_query_func": sqlite_query_func, "rag_pipeline_func": rag_pipeline_func}
+    messages.append(ChatMessage.from_user(message))
+    response = chat_generator.run(messages=messages, generation_kwargs={"tools": tools.tools_call(session_hash)})
 
     while True:
-        # if the response is a tool call
+        # if OpenAI response is a tool call
         if response and response["replies"][0].meta["finish_reason"] == "tool_calls" or response["replies"][0].tool_calls:
             function_calls = response["replies"][0].tool_calls
             for function_call in function_calls:
-                message_dict[session_hash][data_source].append(ChatMessage.from_assistant(tool_calls=[function_call]))
+                messages.append(ChatMessage.from_assistant(tool_calls=[function_call]))
                 ## Parse function calling information
                 function_name = function_call.tool_name
                 function_args = function_call.arguments
 
-                ## Find the corresponding function and call it with the given arguments
+                ## Find the correspoding function and call it with the given arguments
                 function_to_call = available_functions[function_name]
-                function_response = function_to_call(**function_args, session_hash=session_hash, session_folder=data_source, args=args)
-                print(function_name)
+                function_response = function_to_call(**function_args, session_hash=session_hash)
                 ## Append function response to the messages list using `ChatMessage.from_tool`
-                message_dict[session_hash][data_source].append(ChatMessage.from_tool(tool_result=function_response['reply'], origin=function_call))
-                response = chat_generator.run(messages=message_dict[session_hash][data_source], tools=active_tools)
+                messages.append(ChatMessage.from_tool(tool_result=function_response['reply'], origin=function_call))
+                response = chat_generator.run(messages=messages, generation_kwargs={"tools": tools.tools_call(session_hash)})
 
         # Regular Conversation
         else:
-            message_dict[session_hash][data_source].append(response["replies"][0])
+            messages.append(response["replies"][0])
             break
+    return response["replies"][0].text
+
+def delete_db(req: gr.Request):
+    db_file_path = f'data_source_{req.session_hash}.db'
+    if os.path.exists(db_file_path):
+        os.remove(db_file_path)
+
+def run_example(input):
+    return input
+
+def example_display(input):
+    if input == None:
+        display = True
+    else:
+        display = False
+    return gr.update(visible=display)
+
+css= ".file_marker .large{min-height:50px !important;} .example_btn{max-width:300px;}"
+
+with gr.Blocks(css=css) as demo:
+    title = gr.HTML("<h1 style='text-align:center;'>Virtual Data Analyst</h1>")
+    description = gr.HTML("<p style='text-align:center;'>Upload a CSV file and chat with our virtual data analyst to get insights on your data set</p>")
+    example_file = gr.File(visible=False, value="samples/bank_marketing_campaign.csv")
+    example_btn = gr.Button(value="Try Me: bank_marketing_campaign.csv", elem_classes="example_btn", size="md", variant="primary")
+    file_output = gr.File(label="CSV File", show_label=True, elem_classes="file_marker", file_types=['.csv'])
+    example_btn.click(fn=run_example, inputs=example_file, outputs=file_output)
+    file_output.change(fn=example_display, inputs=file_output, outputs=example_btn)
+
+    @gr.render(inputs=file_output)
+    def data_options(filename, request: gr.Request):
+        print(filename)
+        if filename:
+            parameters = gr.Textbox(visible=False, value=request.session_hash)
+            bot = gr.Chatbot(type='messages', label="CSV Chat Window", show_label=True, render=False, visible=True, elem_classes="chatbot")
+            chat = gr.ChatInterface(
+                                fn=chatbot_with_fc,
+                                type='messages',
+                                chatbot=bot,
+                                title="Chat with your data file",
+                                concurrency_limit=None,
+                                examples=[
+                                    ["Describe the dataset"],
+                                    ["List the columns in the dataset"],
+                                    ["What could this data be used for?"],
+                                ],
+                                additional_inputs=parameters
+                                ) 
+            process_upload(filename, request.session_hash)
+    
+    def process_upload(upload_value, session_hash):
+        if upload_value:
+            process_data_upload(upload_value, session_hash)
+        return [], []
+    
+    demo.unload(delete_db)
 
-    return response["replies"][0].text
+                  

functions/query_functions.py

@@ -1,229 +0,0 @@
-from typing import List
-from typing import AnyStr
-from haystack import component
-import pandas as pd
-from pandasql import sqldf
-pd.set_option('display.max_rows', None)
-pd.set_option('display.max_columns', None)
-pd.set_option('display.width', None)
-pd.set_option('display.max_colwidth', None)
-import sqlite3
-import psycopg2
-from pymongo import MongoClient
-import pymongoarrow.monkey
-import json
-import pluck
-from utils import TEMP_DIR
-import ast
-
-@component
-class SQLiteQuery:
-
-    def __init__(self, sql_database: str):
-      self.connection = sqlite3.connect(sql_database, check_same_thread=False)
-
-    @component.output_types(results=List[str], queries=List[str])
-    def run(self, queries: AnyStr, session_hash):
-        print("ATTEMPTING TO RUN SQLITE QUERY")
-        dir_path = TEMP_DIR / str(session_hash)
-        results = []
-        result = pd.read_sql(queries, self.connection)
-        result.to_csv(f'{dir_path}/file_upload/query.csv', index=False)
-        column_names = list(result.columns)
-        results.append(f"{result}")
-        self.connection.close()
-        return {"results": results, "queries": queries, "csv_columns": column_names}
-    
-@component
-class PostgreSQLQuery:
-
-    def __init__(self, url: str, sql_port: int, sql_user: str, sql_pass: str, sql_db_name: str):
-      self.connection = psycopg2.connect(
-            database=sql_db_name,
-            user=sql_user,
-            password=sql_pass,
-            host=url,  # e.g., "localhost" or an IP address
-            port=sql_port  # default is 5432
-        )
-
-    @component.output_types(results=List[str], queries=List[str])
-    def run(self, queries: AnyStr, session_hash):
-        print("ATTEMPTING TO RUN POSTGRESQL QUERY")
-        dir_path = TEMP_DIR / str(session_hash)
-        results = []
-        result = pd.read_sql_query(queries, self.connection)
-        result.to_csv(f'{dir_path}/sql/query.csv', index=False)
-        column_names = list(result.columns)
-        results.append(f"{result}")
-        self.connection.close()
-        return {"results": results, "queries": queries, "csv_columns": column_names}
-
-@component
-class DocDBQuery:
-
-    def __init__(self, connection_string: str, doc_db_name: str):
-      client = MongoClient(connection_string)
-
-      self.client = client
-      self.connection = client[doc_db_name]
-
-    @component.output_types(results=List[str], queries=List[str])
-    def run(self, aggregation_pipeline: List[str], db_collection,  session_hash):
-        pymongoarrow.monkey.patch_all()
-        print("ATTEMPTING TO RUN MONGODB QUERY")
-        dir_path = TEMP_DIR / str(session_hash)
-        results = []
-        print(aggregation_pipeline)
-
-        aggregation_pipeline = aggregation_pipeline.replace(" ", "")
-
-        false_replace = [':false', ': false']
-        false_value = ':False'   
-        true_replace = [':true', ': true']
-        true_value = ':True'
-
-        for replace in false_replace:
-            aggregation_pipeline = aggregation_pipeline.replace(replace, false_value)
-        for replace in true_replace:
-            aggregation_pipeline = aggregation_pipeline.replace(replace, true_value)
-
-        query_list = ast.literal_eval(aggregation_pipeline)
-
-        print("QUERY List")
-        print(query_list)
-        print(db_collection)
-        
-        db = self.connection
-        collection = db[db_collection]
-
-        print(collection)
-        docs = collection.aggregate_pandas_all(query_list)
-        print("DATA FRAME COMPLETE")
-        docs.to_csv(f'{dir_path}/doc_db/query.csv', index=False)
-        column_names = list(docs.columns)
-        print("CSV COMPLETE")
-        results.append(f"{docs}") 
-        self.client.close()
-        return {"results": results, "queries": aggregation_pipeline, "csv_columns": column_names}
-    
-@component
-class GraphQLQuery:
-
-    def __init__(self):
-
-      self.connection = pluck
-
-    @component.output_types(results=List[str], queries=List[str])
-    def run(self, graphql_query, graphql_api_string, graphql_api_token, graphql_token_header, session_hash):
-        print("ATTEMPTING TO RUN GRAPHQL QUERY")
-        dir_path = TEMP_DIR / str(session_hash)
-        results = []
-
-        headers = {"Content-Type": "application/json"}
-        if graphql_token_header and graphql_api_token:
-          headers[graphql_token_header] = graphql_api_token
-
-        print(graphql_query)
-
-        response = self.connection.execute(url=graphql_api_string, headers=headers, query=graphql_query, column_names="short")
-
-        if response.errors:
-           raise ValueError(response.errors)
-        elif response.data:
-          print("DATA FRAME COMPLETE")
-          print(response)
-          response_frame = response.frames['default']
-          print("RESPONSE FRAME")
-          #print(response_frame)
-
-          response_frame.to_csv(f'{dir_path}/graphql/query.csv', index=False)
-          column_names = list(response_frame.columns)
-          print("CSV COMPLETE")
-          results.append(f"{response_frame}") 
-          return {"results": results, "queries": graphql_query, "csv_columns": column_names}
-    
-def query_func(queries:AnyStr, session_hash, session_folder, args, **kwargs):
-    try:
-      print("QUERY")
-      print(queries)
-      if session_folder == "file_upload":
-        dir_path = TEMP_DIR / str(session_hash)
-        sql_query = SQLiteQuery(f'{dir_path}/file_upload/data_source.db')
-        result = sql_query.run(queries, session_hash)
-      elif session_folder == "sql":
-        sql_query = PostgreSQLQuery(args[0], args[1], args[2], args[3], args[4])
-        result = sql_query.run(queries, session_hash)
-      elif session_folder == 'doc_db':
-        doc_db_query = DocDBQuery(args[0], args[1])
-        result = doc_db_query.run(queries, kwargs['db_collection'], session_hash)
-      elif session_folder == 'graphql':
-        graphql_object = GraphQLQuery()
-        result = graphql_object.run(queries, args[0], args[1], args[2], session_hash)
-      print("RESULT")
-      print(result["csv_columns"])
-      if len(result["results"][0]) > 1000:
-        print("QUERY TOO LARGE")
-        return {"reply": f"""query result too large to be processed by llm, the query results are in our query.csv file. 
-                The column names of this query.csv file are: {result["csv_columns"]}. 
-                If you need to display the results directly, perhaps use the table_generation_func function."""}
-      else:   
-        return {"reply": result["results"][0]}
-
-    except Exception as e:
-      reply = f"""There was an error running the {session_folder} Query = {queries}
-              The error is {e},
-              You should probably try again.
-              """
-      print(reply)
-      return {"reply": reply}
-    
-def graphql_schema_query(graphql_type: AnyStr, session_hash, **kwargs):
-    dir_path = TEMP_DIR / str(session_hash)
-    try:
-      with open(f'{dir_path}/graphql/schema.json', 'r') as file:
-        data = json.load(file)
-
-      types_list = data["types"]
-      result = list(filter(lambda item: item["name"] == graphql_type, types_list))
-
-      print("SCHEMA RESULT")
-      print(graphql_type)
-      print(str(result))
-
-      return {"reply": str(result)}
-
-    except Exception as e:
-      reply = f"""There was an error querying our schema.json file with the type:{graphql_type}
-              The error is {e},
-              You should probably try again.
-              """
-      print(reply)
-      return {"reply": reply}
-
-def graphql_csv_query(csv_query: AnyStr, session_hash, **kwargs):
-    dir_path = TEMP_DIR / str(session_hash)
-    try:
-      query = pd.read_csv(f'{dir_path}/graphql/query.csv')
-      query.Name = 'query'
-      print("GRAPHQL CSV QUERY")
-      print(csv_query)
-      queried_df = sqldf(csv_query, locals())
-      print(queried_df)
-      column_names = list(queried_df.columns)
-      queried_df.to_csv(f'{dir_path}/graphql/query.csv', index=False)
-
-      if len(queried_df) > 1000:
-        print("CSV QUERY TOO LARGE")
-        return {"reply": f"""The new query results are in our query.csv file. 
-                The column names of this query.csv file are: {column_names}. 
-                If you need to display the results directly, perhaps use the table_generation_func function."""}
-      else:
-        return {"reply": str(queried_df)}
-
-    except Exception as e:
-      reply = f"""There was an error querying our query.csv file with the query:{csv_query}
-              The error is {e},
-              You should probably try again.
-              """
-      print(reply)
-      return {"reply": reply}

functions/sqlite_functions.py

@@ -0,0 +1,35 @@
+from typing import List
+from haystack import component
+import pandas as pd
+import sqlite3
+
+@component
+class SQLiteQuery:
+
+    def __init__(self, sql_database: str):
+      self.connection = sqlite3.connect(sql_database, check_same_thread=False)
+
+    @component.output_types(results=List[str], queries=List[str])
+    def run(self, queries: List[str]):
+        print("ATTEMPTING TO RUN QUERY")
+        results = []
+        for query in queries:
+          result = pd.read_sql(query, self.connection)
+          results.append(f"{result}")
+        self.connection.close()
+        return {"results": results, "queries": queries}
+    
+
+
+def sqlite_query_func(queries: List[str], session_hash):
+    sql_query = SQLiteQuery(f'data_source_{session_hash}.db')
+    try:
+      result = sql_query.run(queries)
+      return {"reply": result["results"][0]}
+
+    except Exception as e:
+      reply = f"""There was an error running the SQL Query = {queries}
+              The error is {e},
+              You should probably try again.
+              """
+      return {"reply": reply}

functions/stat_functions.py

@@ -1,285 +0,0 @@
-
-import pandas as pd
-from typing import List
-from utils import TEMP_DIR
-import plotly.express as px
-import plotly.io as pio
-import os
-from functions.chart_functions import scatter_chart_fig, llm_chart_data_scrub, _write_chart
-from dotenv import load_dotenv
-
-load_dotenv()
-
-root_url = os.getenv("ROOT_URL", "")
-
-def descriptive_stats_func(session_hash, session_folder, columns: List[str]=[], **kwargs):
-    print("DESCRIPTIVE STATISTICS")
-    try:
-        from html import escape
-
-        dir_path = TEMP_DIR / str(session_hash) / str(session_folder)
-        csv_query_path = f'{dir_path}/query.csv'
-
-        df = pd.read_csv(csv_query_path)
-
-        if columns:
-            df = df[[c for c in columns if c in df.columns]]
-
-        desc = df.describe().round(4)
-
-        header_cells = '<th style="background:#1e40af;">Statistic</th>' + ''.join(
-            f'<th>{escape(str(col))}</th>' for col in desc.columns
-        )
-        row_html = [
-            '<tr>'
-            + f'<td style="font-weight:600;color:#1e40af;background:#eff6ff;white-space:nowrap;">{escape(str(idx))}</td>'
-            + ''.join(f'<td>{escape(str(val))}</td>' for val in row)
-            + '</tr>'
-            for idx, row in desc.iterrows()
-        ]
-
-        style = (
-            '<style>'
-            '.vda-table-wrap{overflow-x:auto;margin:8px 0;border-radius:8px;border:1px solid #e5e7eb;}'
-            '.vda-table{width:100%;border-collapse:collapse;font-size:13px;font-family:Inter,system-ui,sans-serif;}'
-            '.vda-table thead th{background:#3B82F6;color:#fff;padding:9px 14px;text-align:left;white-space:nowrap;font-weight:600;}'
-            '.vda-table tbody td{padding:7px 14px;border-bottom:1px solid #f1f5f9;white-space:nowrap;}'
-            '.vda-table tbody tr:nth-child(even){background:#f8fafc;}'
-            '.vda-table tbody tr:last-child td{border-bottom:none;}'
-            '</style>'
-        )
-        table = (
-            '<div class="vda-table-wrap"><table class="vda-table">'
-            f'<thead><tr>{header_cells}</tr></thead>'
-            '<tbody>' + '\n'.join(row_html) + '</tbody>'
-            '</table></div>'
-        )
-
-        return {"reply": style + table}
-
-    except Exception as e:
-        print("DESCRIPTIVE STATS ERROR")
-        print(e)
-        return {"reply": f"There was an error generating descriptive statistics. Error: {e}. You should probably try again."}
-
-
-def kmeans_clustering_func(feature_columns: List[str], x_column: str, y_column: str,
-                           session_hash, session_folder, n_clusters: int = 3,
-                           layout: List[dict] = [{}], **kwargs):
-    print("KMEANS CLUSTERING")
-    try:
-        from sklearn.cluster import KMeans
-        from sklearn.preprocessing import StandardScaler
-        from html import escape
-
-        dir_path = TEMP_DIR / str(session_hash) / str(session_folder)
-        chart_path = f'{dir_path}/chart.html'
-        csv_query_path = f'{dir_path}/query.csv'
-
-        df = pd.read_csv(csv_query_path)
-
-        feature_df = df[feature_columns].select_dtypes(include='number').dropna()
-        if feature_df.shape[1] < 1:
-            return {"reply": "No numeric feature columns found for clustering. Please refine your query to include numeric columns."}
-
-        X_scaled = StandardScaler().fit_transform(feature_df)
-        labels = KMeans(n_clusters=n_clusters, random_state=42, n_init=10).fit_predict(X_scaled)
-
-        df_clustered = df.loc[feature_df.index].copy()
-        df_clustered['Cluster'] = [f'Cluster {l}' for l in labels]
-
-        fig = px.scatter(
-            df_clustered, x=x_column, y=y_column, color='Cluster',
-            title=f'K-Means Clustering (k={n_clusters})',
-        )
-        fig.update_layout(font=dict(family='Inter, system-ui, sans-serif'))
-
-        _, layout_dict = llm_chart_data_scrub({}, layout)
-        if layout_dict:
-            fig.update_layout(**layout_dict)
-
-        chart_url = f'{root_url}/gradio_api/file/temp/{session_hash}/{session_folder}/chart.html'
-        iframe = _write_chart(fig, chart_path, chart_url)
-
-        cluster_summary = df_clustered.groupby('Cluster')[feature_columns].mean().round(3)
-        header_cells = '<th style="background:#1e40af;">Cluster</th>' + ''.join(
-            f'<th>{escape(str(col))}</th>' for col in cluster_summary.columns
-        )
-        row_html = [
-            '<tr>'
-            + f'<td style="font-weight:600;color:#1e40af;background:#eff6ff;white-space:nowrap;">{escape(str(idx))}</td>'
-            + ''.join(f'<td>{escape(str(val))}</td>' for val in row)
-            + '</tr>'
-            for idx, row in cluster_summary.iterrows()
-        ]
-        style = (
-            '<style>'
-            '.vda-table-wrap{overflow-x:auto;margin:8px 0;border-radius:8px;border:1px solid #e5e7eb;}'
-            '.vda-table{width:100%;border-collapse:collapse;font-size:13px;font-family:Inter,system-ui,sans-serif;}'
-            '.vda-table thead th{background:#3B82F6;color:#fff;padding:9px 14px;text-align:left;white-space:nowrap;font-weight:600;}'
-            '.vda-table tbody td{padding:7px 14px;border-bottom:1px solid #f1f5f9;white-space:nowrap;}'
-            '.vda-table tbody tr:nth-child(even){background:#f8fafc;}'
-            '.vda-table tbody tr:last-child td{border-bottom:none;}'
-            '</style>'
-        )
-        summary_table = (
-            '<div class="vda-table-wrap"><table class="vda-table">'
-            f'<thead><tr>{header_cells}</tr></thead>'
-            '<tbody>' + '\n'.join(row_html) + '</tbody>'
-            '</table></div>'
-        )
-
-        return {"reply": f'{iframe}\n\n**Cluster Centroids (feature means per cluster):**\n{style}{summary_table}'}
-
-    except Exception as e:
-        print("KMEANS CLUSTERING ERROR")
-        print(e)
-        return {"reply": f"There was an error running K-Means clustering. Error: {e}. You should probably try again."}
-
-
-def hypothesis_test_func(test_type: str, column: str, session_hash, session_folder,
-                         column2: str = "", group_column: str = "",
-                         group_values: List[str] = [], pop_mean: float = 0.0, **kwargs):
-    print("HYPOTHESIS TEST")
-    try:
-        from scipy import stats
-        from html import escape
-
-        dir_path = TEMP_DIR / str(session_hash) / str(session_folder)
-        csv_query_path = f'{dir_path}/query.csv'
-        df = pd.read_csv(csv_query_path)
-
-        if test_type == "t_test_independent":
-            if not group_column or group_column not in df.columns:
-                return {"reply": "Please specify a valid group_column for the independent t-test."}
-            unique_groups = df[group_column].dropna().unique().tolist()
-            if group_values and len(group_values) == 2:
-                g1_label, g2_label = group_values[0], group_values[1]
-            elif len(unique_groups) == 2:
-                g1_label, g2_label = unique_groups[0], unique_groups[1]
-            else:
-                return {"reply": f"For an independent t-test, exactly 2 groups are needed. Found: {unique_groups}. Specify group_values with 2 entries."}
-
-            g1 = df[df[group_column] == g1_label][column].dropna()
-            g2 = df[df[group_column] == g2_label][column].dropna()
-            t_stat, p_value = stats.ttest_ind(g1, g2)
-
-            result_rows = [
-                ("Test", "Independent Samples T-Test"),
-                ("Column", column),
-                ("Group Column", group_column),
-                (f"Group 1", str(g1_label)),
-                (f"Group 2", str(g2_label)),
-                (f"Group 1 Mean (n={len(g1)})", f"{g1.mean():.4f}"),
-                (f"Group 2 Mean (n={len(g2)})", f"{g2.mean():.4f}"),
-                ("T-Statistic", f"{t_stat:.4f}"),
-                ("P-Value", f"{p_value:.6f}"),
-                ("Significant at α=0.05", "Yes ✓" if p_value < 0.05 else "No ✗"),
-            ]
-            title = f"T-Test: {column} by {group_column}"
-
-        elif test_type == "t_test_one_sample":
-            sample = df[column].dropna()
-            t_stat, p_value = stats.ttest_1samp(sample, pop_mean)
-            result_rows = [
-                ("Test", "One-Sample T-Test"),
-                ("Column", column),
-                ("Hypothesized Mean (μ₀)", f"{pop_mean:.4f}"),
-                (f"Sample Mean (n={len(sample)})", f"{sample.mean():.4f}"),
-                ("Sample Std Dev", f"{sample.std():.4f}"),
-                ("T-Statistic", f"{t_stat:.4f}"),
-                ("P-Value", f"{p_value:.6f}"),
-                ("Significant at α=0.05", "Yes ✓" if p_value < 0.05 else "No ✗"),
-            ]
-            title = f"One-Sample T-Test: {column} vs μ={pop_mean}"
-
-        elif test_type == "chi_square":
-            if not column2 or column2 not in df.columns:
-                return {"reply": "Please specify a valid column2 for the chi-square test."}
-            contingency = pd.crosstab(df[column], df[column2])
-            chi2, p_value, dof, _ = stats.chi2_contingency(contingency)
-            result_rows = [
-                ("Test", "Chi-Square Test of Independence"),
-                ("Column 1", column),
-                ("Column 2", column2),
-                ("Chi-Square Statistic", f"{chi2:.4f}"),
-                ("Degrees of Freedom", str(dof)),
-                ("P-Value", f"{p_value:.6f}"),
-                ("Significant at α=0.05", "Yes ✓" if p_value < 0.05 else "No ✗"),
-            ]
-            title = f"Chi-Square: {column} × {column2}"
-
-        else:
-            return {"reply": f"Unknown test_type '{test_type}'. Use one of: t_test_independent, t_test_one_sample, chi_square."}
-
-        style = (
-            '<style>'
-            '.vda-table-wrap{overflow-x:auto;margin:8px 0;border-radius:8px;border:1px solid #e5e7eb;}'
-            '.vda-table{width:100%;border-collapse:collapse;font-size:13px;font-family:Inter,system-ui,sans-serif;}'
-            '.vda-table thead th{background:#3B82F6;color:#fff;padding:9px 14px;text-align:left;white-space:nowrap;font-weight:600;}'
-            '.vda-table tbody td{padding:7px 14px;border-bottom:1px solid #f1f5f9;white-space:nowrap;}'
-            '.vda-table tbody tr:nth-child(even){background:#f8fafc;}'
-            '.vda-table tbody tr:last-child td{border-bottom:none;}'
-            '</style>'
-        )
-        header_cells = f'<th style="background:#1e40af;" colspan="2">{escape(title)}</th>'
-        row_html = [
-            '<tr>'
-            + f'<td style="font-weight:600;color:#1e40af;background:#eff6ff;white-space:nowrap;">{escape(label)}</td>'
-            + f'<td>{escape(value)}</td>'
-            + '</tr>'
-            for label, value in result_rows
-        ]
-        table = (
-            '<div class="vda-table-wrap"><table class="vda-table">'
-            f'<thead><tr>{header_cells}</tr></thead>'
-            '<tbody>' + '\n'.join(row_html) + '</tbody>'
-            '</table></div>'
-        )
-        return {"reply": style + table}
-
-    except Exception as e:
-        print("HYPOTHESIS TEST ERROR")
-        print(e)
-        return {"reply": f"There was an error running the hypothesis test. Error: {e}. You should probably try again."}
-
-
-def regression_func(independent_variables: List[str], dependent_variable: str, session_hash, session_folder, category: str='', **kwargs):
-    print("LINEAR REGRESSION CALCULATION")
-    print(independent_variables)
-    print(dependent_variable)
-    try:
-        dir_path = TEMP_DIR / str(session_hash) / str(session_folder)
-        chart_path = f'{dir_path}/chart.html'
-        csv_query_path = f'{dir_path}/query.csv'
-
-        df = pd.read_csv(csv_query_path)
-
-        if category in df.columns:
-           fig = scatter_chart_fig(df=df, x_column=independent_variables,y_column=dependent_variable, 
-                                    category=category,trendline="ols")
-        else:
-           fig = scatter_chart_fig(df=df,x_column=independent_variables,y_column=dependent_variable, 
-                                    trendline="ols")
-
-        chart_url = f'{root_url}/gradio_api/file/temp/{session_hash}/{session_folder}/chart.html'
-        iframe = _write_chart(fig, chart_path, chart_url)
-
-        results_frame = px.get_trendline_results(fig)
-
-        print("RESULTS")
-        print(results_frame)
-        print(results_frame.at[0, 'px_fit_results'])
-        results = results_frame.at[0, 'px_fit_results']
-        print(results.summary())
-
-        return {"reply": '{"regression_summary": %s, "regression_chart": %s' % (str(results.summary()), str(iframe))} 
-    
-    except Exception as e:
-      print("LINEAR REGRESSION ERROR")
-      print(e)
-      reply = f"""There was an error generating the linear regression calculation from {independent_variables} and {dependent_variable}
-              The error is {e},
-              You should probably try again.
-              """
-      return {"reply": reply}

pipelines/__init__.py

@@ -0,0 +1,3 @@
+from .pipelines import rag_pipeline_func
+
+__all__ = ["rag_pipeline_func"]

pipelines/pipelines.py

@@ -0,0 +1,91 @@
+from haystack import Pipeline
+from haystack.components.builders import PromptBuilder
+from haystack.components.generators.openai import OpenAIGenerator
+from haystack.components.routers import ConditionalRouter
+
+from functions import SQLiteQuery
+
+from typing import List
+import sqlite3
+
+import os
+from getpass import getpass
+from dotenv import load_dotenv
+
+load_dotenv()
+
+if "OPENAI_API_KEY" not in os.environ:
+    os.environ["OPENAI_API_KEY"] = getpass("Enter OpenAI API key:") 
+
+from haystack.components.builders import PromptBuilder
+from haystack.components.generators import OpenAIGenerator
+
+llm = OpenAIGenerator(model="gpt-4o")
+def rag_pipeline_func(queries: str, columns: str, session_hash):
+   sql_query = SQLiteQuery(f'data_source_{session_hash}.db')
+
+   connection = sqlite3.connect(f'data_source_{session_hash}.db')
+   cur=connection.execute('select * from data_source')
+   columns = [i[0] for i in cur.description]
+   cur.close()
+
+   #Rag Pipeline
+   prompt = PromptBuilder(template="""Please generate an SQL query. The query should answer the following Question: {{question}};
+               If the question cannot be answered given the provided table and columns, return 'no_answer'
+               The query is to be answered for the table is called 'data_source' with the following
+               Columns: {{columns}};
+               Answer:""")
+
+   routes = [
+      {
+         "condition": "{{'no_answer' not in replies[0]}}",
+         "output": "{{replies}}",
+         "output_name": "sql",
+         "output_type": List[str],
+      },
+      {
+         "condition": "{{'no_answer' in replies[0]}}",
+         "output": "{{question}}",
+         "output_name": "go_to_fallback",
+         "output_type": str,
+      },
+   ]
+
+   router = ConditionalRouter(routes)
+
+   fallback_prompt = PromptBuilder(template="""User entered a query that cannot be answered with the given table.
+                                             The query was: {{question}} and the table had columns: {{columns}}.
+                                             Let the user know why the question cannot be answered""")
+   fallback_llm = OpenAIGenerator(model="gpt-4")
+
+   conditional_sql_pipeline = Pipeline()
+   conditional_sql_pipeline.add_component("prompt", prompt)
+   conditional_sql_pipeline.add_component("llm", llm)
+   conditional_sql_pipeline.add_component("router", router)
+   conditional_sql_pipeline.add_component("fallback_prompt", fallback_prompt)
+   conditional_sql_pipeline.add_component("fallback_llm", fallback_llm)
+   conditional_sql_pipeline.add_component("sql_querier", sql_query)
+
+   conditional_sql_pipeline.connect("prompt", "llm")
+   conditional_sql_pipeline.connect("llm.replies", "router.replies")
+   conditional_sql_pipeline.connect("router.sql", "sql_querier.queries")
+   conditional_sql_pipeline.connect("router.go_to_fallback", "fallback_prompt.question")
+   conditional_sql_pipeline.connect("fallback_prompt", "fallback_llm")
+
+   print("RAG PIPELINE FUNCTION")
+   result = conditional_sql_pipeline.run({"prompt": {"question": queries,
+                                                  "columns": columns},
+                                       "router": {"question": queries},
+                                       "fallback_prompt": {"columns": columns}})
+   
+   if 'sql_querier' in result:
+      reply = result['sql_querier']['results'][0]
+   elif 'fallback_llm' in result:
+      reply = result['fallback_llm']['replies'][0]
+   else:
+      reply = result["llm"]["replies"][0]
+
+   print("reply content") 
+   print(reply.content)  
+
+   return {"reply": reply.content}

requirements.txt

@@ -1,18 +1,4 @@
-haystack-ai>=2.7.0
-anthropic-haystack
+haystack-ai
 python-dotenv
 gradio
-pandas
-plotly
-openpyxl
-statsmodels
-xlrd
-psycopg2-binary
-pymongo
-pymongoarrow
-pymongo_schema
-pandasql
-pluck-graphql
-certifi==2025.1.31
-scipy
-scikit-learn
+pandas

temp/.gitignore

@@ -1,2 +0,0 @@
-*
-!.gitignore

templates/data_file.py

@@ -1,286 +0,0 @@
-import gradio as gr
-from functions import example_question_generator, chatbot_func
-from data_sources import process_data_upload
-from utils import message_dict
-import ast
-import html as _html
-
-def build_summary_modal(stats):
-    num_rows = stats['num_rows']
-    num_cols = stats['num_cols']
-    total_missing = stats['total_missing']
-    duplicate_rows = stats.get('duplicate_rows', 0)
-    file_size_bytes = stats.get('file_size_bytes', 0)
-
-    def _fmt_num(v):
-        try:
-            if v != v: return '—'  # NaN
-            abs_v = abs(v)
-            if abs_v >= 1e9: return f"{v/1e9:.1f}B"
-            if abs_v >= 1e6: return f"{v/1e6:.1f}M"
-            if abs_v >= 1e3: return f"{v:,.0f}" if v == int(v) else f"{v:,.1f}"
-            return f"{v:,.0f}" if v == int(v) else f"{v:.2f}"
-        except Exception:
-            return str(v)
-
-    def _fmt_size(b):
-        if not b: return ''
-        if b < 1024: return f"{b} B"
-        if b < 1024 ** 2: return f"{b / 1024:.1f} KB"
-        if b < 1024 ** 3: return f"{b / 1024 ** 2:.1f} MB"
-        return f"{b / 1024 ** 3:.2f} GB"
-
-    file_size_label = _fmt_size(file_size_bytes)
-    dup_color = "#ef4444" if duplicate_rows > 0 else "#a16207"
-    dup_bg = "#fef2f2" if duplicate_rows > 0 else "#fefce8"
-    dup_border = "#fecaca" if duplicate_rows > 0 else "#fde68a"
-
-    dtype_rows_html = ""
-    for i, (col, dtype) in enumerate(stats['dtypes'].items()):
-        bg = "#ffffff" if i % 2 == 0 else "#f9fafb"
-        missing = stats['missing_per_col'].get(col, 0)
-        pct_missing = (missing / num_rows * 100) if num_rows > 0 else 0
-        missing_color = "#ef4444" if missing > 0 else "#9ca3af"
-        missing_weight = "600" if missing > 0 else "400"
-        missing_cell = f'{missing:,} <span style="color:#9ca3af;font-size:0.7rem;">({pct_missing:.1f}%)</span>'
-
-        unique = stats.get('unique_counts', {}).get(col, '—')
-        is_id = isinstance(unique, int) and num_rows > 0 and (unique / num_rows) >= 0.95 and unique > 10
-        id_badge = ' <span style="background:#fef3c7;color:#92400e;padding:1px 5px;border-radius:3px;font-size:0.65rem;vertical-align:middle;">ID?</span>' if is_id else ''
-        unique_cell = f'{unique:,}{id_badge}' if isinstance(unique, int) else str(unique)
-
-        cs = stats.get('col_stats', {}).get(col, {})
-        if cs.get('type') == 'numeric':
-            stats_cell = (
-                f'<span style="font-size:0.74rem;color:#6b7280;line-height:1.6;">'
-                f'{_fmt_num(cs["min"])} – {_fmt_num(cs["max"])}'
-                f'<br><span style="color:#9ca3af;">avg {_fmt_num(cs["mean"])}</span></span>'
-            )
-        elif cs.get('type') == 'datetime':
-            stats_cell = (
-                f'<span style="font-size:0.74rem;color:#6b7280;line-height:1.6;">'
-                f'{cs["min"]}<br>→ {cs["max"]}</span>'
-            )
-        else:
-            stats_cell = '<span style="color:#d1d5db;">—</span>'
-
-        dtype_rows_html += (
-            f'<tr style="background:{bg}">'
-            f'<td style="padding:7px 12px;border-bottom:1px solid #f3f4f6;color:#111827;white-space:nowrap;">{_html.escape(col)}</td>'
-            f'<td style="padding:7px 12px;border-bottom:1px solid #f3f4f6;white-space:nowrap;"><span style="background:#dbeafe;color:#1e40af;padding:2px 8px;border-radius:4px;font-size:0.74rem;">{dtype}</span></td>'
-            f'<td style="padding:7px 12px;border-bottom:1px solid #f3f4f6;text-align:right;color:{missing_color};font-weight:{missing_weight};white-space:nowrap;">{missing_cell}</td>'
-            f'<td style="padding:7px 12px;border-bottom:1px solid #f3f4f6;text-align:right;white-space:nowrap;color:#374151;">{unique_cell}</td>'
-            f'<td style="padding:7px 12px;border-bottom:1px solid #f3f4f6;">{stats_cell}</td>'
-            f'</tr>'
-        )
-
-    preview_headers_html = "".join(
-        f'<th style="padding:8px 12px;color:#6b7280;font-weight:500;border-bottom:1px solid #e5e7eb;white-space:nowrap;text-align:left;">{_html.escape(col)}</th>'
-        for col in stats['preview_cols']
-    )
-
-    preview_rows_html = ""
-    for i, row in enumerate(stats['preview']):
-        bg = "#ffffff" if i % 2 == 0 else "#f9fafb"
-        cells = "".join(
-            f'<td style="padding:7px 12px;border-bottom:1px solid #f3f4f6;color:#374151;white-space:nowrap;">{_html.escape(str(cell))}</td>'
-            for cell in row
-        )
-        preview_rows_html += f'<tr style="background:{bg}">{cells}</tr>'
-
-    size_tag = f'<span style="background:rgba(255,255,255,0.2);color:#fff;padding:2px 10px;border-radius:12px;font-size:0.75rem;font-weight:400;">{file_size_label}</span>' if file_size_label else ''
-
-    return f"""
-<div class="vda-modal-overlay" style="position:fixed;inset:0;background:rgba(0,0,0,0.55);z-index:9999;display:flex;align-items:center;justify-content:center;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;">
-  <div style="background:#fff;border-radius:14px;width:90%;max-width:800px;max-height:88vh;display:flex;flex-direction:column;box-shadow:0 25px 50px -12px rgba(0,0,0,0.35);overflow:hidden;">
-    <div style="background:linear-gradient(135deg,#3B82F6,#0ea5e9);padding:16px 20px;display:flex;justify-content:space-between;align-items:center;flex-shrink:0;gap:12px;">
-      <div style="display:flex;align-items:center;gap:10px;">
-        <span style="color:#fff;font-weight:600;font-size:1rem;">Dataset Summary</span>
-        {size_tag}
-      </div>
-      <button onclick="document.querySelectorAll('.vda-modal-overlay').forEach(function(e){{e.remove()}})" style="background:rgba(255,255,255,0.2);border:none;color:#fff;width:30px;height:30px;border-radius:50%;cursor:pointer;font-size:1rem;line-height:1;flex-shrink:0;">&#x2715;</button>
-    </div>
-    <div style="padding:20px;overflow-y:auto;flex:1;">
-      <div style="display:grid;grid-template-columns:1fr 1fr 1fr 1fr;gap:10px;margin-bottom:20px;">
-        <div style="background:#eff6ff;border:1px solid #bfdbfe;border-radius:8px;padding:12px;text-align:center;">
-          <div style="font-size:1.4rem;font-weight:700;color:#1d4ed8;">{num_rows:,}</div>
-          <div style="font-size:0.7rem;color:#64748b;text-transform:uppercase;letter-spacing:0.06em;margin-top:4px;">Rows</div>
-        </div>
-        <div style="background:#f0fdf4;border:1px solid #bbf7d0;border-radius:8px;padding:12px;text-align:center;">
-          <div style="font-size:1.4rem;font-weight:700;color:#15803d;">{num_cols}</div>
-          <div style="font-size:0.7rem;color:#64748b;text-transform:uppercase;letter-spacing:0.06em;margin-top:4px;">Columns</div>
-        </div>
-        <div style="background:#fefce8;border:1px solid #fde68a;border-radius:8px;padding:12px;text-align:center;">
-          <div style="font-size:1.4rem;font-weight:700;color:#a16207;">{total_missing:,}</div>
-          <div style="font-size:0.7rem;color:#64748b;text-transform:uppercase;letter-spacing:0.06em;margin-top:4px;">Missing Values</div>
-        </div>
-        <div style="background:{dup_bg};border:1px solid {dup_border};border-radius:8px;padding:12px;text-align:center;">
-          <div style="font-size:1.4rem;font-weight:700;color:{dup_color};">{duplicate_rows:,}</div>
-          <div style="font-size:0.7rem;color:#64748b;text-transform:uppercase;letter-spacing:0.06em;margin-top:4px;">Duplicate Rows</div>
-        </div>
-      </div>
-      <div style="margin-bottom:20px;">
-        <div style="font-size:0.78rem;font-weight:600;color:#374151;text-transform:uppercase;letter-spacing:0.06em;margin-bottom:10px;">Column Info</div>
-        <div style="border:1px solid #e5e7eb;border-radius:8px;overflow:hidden;">
-          <div style="max-height:210px;overflow:auto;">
-            <table style="border-collapse:collapse;font-size:0.83rem;min-width:100%;">
-              <thead style="background:#f9fafb;position:sticky;top:0;z-index:1;">
-                <tr>
-                  <th style="text-align:left;padding:8px 12px;color:#6b7280;font-weight:500;border-bottom:1px solid #e5e7eb;white-space:nowrap;">Column</th>
-                  <th style="text-align:left;padding:8px 12px;color:#6b7280;font-weight:500;border-bottom:1px solid #e5e7eb;white-space:nowrap;">Type</th>
-                  <th style="text-align:right;padding:8px 12px;color:#6b7280;font-weight:500;border-bottom:1px solid #e5e7eb;white-space:nowrap;">Missing</th>
-                  <th style="text-align:right;padding:8px 12px;color:#6b7280;font-weight:500;border-bottom:1px solid #e5e7eb;white-space:nowrap;">Unique</th>
-                  <th style="text-align:left;padding:8px 12px;color:#6b7280;font-weight:500;border-bottom:1px solid #e5e7eb;white-space:nowrap;">Stats / Range</th>
-                </tr>
-              </thead>
-              <tbody>{dtype_rows_html}</tbody>
-            </table>
-          </div>
-        </div>
-      </div>
-      <div>
-        <div style="font-size:0.78rem;font-weight:600;color:#374151;text-transform:uppercase;letter-spacing:0.06em;margin-bottom:10px;">Data Preview (first 5 rows)</div>
-        <div style="border:1px solid #e5e7eb;border-radius:8px;overflow:hidden;">
-          <div style="overflow:auto;max-height:200px;">
-            <table style="border-collapse:collapse;font-size:0.8rem;">
-              <thead style="background:#f9fafb;position:sticky;top:0;z-index:1;">
-                <tr>{preview_headers_html}</tr>
-              </thead>
-              <tbody>{preview_rows_html}</tbody>
-            </table>
-          </div>
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-"""
-
-def run_example(input):
-    return input
-
-def example_display(input):
-    if input == None:
-        display = True
-    else:
-        display = False
-    return [gr.update(visible=display), gr.update(visible=display), gr.update(visible=display), gr.update(visible=display)]
-
-with gr.Blocks() as demo:
-    description = gr.HTML("""
-                        <div class="max-w-4xl mx-auto mb-12 text-center">
-                            <div class="bg-blue-50 border border-blue-200 rounded-lg max-w-2xl mx-auto">
-                                <h2 class="font-semibold text-blue-800 ">
-                                    <i class="fas fa-info-circle mr-2"></i>Supported Files
-                                </h2>
-                                <div class="flex flex-wrap justify-center gap-3 pb-4 text-blue-700">
-                                    <span class="tooltip">
-                                        <i class="fas fa-file-csv mr-1"></i>CSV
-                                        <span class="tooltip-text">Comma-separated values</span>
-                                    </span>
-                                    <span class="tooltip">
-                                        <i class="fas fa-file-alt mr-1"></i>TSV
-                                        <span class="tooltip-text">Tab-separated values</span>
-                                    </span>
-                                    <span class="tooltip">
-                                        <i class="fas fa-file-alt mr-1"></i>TXT
-                                        <span class="tooltip-text">Text files</span>
-                                    </span>
-                                    <span class="tooltip">
-                                        <i class="fas fa-file-excel mr-1"></i>XLS/XLSX
-                                        <span class="tooltip-text">Excel spreadsheets</span>
-                                    </span>
-                                    <span class="tooltip">
-                                        <i class="fas fa-file-code mr-1"></i>XML
-                                        <span class="tooltip-text">XML documents</span>
-                                    </span>
-                                    <span class="tooltip">
-                                        <i class="fas fa-file-code mr-1"></i>JSON
-                                        <span class="tooltip-text">JSON data files</span>
-                                    </span>
-                                </div>
-                            </div>
-                        </div>
-                          """, elem_classes="description_component")
-    example_file_1 = gr.File(visible=False, value="samples/bank_marketing_campaign.csv")
-    example_file_2 = gr.File(visible=False, value="samples/online_retail_data.csv")
-    example_file_3 = gr.File(visible=False, value="samples/tb_illness_data.csv")
-    with gr.Row():
-        example_btn_1 = gr.Button(value="Try Me: bank_marketing_campaign.csv", elem_classes="sample-btn bg-gradient-to-r from-blue-500 to-sky-600 text-white p-6 rounded-lg text-left hover:shadow-lg", size="md", variant="primary")
-        example_btn_2 = gr.Button(value="Try Me: online_retail_data.csv", elem_classes="sample-btn bg-gradient-to-r from-blue-500 to-sky-600 text-white p-6 rounded-lg text-left hover:shadow-lg", size="md", variant="primary")
-        example_btn_3 = gr.Button(value="Try Me: tb_illness_data.csv", elem_classes="sample-btn bg-gradient-to-r from-blue-500 to-sky-600 text-white p-6 rounded-lg text-left hover:shadow-lg", size="md", variant="primary")
-
-    file_output = gr.File(label="Data File (CSV, TSV, TXT, XLS, XLSX, XML, JSON)", show_label=True, elem_classes="file_marker drop-zone border-2 border-dashed border-gray-300 rounded-lg hover:border-primary cursor-pointer bg-gray-50 hover:bg-blue-50 transition-colors duration-300", file_types=['.csv', '.xlsx', '.txt', '.json', '.ndjson', '.xml', '.xls', '.tsv'])
-    example_btn_1.click(fn=run_example, inputs=example_file_1, outputs=file_output)
-    example_btn_2.click(fn=run_example, inputs=example_file_2, outputs=file_output)
-    example_btn_3.click(fn=run_example, inputs=example_file_3, outputs=file_output)
-    file_output.change(fn=example_display, inputs=file_output, outputs=[example_btn_1, example_btn_2, example_btn_3, description])
-
-    @gr.render(inputs=file_output)
-    def data_options(filename, request: gr.Request):
-        print(filename)
-        if request.session_hash not in message_dict:
-            message_dict[request.session_hash] = {}
-        message_dict[request.session_hash]['file_upload'] = None
-        if filename:
-            process_message = process_upload(filename, request.session_hash)
-            gr.HTML(value=process_message[1], padding=False)
-            if process_message[0] == "success":
-                gr.HTML(value=build_summary_modal(process_message[3]), padding=False)
-                if "bank_marketing_campaign" in filename:
-                    example_questions = [
-                                            ["Describe the dataset"],
-                                            ["What levels of education have the highest and lowest average balance?"],
-                                            ["What job is most and least common for a yes response from the individuals, not counting 'unknown'?"],
-                                            ["Can you generate a bar chart of education vs. average balance?"],
-                                            ["Can you generate a table of levels of education versus average balance, percent married, percent with a loan, and percent in default?"],
-                                            ["Can we predict the relationship between the number of contacts performed before this campaign and the average balance?"],
-                                            ["Can you plot the number of contacts performed before this campaign versus the duration and use balance as the size in a bubble chart?"]
-                                        ]
-                elif "online_retail_data" in filename:
-                    example_questions = [
-                                            ["Describe the dataset"],
-                                            ["What month had the highest revenue?"],
-                                            ["Is revenue higher in the morning or afternoon?"],
-                                            ["Can you generate a line graph of revenue per month?"],
-                                            ["Can you generate a table of revenue per month?"],
-                                            ["Can we predict how time of day affects transaction value in this data set?"],
-                                            ["Can you plot revenue per month with size being the number of units sold that month in a bubble chart?"]
-                                        ]
-                else:
-                    try:
-                        generated_examples = ast.literal_eval(example_question_generator(request.session_hash, 'file_upload', '', process_message[2], ''))
-                        example_questions = [["Describe the dataset"]]
-                        for example in generated_examples:
-                            example_questions.append([example])
-                    except Exception as e:
-                        print("DATA FILE QUESTION GENERATION ERROR")
-                        print(e)
-                        example_questions = [
-                                            ["Describe the dataset"],
-                                            ["List the columns in the dataset"],
-                                            ["What could this data be used for?"],
-                                        ]
-                session_hash = gr.Textbox(visible=False, value=request.session_hash)
-                data_source = gr.Textbox(visible=False, value='file_upload')
-                schema = gr.Textbox(visible=False, value='')
-                titles = gr.Textbox(value=process_message[2], interactive=False, visible=False)
-                bot = gr.Chatbot(type='messages', label="CSV Chat Window", render_markdown=True, sanitize_html=False, show_label=True, render=False, visible=True, elem_classes="chatbot")
-                chat = gr.ChatInterface(
-                                    fn=chatbot_func,
-                                    type='messages',
-                                    chatbot=bot,
-                                    title="Chat with your data file",
-                                    concurrency_limit=None,
-                                    examples=example_questions,
-                                    additional_inputs=[session_hash, data_source, titles, schema]
-                                    )
-
-    def process_upload(upload_value, session_hash):
-        if upload_value:
-            process_message = process_data_upload(upload_value, session_hash)
-        return process_message
-
-
-if __name__ == "__main__":
-    demo.launch()

templates/doc_db.py

@@ -1,105 +0,0 @@
-import ast
-import gradio as gr
-from functions import example_question_generator, chatbot_func
-from data_sources import connect_doc_db
-from utils import message_dict
-
-with gr.Blocks() as demo:
-    with gr.Accordion("ℹ️  About the MongoDB Connector", open=False):
-        gr.HTML("""
-            <div class="max-w-4xl mx-auto text-center">
-                <div class="bg-blue-50 border border-blue-200 rounded-lg max-w-2xl mx-auto p-4">
-                    <p>Connect to a MongoDB database and query it using natural language.</p>
-                    <p style="font-weight:bold;">
-                        No credentials are retained — they are passed as session variables and disappear when you leave or refresh.
-                        Queries use PyMongoArrow's <code>aggregate_pandas_all</code>, which cannot delete, drop, or insert documents.
-                        Use caution connecting production databases to third-party tools.
-                    </p>
-                    <p>Contact me if you'd like this built for your organization with proper infrastructure and security controls.</p>
-                </div>
-            </div>
-        """)
-
-    gr.HTML("""
-        <div style="max-width:560px;margin:8px auto 4px;padding:8px 14px;background:#f0f9ff;
-                    border:1px solid #bae6fd;border-radius:8px;text-align:center;">
-            <p style="margin:0;font-size:13px;color:#0369a1;">
-                <i class="fas fa-flask" style="margin-right:6px;"></i>
-                <strong>Demo credentials pre-filled.</strong>
-                &nbsp;Replace with your own database details to analyze your own data.
-            </p>
-        </div>
-    """)
-
-    connection_string = gr.Textbox(label="Connection String", value="dataanalyst0.l1klmww.mongodb.net/")
-    with gr.Row():
-        connection_user = gr.Textbox(label="Connection User", value="virtual-data-analyst")
-        connection_password = gr.Textbox(label="Connection Password", value="zcpbmoGJ3mC8o", type="password")
-        doc_db_name = gr.Textbox(label="Database Name", value="sample_mflix")
-
-    gr.HTML("""
-        <p style="text-align:center;font-size:13px;color:#6b7280;margin:4px 0 8px;">
-            <i class="fas fa-circle-info" style="margin-right:4px;"></i>
-            Schema analysis runs on connect — this may take 1–2 minutes for large databases.
-        </p>
-    """)
-    submit = gr.Button(value="Connect", variant="primary")
-
-    @gr.render(inputs=[connection_string, connection_user, connection_password, doc_db_name], triggers=[submit.click])
-    def db_chat(request: gr.Request, connection_string=connection_string.value, connection_user=connection_user.value, connection_password=connection_password.value, doc_db_name=doc_db_name.value):
-        if request.session_hash not in message_dict:
-            message_dict[request.session_hash] = {}
-        message_dict[request.session_hash]['doc_db'] = None
-        connection_login_value = "mongodb+srv://" + connection_user + ":" + connection_password + "@" + connection_string
-        if connection_login_value:
-            print("MONGO APP")
-            process_message = process_doc_db(connection_login_value, doc_db_name, request.session_hash)
-            gr.HTML(value=process_message[1], padding=False)
-            if process_message[0] == "success":
-                if "dataanalyst0.l1klmww.mongodb.net" in connection_login_value:
-                    example_questions = [
-                                            ["Describe the dataset"],
-                                            ["What are the top 5 most common movie genres?"],
-                                            ["How do user comment counts on a movie correlate with the movie award wins?"],
-                                            ["Can you generate a pie chart showing the top 10 states with the most movie theaters?"],
-                                            ["What are the top 10 most represented directors in the database?"],
-                                            ["What are the different movie categories and how many movies are in each category?"]
-                                        ]
-                else:
-                    try:
-                        generated_examples = ast.literal_eval(example_question_generator(request.session_hash, 'doc_db', doc_db_name, process_message[2], process_message[3]))
-                        example_questions = [["Describe the dataset"]]
-                        for example in generated_examples:
-                            example_questions.append([example])
-                    except Exception as e:
-                        print("DOC DB QUESTION GENERATION ERROR")
-                        print(e)
-                        example_questions = [
-                                            ["Describe the dataset"],
-                                            ["List the collections in the database"],
-                                            ["What could this data be used for?"],
-                                        ]
-                session_hash = gr.Textbox(visible=False, value=request.session_hash)
-                db_connection_string = gr.Textbox(visible=False, value=connection_login_value)
-                db_name = gr.Textbox(visible=False, value=doc_db_name)
-                titles = gr.Textbox(value=process_message[2], interactive=False, label="DB Collections")
-                data_source = gr.Textbox(visible=False, value='doc_db')
-                schema = gr.Textbox(visible=False, value=process_message[3])
-                bot = gr.Chatbot(type='messages', label="MongoDB Chat Window", render_markdown=True, sanitize_html=False, show_label=True, render=False, visible=True, elem_classes="chatbot")
-                chat = gr.ChatInterface(
-                                    fn=chatbot_func,
-                                    type='messages',
-                                    chatbot=bot,
-                                    title="Chat with your Database",
-                                    examples=example_questions,
-                                    concurrency_limit=None,
-                                    additional_inputs=[session_hash, data_source, titles, schema, db_connection_string, db_name]
-                                    )
-
-    def process_doc_db(connection_string, nosql_db_name, session_hash):
-        if connection_string:
-            process_message = connect_doc_db(connection_string, nosql_db_name, session_hash)
-        return process_message
-
-if __name__ == "__main__":
-    demo.launch()

templates/graphql.py

@@ -1,110 +0,0 @@
-import ast
-import gradio as gr
-from functions import example_question_generator, chatbot_func
-from data_sources import connect_graphql
-from utils import message_dict
-
-import os
-from dotenv import load_dotenv
-
-load_dotenv()
-
-graphql_sample_endpoint = os.getenv("GRAPHQL_SAMPLE_ENDPOINT")
-graphql_sample_api_token = os.getenv("GRAPHQL_SAMPLE_API_TOKEN")
-graphql_sample_header_name = os.getenv("GRAPHQL_SAMPLE_HEADER_NAME")
-
-with gr.Blocks() as demo:
-    with gr.Accordion("ℹ️  About the GraphQL Connector", open=False):
-        gr.HTML("""
-            <div class="max-w-4xl mx-auto text-center">
-                <div class="bg-blue-50 border border-blue-200 rounded-lg max-w-2xl mx-auto p-4">
-                    <p>Connect to any GraphQL API endpoint and query it using natural language.</p>
-                    <p style="font-weight:bold;">
-                        API querying is the most experimental feature and performance may vary.
-                        No credentials are retained — they are passed as session variables and disappear when you leave or refresh.
-                        Mutations are not exposed and the agent is instructed not to alter data, though restricting
-                        your API token's permissions is still strongly recommended.
-                    </p>
-                    <p>Contact me if you'd like this built for your organization with proper infrastructure and security controls.</p>
-                </div>
-            </div>
-        """)
-
-    gr.HTML("""
-        <div style="max-width:560px;margin:8px auto 4px;padding:8px 14px;background:#f0f9ff;
-                    border:1px solid #bae6fd;border-radius:8px;text-align:center;">
-            <p style="margin:0;font-size:13px;color:#0369a1;">
-                <i class="fas fa-flask" style="margin-right:6px;"></i>
-                <strong>Demo credentials pre-filled.</strong>
-                &nbsp;Replace with your own endpoint and token to analyze your own API.
-            </p>
-        </div>
-    """)
-
-    graphql_url = gr.Textbox(label="GraphQL Endpoint URL", value=graphql_sample_endpoint)
-    with gr.Row():
-        api_token_header_name = gr.Textbox(label="API Token Header Name", value=graphql_sample_header_name)
-        api_token = gr.Textbox(label="API Token", value=graphql_sample_api_token, type="password")
-
-    submit = gr.Button(value="Connect", variant="primary")
-
-    @gr.render(inputs=[graphql_url, api_token, api_token_header_name], triggers=[submit.click])
-    def api_chat(request: gr.Request, graphql_url=graphql_url.value, api_token=api_token.value, api_token_header_name=api_token_header_name.value):
-        if request.session_hash not in message_dict:
-            message_dict[request.session_hash] = {}
-        message_dict[request.session_hash]['graphql'] = None
-        if graphql_url:
-            print("GraphQL API")
-            process_message = process_graphql(graphql_url, api_token, api_token_header_name, request.session_hash)
-            gr.HTML(value=process_message[1], padding=False)
-            if process_message[0] == "success":
-                if "qdl-app-testing" in graphql_url:
-                    example_questions = [
-                                            ["Describe the dataset"],
-                                            ["What is the total revenue for this shopify store?"],
-                                            ["What is the average duration from the fulfillment of an order to its delivery?"],
-                                            ["What is the total value of orders processed in the current month?"],
-                                            ["Which product has the highest number of variants in the inventory?"],
-                                            ["How many gift cards have been issued this year, and what is their total value?"],
-                                            ["How many active apps are currently installed on the store?"],
-                                            ["What is the total count of abandoned checkouts over the last month?"]
-                                        ]
-                else:
-                    try:
-                        generated_examples = ast.literal_eval(example_question_generator(request.session_hash, 'graphql', graphql_url, process_message[2], ''))
-                        example_questions = [["Describe the dataset"]]
-                        for example in generated_examples:
-                            example_questions.append([example])
-                    except Exception as e:
-                        print("GRAPHQL QUESTION GENERATION ERROR")
-                        print(e)
-                        example_questions = [
-                                            ["Describe the dataset"],
-                                            ["List the types in this API"],
-                                            ["What could this data be used for?"],
-                                        ]
-                session_hash = gr.Textbox(visible=False, value=request.session_hash)
-                graphql_api_string = gr.Textbox(visible=False, value=graphql_url)
-                graphql_api_token = gr.Textbox(visible=False, value=api_token)
-                graphql_token_header = gr.Textbox(visible=False, value=api_token_header_name)
-                titles = gr.Textbox(value=process_message[2], interactive=False, label="GraphQL Types")
-                data_source = gr.Textbox(visible=False, value='graphql')
-                schema = gr.Textbox(visible=False, value='')
-                bot = gr.Chatbot(type='messages', label="GraphQL Chat Window", render_markdown=True, sanitize_html=False, show_label=True, render=False, visible=True, elem_classes="chatbot")
-                chat = gr.ChatInterface(
-                                    fn=chatbot_func,
-                                    type='messages',
-                                    chatbot=bot,
-                                    title="Chat with your GraphQL API",
-                                    examples=example_questions,
-                                    concurrency_limit=None,
-                                    additional_inputs=[session_hash, data_source, titles, schema, graphql_api_string, graphql_api_token, graphql_token_header]
-                                    )
-
-    def process_graphql(graphql_url, api_token, api_token_header_name, session_hash):
-        if graphql_url:
-            process_message = connect_graphql(graphql_url, api_token, api_token_header_name, session_hash)
-        return process_message
-
-if __name__ == "__main__":
-    demo.launch()

templates/sql_db.py

@@ -1,102 +0,0 @@
-import ast
-import gradio as gr
-from functions import example_question_generator, chatbot_func
-from data_sources import connect_sql_db
-from utils import message_dict
-
-with gr.Blocks() as demo:
-    with gr.Accordion("ℹ️  About the SQL Connector", open=False):
-        gr.HTML("""
-            <div class="max-w-4xl mx-auto text-center">
-                <div class="bg-blue-50 border border-blue-200 rounded-lg max-w-2xl mx-auto p-4">
-                    <p>Connect to a PostgreSQL database and query it using natural language.</p>
-                    <p style="font-weight:bold;">
-                        No credentials are retained — they are passed as session variables and disappear when you leave or refresh.
-                        Queries run through Pandas <code>read_sql_query</code>, which cannot delete, drop, or insert rows.
-                        Use caution connecting production databases to third-party tools.
-                    </p>
-                    <p>Contact me if you'd like this built for your organization with proper infrastructure and security controls.</p>
-                </div>
-            </div>
-        """)
-
-    gr.HTML("""
-        <div style="max-width:560px;margin:8px auto 4px;padding:8px 14px;background:#f0f9ff;
-                    border:1px solid #bae6fd;border-radius:8px;text-align:center;">
-            <p style="margin:0;font-size:13px;color:#0369a1;">
-                <i class="fas fa-flask" style="margin-right:6px;"></i>
-                <strong>Demo credentials pre-filled.</strong>
-                &nbsp;Replace with your own database details to analyze your own data.
-            </p>
-        </div>
-    """)
-
-    sql_url = gr.Textbox(label="URL", value="virtual-data-analyst-pg.cyetm2yjzppu.us-west-1.rds.amazonaws.com")
-    with gr.Row():
-        sql_port = gr.Textbox(label="Port", value="5432")
-        sql_user = gr.Textbox(label="Username", value="postgres")
-        sql_pass = gr.Textbox(label="Password", value="Vda-1988", type="password")
-        sql_db_name = gr.Textbox(label="Database Name", value="dvdrental")
-
-    submit = gr.Button(value="Connect", variant="primary")
-
-    @gr.render(inputs=[sql_url, sql_port, sql_user, sql_pass, sql_db_name], triggers=[submit.click])
-    def sql_chat(request: gr.Request, url=sql_url.value, sql_port=sql_port.value, sql_user=sql_user.value, sql_pass=sql_pass.value, sql_db_name=sql_db_name.value):
-        if request.session_hash not in message_dict:
-            message_dict[request.session_hash] = {}
-        message_dict[request.session_hash]['sql'] = None
-        if url:
-            print("SQL APP")
-            process_message = process_sql_db(url, sql_user, sql_port, sql_pass, sql_db_name, request.session_hash)
-            gr.HTML(value=process_message[1], padding=False)
-            if process_message[0] == "success":
-                if "virtual-data-analyst-pg.cyetm2yjzppu.us-west-1.rds.amazonaws.com" in url:
-                    example_questions = [
-                                            ["Describe the dataset"],
-                                            ["What is the total revenue generated by each store?"],
-                                            ["Can you generate and display a bar chart of film category to number of films in that category?"],
-                                            ["Can you generate a pie chart showing the top 10 most rented films by revenue?"],
-                                            ["Can you generate a line chart of rental revenue over time?"],
-                                            ["What is the relationship between film length and rental frequency?"]
-                                        ]
-                else:
-                    try:
-                        generated_examples = ast.literal_eval(example_question_generator(request.session_hash, 'sql', sql_db_name, process_message[2], ""))
-                        example_questions = [["Describe the dataset"]]
-                        for example in generated_examples:
-                            example_questions.append([example])
-                    except Exception as e:
-                        print("SQL QUESTION GENERATION ERROR")
-                        print(e)
-                        example_questions = [
-                                            ["Describe the dataset"],
-                                            ["List the tables in the database"],
-                                            ["What could this data be used for?"],
-                                        ]
-                session_hash = gr.Textbox(visible=False, value=request.session_hash)
-                db_url = gr.Textbox(visible=False, value=url)
-                db_port = gr.Textbox(visible=False, value=sql_port)
-                db_user = gr.Textbox(visible=False, value=sql_user)
-                db_pass = gr.Textbox(visible=False, value=sql_pass)
-                db_name = gr.Textbox(visible=False, value=sql_db_name)
-                titles = gr.Textbox(value=process_message[2], interactive=False, label="SQL Tables")
-                data_source = gr.Textbox(visible=False, value='sql')
-                schema = gr.Textbox(visible=False, value='')
-                bot = gr.Chatbot(type='messages', label="SQL DB Chat Window", render_markdown=True, sanitize_html=False, show_label=True, render=False, visible=True, elem_classes="chatbot")
-                chat = gr.ChatInterface(
-                                    fn=chatbot_func,
-                                    type='messages',
-                                    chatbot=bot,
-                                    title="Chat with your Database",
-                                    examples=example_questions,
-                                    concurrency_limit=None,
-                                    additional_inputs=[session_hash, data_source, titles, schema, db_url, db_port, db_user, db_pass, db_name]
-                                    )
-
-    def process_sql_db(url, sql_user, sql_port, sql_pass, sql_db_name, session_hash):
-        if url:
-            process_message = connect_sql_db(url, sql_user, sql_port, sql_pass, sql_db_name, session_hash)
-        return process_message
-
-if __name__ == "__main__":
-    demo.launch()

tools.py

@@ -0,0 +1,54 @@
+import sqlite3
+
+def tools_call(session_hash):
+    connection = sqlite3.connect(f'data_source_{session_hash}.db')
+    print("Querying Database in Tools.py");
+    cur=connection.execute('select * from data_source')
+    columns = [i[0] for i in cur.description]
+    print("COLUMNS 2")
+    print(columns)
+    cur.close()
+    connection.close()
+
+    return [
+        {
+            "type": "function",
+            "function": {
+                "name": "sql_query_func",
+                "description": f"This a tool useful to query a SQL table called 'data_source' with the following Columns: {columns}",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "queries": {
+                            "type": "array",
+                            "description": "The query to use in the search. Infer this from the user's message. It should be a question or a statement",
+                            "items": {
+                                "type": "string",
+                            }
+                        }
+                    },
+                    "required": ["question"],
+                },
+            },
+        },
+        {
+        "type": "function",
+            "function": {
+                "name": "rag_pipeline_func",
+                "description": f"This a tool useful to query a SQL table called 'data_source' with the following Columns: {columns}",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "queries": {
+                            "type": "array",
+                            "description": "The query to use in the search. Infer this from the user's message. It should be a question or a statement",
+                            "items": {
+                                "type": "string",
+                            }
+                        }
+                    },
+                    "required": ["question"],
+                },
+            }, 
+        }
+    ]

tools/chart_tools.py

@@ -1,308 +0,0 @@
-# Shared parameter snippets reused across chart tool schemas.
-# Update here to change the description everywhere at once.
-
-_LAYOUT_PARAM = {
-    "type": "array",
-    "description": (
-        "Optional. An array containing a single JSON-formatted Plotly layout dictionary. "
-        "Use to set chart title, axis labels, colours, fonts, and other layout properties. "
-        "Example: [{\"title\": \"Monthly Sales\", \"xaxis\": {\"title\": \"Month\"}}]"
-    ),
-    "items": {"type": "string"},
-}
-
-_TRACE_STYLE_PARAM = {
-    "type": "array",
-    "description": (
-        "Optional. An array containing a single JSON-formatted Plotly trace styling dictionary. "
-        "Use to control visual properties such as line colour, opacity, and marker style. "
-        "Do NOT include 'x', 'y', or 'type' keys — those are set automatically from query.csv."
-    ),
-    "items": {"type": "string"},
-}
-
-chart_tool_schemas = [
-    {
-        "name": "scatter_chart_generation_func",
-        "description": (
-            "Generates a Plotly scatter plot from query.csv data. "
-            "Use when the user wants to visualise the relationship between two numeric columns, "
-            "create a bubble chart (via the size parameter), or overlay a trendline. "
-            "Returns an HTML iframe — display it verbatim in the chat."
-        ),
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "x_column": {
-                    "type": "array",
-                    "description": (
-                        "One or more column names from query.csv to plot on the x-axis. "
-                        "Multiple columns produce multiple series, each plotted against y_column."
-                    ),
-                    "items": {"type": "string"},
-                },
-                "y_column": {
-                    "type": "string",
-                    "description": "Column name from query.csv to plot on the y-axis.",
-                },
-                "category": {
-                    "type": "string",
-                    "description": "Optional column name used to colour-code points by a categorical grouping.",
-                },
-                "trendline": {
-                    "type": "string",
-                    "description": (
-                        "Optional trendline type. One of: 'ols' (linear regression), "
-                        "'lowess' (local smoothing), 'rolling', 'ewm', 'expanding'. "
-                        "Requires trendline_options when using 'lowess', 'rolling', or 'ewm'."
-                    ),
-                },
-                "trendline_options": {
-                    "type": "array",
-                    "description": (
-                        "Required when trendline is 'lowess', 'rolling', or 'ewm'. "
-                        "An array containing a single JSON-formatted dict of trendline options "
-                        "(e.g. [{\"window\": 7}] for a 7-point rolling average)."
-                    ),
-                    "items": {"type": "string"},
-                },
-                "marginal_x": {
-                    "type": "string",
-                    "description": "Optional marginal distribution plot along the x-axis. One of: 'histogram', 'rug', 'box', 'violin'.",
-                },
-                "marginal_y": {
-                    "type": "string",
-                    "description": "Optional marginal distribution plot along the y-axis. One of: 'histogram', 'rug', 'box', 'violin'.",
-                },
-                "size": {
-                    "type": "string",
-                    "description": "Optional column name whose values control the size of each point (bubble chart). Negative values are clamped to zero.",
-                },
-                "data": _TRACE_STYLE_PARAM,
-                "layout": _LAYOUT_PARAM,
-            },
-            "required": ["x_column", "y_column"],
-        },
-    },
-    {
-        "name": "line_chart_generation_func",
-        "description": (
-            "Generates a Plotly line chart from query.csv data. "
-            "Use for trends over time or any ordered sequence. "
-            "Returns an HTML iframe — display it verbatim in the chat."
-        ),
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "x_column": {
-                    "type": "string",
-                    "description": "Column name from query.csv for the x-axis (typically a date or ordered index).",
-                },
-                "y_column": {
-                    "type": "string",
-                    "description": "Column name from query.csv for the y-axis (numeric values).",
-                },
-                "category": {
-                    "type": "string",
-                    "description": "Optional column name used to split the data into multiple colour-coded lines.",
-                },
-                "data": _TRACE_STYLE_PARAM,
-                "layout": _LAYOUT_PARAM,
-            },
-            "required": ["x_column", "y_column"],
-        },
-    },
-    {
-        "name": "bar_chart_generation_func",
-        "description": (
-            "Generates a Plotly bar chart from query.csv data. "
-            "Use for comparing values across categories. Supports grouped/stacked bars via category, "
-            "and faceted subplots via facet_row or facet_col. "
-            "Returns an HTML iframe — display it verbatim in the chat."
-        ),
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "x_column": {
-                    "type": "string",
-                    "description": "Column name from query.csv for the x-axis (category labels).",
-                },
-                "y_column": {
-                    "type": "string",
-                    "description": "Column name from query.csv for the y-axis (numeric values).",
-                },
-                "category": {
-                    "type": "string",
-                    "description": "Optional column name used to colour-code bars into grouped or stacked series.",
-                },
-                "facet_row": {
-                    "type": "string",
-                    "description": "Optional column name. Creates one subplot row per unique value — useful for comparing distributions across a second dimension.",
-                },
-                "facet_col": {
-                    "type": "string",
-                    "description": "Optional column name. Creates one subplot column per unique value.",
-                },
-                "data": _TRACE_STYLE_PARAM,
-                "layout": _LAYOUT_PARAM,
-            },
-            "required": ["x_column", "y_column"],
-        },
-    },
-    {
-        "name": "pie_chart_generation_func",
-        "description": (
-            "Generates a Plotly pie chart from query.csv data. "
-            "Use when the user wants to show part-to-whole proportions. "
-            "Returns an HTML iframe — display it verbatim in the chat."
-        ),
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "values": {
-                    "type": "string",
-                    "description": "Column name from query.csv containing the numeric value for each slice.",
-                },
-                "names": {
-                    "type": "string",
-                    "description": "Column name from query.csv containing the label for each slice.",
-                },
-                "data": _TRACE_STYLE_PARAM,
-                "layout": _LAYOUT_PARAM,
-            },
-            "required": ["values", "names"],
-        },
-    },
-    {
-        "name": "histogram_generation_func",
-        "description": (
-            "Generates a Plotly histogram from query.csv data. "
-            "Use to show the frequency distribution of a numeric column. "
-            "Supports normalisation (percent, probability, density) and aggregation functions per bin. "
-            "Returns an HTML iframe — display it verbatim in the chat."
-        ),
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "x_column": {
-                    "type": "string",
-                    "description": "Column name from query.csv whose values are binned on the x-axis.",
-                },
-                "y_column": {
-                    "type": "string",
-                    "description": "Optional column name aggregated per bin via histfunc (e.g. sum of sales per price bucket).",
-                },
-                "histnorm": {
-                    "type": "string",
-                    "description": "Optional normalisation. One of: 'percent', 'probability', 'density', 'probability density'.",
-                },
-                "category": {
-                    "type": "string",
-                    "description": "Optional column name used to overlay multiple colour-coded histograms.",
-                },
-                "histfunc": {
-                    "type": "string",
-                    "description": "Optional aggregation function applied to y_column per bin. One of: 'avg', 'sum', 'count'.",
-                },
-                "data": _TRACE_STYLE_PARAM,
-                "layout": _LAYOUT_PARAM,
-            },
-            "required": ["x_column"],
-        },
-    },
-    {
-        "name": "box_chart_generation_func",
-        "description": (
-            "Generates a Plotly box plot from query.csv data. "
-            "Use to visualise the distribution of a numeric column and identify outliers. "
-            "Especially useful for comparing distributions across categories. "
-            "Returns an HTML iframe — display it verbatim in the chat."
-        ),
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "y_column": {
-                    "type": "string",
-                    "description": "Column name from query.csv containing the numeric values to distribute on the y-axis.",
-                },
-                "x_column": {
-                    "type": "string",
-                    "description": "Optional column name. Groups data into one box per unique value on the x-axis.",
-                },
-                "category": {
-                    "type": "string",
-                    "description": "Optional column name used to colour-code boxes by a secondary grouping.",
-                },
-                "layout": _LAYOUT_PARAM,
-            },
-            "required": ["y_column"],
-        },
-    },
-    {
-        "name": "correlation_heatmap_func",
-        "description": (
-            "Computes pairwise Pearson correlations between numeric columns in query.csv and renders "
-            "the result as a colour-coded heatmap (blue = positive, red = negative). "
-            "Use when the user asks which variables are related, correlated, or associated with each other. "
-            "Returns an HTML iframe — display it verbatim in the chat."
-        ),
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "columns": {
-                    "type": "array",
-                    "description": "Optional list of numeric column names to include in the matrix. If omitted, all numeric columns from query.csv are used. Avoid ID or index columns.",
-                    "items": {"type": "string"},
-                },
-            },
-            "required": [],
-        },
-    },
-    {
-        "name": "rolling_stats_func",
-        "description": (
-            "Generates a rolling statistics / moving average chart from query.csv data. "
-            "Overlays rolling aggregations (mean, std, min, max) on top of the original series. "
-            "Use when the user asks for a moving average, rolling average, rolling statistics, or wants to smooth a time series. "
-            "Returns an HTML iframe — display it verbatim in the chat."
-        ),
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "x_column": {
-                    "type": "string",
-                    "description": "Column name from query.csv for the x-axis — typically a date or sequential index.",
-                },
-                "y_column": {
-                    "type": "string",
-                    "description": "Column name from query.csv containing the numeric values to compute rolling stats on.",
-                },
-                "window": {
-                    "type": "integer",
-                    "description": "Rolling window size in number of rows. Default 7. Infer from the user's request.",
-                },
-                "stats": {
-                    "type": "array",
-                    "description": "Statistics to overlay. Valid values: 'mean', 'std', 'min', 'max'. Defaults to ['mean'] if omitted.",
-                    "items": {"type": "string"},
-                },
-                "category": {
-                    "type": "string",
-                    "description": "Optional column name to group the data, producing separate rolling stat lines per group.",
-                },
-                "layout": _LAYOUT_PARAM,
-            },
-            "required": ["x_column", "y_column"],
-        },
-    },
-    {
-        "name": "table_generation_func",
-        "description": (
-            "Formats query.csv results as a styled HTML table. "
-            "Use when the user wants to view raw query results in a readable format, "
-            "or when result data is too large to describe in text. Displays up to 200 rows. "
-            "Returns an HTML table — display it verbatim in the chat."
-        ),
-        "parameters": {"type": "object", "properties": {}},
-    },
-]

tools/stats_tools.py

@@ -1,130 +0,0 @@
-stats_tool_schemas = [
-    {
-        "name": "descriptive_stats_func",
-        "description": (
-            "Computes summary statistics for numeric columns in query.csv: "
-            "count, mean, std, min, 25th/50th/75th percentile, and max. "
-            "Use when the user asks for summary statistics, descriptive statistics, or a statistical overview. "
-            "Returns a formatted HTML table."
-        ),
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "columns": {
-                    "type": "array",
-                    "description": "Optional list of column names to include. If omitted, all numeric columns from query.csv are used. Avoid ID or index columns.",
-                    "items": {"type": "string"},
-                },
-            },
-            "required": [],
-        },
-    },
-    {
-        "name": "kmeans_clustering_func",
-        "description": (
-            "Runs K-Means clustering on numeric feature columns from query.csv. "
-            "Groups rows into k clusters, displays a scatter plot coloured by cluster assignment, "
-            "and returns a centroid summary table showing the mean of each feature per cluster. "
-            "Use when the user asks to cluster the data, find natural segments or groups, or apply K-Means. "
-            "Returns an HTML iframe and summary table."
-        ),
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "feature_columns": {
-                    "type": "array",
-                    "description": "List of numeric column names from query.csv to use as clustering features.",
-                    "items": {"type": "string"},
-                },
-                "x_column": {
-                    "type": "string",
-                    "description": "Column name from query.csv for the x-axis of the scatter plot. Usually one of the feature columns.",
-                },
-                "y_column": {
-                    "type": "string",
-                    "description": "Column name from query.csv for the y-axis of the scatter plot. Usually one of the feature columns.",
-                },
-                "n_clusters": {
-                    "type": "integer",
-                    "description": "Number of clusters (k). Default 3. Infer from the user's request.",
-                },
-                "layout": {
-                    "type": "array",
-                    "description": "Optional. An array containing a single JSON-formatted Plotly layout dictionary.",
-                    "items": {"type": "string"},
-                },
-            },
-            "required": ["feature_columns", "x_column", "y_column"],
-        },
-    },
-    {
-        "name": "hypothesis_test_func",
-        "description": (
-            "Performs a statistical hypothesis test on query.csv data and returns a formatted results table "
-            "with test statistic, p-value, and significance at α=0.05. "
-            "Supported tests:\n"
-            "- 't_test_independent': compare means of a numeric column across two groups "
-            "(requires group_column; use group_values if the column has more than 2 unique values).\n"
-            "- 't_test_one_sample': test whether a column's mean equals a hypothesized value (requires pop_mean).\n"
-            "- 'chi_square': test independence between two categorical columns (requires column and column2)."
-        ),
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "test_type": {
-                    "type": "string",
-                    "description": "Test to run. One of: 't_test_independent', 't_test_one_sample', 'chi_square'.",
-                },
-                "column": {
-                    "type": "string",
-                    "description": "Primary column for the test. Numeric for t-tests; first categorical column for chi-square.",
-                },
-                "column2": {
-                    "type": "string",
-                    "description": "Second categorical column. Required for 'chi_square'.",
-                },
-                "group_column": {
-                    "type": "string",
-                    "description": "Grouping column. Required for 't_test_independent'. Must have exactly 2 unique values, or specify group_values.",
-                },
-                "group_values": {
-                    "type": "array",
-                    "description": "Exactly 2 group labels to compare. Use when group_column has more than 2 unique values.",
-                    "items": {"type": "string"},
-                },
-                "pop_mean": {
-                    "type": "number",
-                    "description": "Hypothesized population mean (μ₀). Required for 't_test_one_sample'.",
-                },
-            },
-            "required": ["test_type", "column"],
-        },
-    },
-    {
-        "name": "regression_func",
-        "description": (
-            "Runs an OLS linear regression on query.csv data. "
-            "Use when the user wants to model the relationship between variables, assess predictors, or run a regression. "
-            "Returns a regression summary (coefficients, R², p-values) and a scatter plot with the fitted line as an HTML iframe."
-        ),
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "independent_variables": {
-                    "type": "array",
-                    "description": "Column names from query.csv to use as independent (predictor) variables.",
-                    "items": {"type": "string"},
-                },
-                "dependent_variable": {
-                    "type": "string",
-                    "description": "Column name from query.csv to use as the dependent (outcome) variable.",
-                },
-                "category": {
-                    "type": "string",
-                    "description": "Optional column name used to colour-code points and fit separate regression lines per group.",
-                },
-            },
-            "required": ["independent_variables", "dependent_variable"],
-        },
-    },
-]

tools/tools.py

@@ -1,130 +0,0 @@
-from .stats_tools import stats_tool_schemas
-from .chart_tools import chart_tool_schemas
-
-def tools_call(session_hash, data_source, titles):
-    from haystack.tools import Tool
-
-    _noop = lambda **kwargs: None
-
-    def make_tool(schema):
-        return Tool(
-            name=schema["name"],
-            description=schema["description"],
-            parameters=schema["parameters"],
-            function=_noop,
-        )
-
-    titles_string = (titles[:625] + '..') if len(titles) > 625 else titles
-
-    query_tool_schemas = {
-        'file_upload': {
-            "name": "query_func",
-            "description": f"""This is a tool useful to query a SQLite table called 'data_source' with the following Columns: {titles_string}.
-            There may also be more columns in the table if the number of columns is too large to process.
-            This function also saves the results of the query to csv file called query.csv.""",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "queries": {
-                        "type": "string",
-                        "description": "The query to use in the search. Infer this from the user's message. It should be a question or a statement."
-                    }
-                },
-                "required": ["queries"]
-            },
-        },
-        'sql': {
-            "name": "query_func",
-            "description": f"""This is a tool useful to query a PostgreSQL database with the following tables, {titles_string}.
-            There may also be more tables in the database if the number of tables is too large to process.
-            This function also saves the results of the query to csv file called query.csv.""",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "queries": {
-                        "type": "string",
-                        "description": "The PostgreSQL query to use in the search. Infer this from the user's message. It should be a question or a statement."
-                    }
-                },
-                "required": ["queries"]
-            },
-        },
-        'doc_db': {
-            "name": "query_func",
-            "description": f"""This is a tool useful to build an aggregation pipeline to query a MongoDB NoSQL document database with the following collections, {titles_string}.
-            There may also be more collections in the database if the number of collections is too large to process.
-            This function also saves the results of the query to a csv file called query.csv.""",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "queries": {
-                        "type": "string",
-                        "description": "The MongoDB aggregation pipeline to use in the search. Infer this from the user's message. It should be a question or a statement."
-                    },
-                    "db_collection": {
-                        "type": "string",
-                        "description": "The MongoDB collection to use in the search. Infer this from the user's message. It should be a question or a statement."
-                    }
-                },
-                "required": ["queries", "db_collection"]
-            },
-        },
-        'graphql': [
-            {
-                "name": "query_func",
-                "description": f"""This is a tool useful to build a GraphQL query for a GraphQL API endpoint with the following types, {titles_string}.
-                There may also be more types in the GraphQL endpoint if the number of types is too large to process.
-                This function also saves the results of the query to a csv file called query.csv.""",
-                "parameters": {
-                    "type": "object",
-                    "properties": {
-                        "queries": {
-                            "type": "string",
-                            "description": "The GraphQL query to use in the search. Infer this from the user's message. It should be a question or a statement."
-                        }
-                    },
-                    "required": ["queries"]
-                },
-            },
-            {
-                "name": "graphql_schema_query",
-                "description": f"""This is a tool useful to query a GraphQL type and receive back information about its schema. This is useful because
-                the GraphQL introspection query is too large to be ingested all at once and this allows us to query the schema one type at a time to
-                view it in manageable bites. You may realize after viewing the schema, that the type you selected was not appropriate for the question
-                you are attempting answer. You may then query additional types to find the appropriate types to use for your GraphQL API query.""",
-                "parameters": {
-                    "type": "object",
-                    "properties": {
-                        "graphql_type": {
-                            "type": "string",
-                            "description": "The GraphQL type that we want to view the schema of in order to make the proper query with our graphql_query_func. Infer this from the user's message. It should be a question or a statement."
-                        }
-                    },
-                    "required": ["graphql_type"]
-                },
-            },
-            {
-                "name": "graphql_csv_query",
-                "description": f"""This is a tool useful to SQL query our query.csv file that is generated from our GraphQL query. This is useful in a situation
-                where the results of the GraphQL query need additional querying to answer the user question. The query.csv file is converted to a Pandas dataframe
-                and we query that dataframe with SQL on a table called 'query' before converting it back to a csv file.""",
-                "parameters": {
-                    "type": "object",
-                    "properties": {
-                        "csv_query": {
-                            "type": "string",
-                            "description": "The pandas dataframe SQL query to use in the search. The table that we query is named 'query'. Infer this from the user's message. It should be a question or a statement."
-                        }
-                    },
-                    "required": ["csv_query"]
-                },
-            },
-        ]
-    }
-
-    source_schemas = query_tool_schemas[data_source]
-    source_tools = [make_tool(s) for s in (source_schemas if isinstance(source_schemas, list) else [source_schemas])]
-    chart_tools = [make_tool(s) for s in chart_tool_schemas]
-    stats_tools = [make_tool(s) for s in stats_tool_schemas]
-
-    return source_tools + chart_tools + stats_tools

utils.py

@@ -1,9 +0,0 @@
-from pathlib import Path
-
-current_dir = Path(__file__).parent
-
-TEMP_DIR = current_dir / 'temp'
-
-message_dict = {}
-api_key_store = {}
-model_store = {}