Upload folder using huggingface_hub

.gitignore

@@ -0,0 +1,52 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Packages
+*.egg
+*.egg-info/
+dist/
+build/
+eggs/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual environments
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+.venv
+
+# SQLite database
+*.db
+
+# dotenv environment variables
+.env
+
+# Jupyter Notebook checkpoints
+.ipynb_checkpoints
+
+# Gradio cache and state files
+gradio_cache/
+gradio_state/
+
+# Logs and databases
+*.log
+*.sqlite3
+*.sqlite
+query_logs.db
+
+# OS specific files
+.DS_Store
+Thumbs.db
+
+# Backup
+BU/

README.md

@@ -1,12 +1,6 @@
 ---
-title: Ai Eee Sql Gen
-emoji: 👁
-colorFrom: red
-colorTo: gray
-sdk: gradio
-sdk_version: 4.44.0
+title: ai_eee_sql_gen
 app_file: app.py
-pinned: false
+sdk: gradio
+sdk_version: 4.43.0
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,140 @@
+import gradio as gr
+import logging
+
+from openai_integration import generate_sql
+from db import get_last_50_saved_queries, initialize_local_db, export_saved_queries_to_csv, execute_sql_query, fetch_and_save_schema, show_last_50_saved_queries  # Import the correct function
+
+
+# Initialize logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname=s - %(message)s')
+
+# Call the function to ensure the table is created
+initialize_local_db()
+
+# Function to handle user query input and SQL generation with progress
+def query_database(nl_query, progress=gr.Progress()):
+    try:
+        progress(0, desc="Starting Query Process")
+
+        # Generate SQL and reformulated query
+        progress(0.2, desc="Generating Reformulated Query")
+        reformulated_query, sql_query = generate_sql(nl_query)
+
+        # Default empty result in case of SQL query failure
+        execution_result = []
+
+        # If we have a SQL query, attempt execution
+        if sql_query and not sql_query.startswith("Error"):
+            progress(0.5, desc="Executing SQL Query")
+            execution_result = execute_sql_query(sql_query)
+            
+            # Ensure execution_result is in a valid format for a DataFrame
+            if not isinstance(execution_result, list) or len(execution_result) == 0:
+                execution_result = [["No results available."]]
+        else:
+            execution_result = [["No results available."]]
+
+        progress(1, desc="Query Completed")
+        return reformulated_query, sql_query, execution_result
+
+    except Exception as e:
+        logging.error(f"Error during query generation or execution: {e}")
+        return "Error during query processing.", "", [["No results available due to an error."]]
+
+# Function to update the schema when requested
+def update_schema():
+    schema_info = fetch_and_save_schema()
+    
+    # Case 1: Check if there is an actual error in the schema fetch process
+    if "error" in schema_info:
+        raise gr.Error("Error fetching schema from the database.", duration=3)
+    
+    # Case 2: Check if the schema is empty
+    if not schema_info:  # Empty dictionary or None
+        raise gr.Error("No schema data was returned. The schema is empty.", duration=3)
+    
+    # Case 3: Schema successfully fetched
+    return "Query executed successfully", gr.Info("DB Schema Updated ℹ️", duration=3)
+
+
+# Function to make hidden components visible after the process
+def continue_process():
+    # Ensure both the SQL and result outputs are shown (since reformulated_output is always visible now)
+    return gr.update(visible=True), gr.update(visible=True)
+
+# Function to reset the interface to its initial state
+def reset_interface():
+    return gr.update(value=""), gr.update(value=""), gr.update(visible=False), gr.update(visible=False), gr.update(interactive=False)
+
+# Enable the submit button only when text is entered
+def update_button_state(text):
+    if text.strip():
+        return gr.update(interactive=True)
+    else:
+        return gr.update(interactive=False)
+
+# Gradio interface setup
+with gr.Blocks(theme=gr.themes.Soft(font=[gr.themes.GoogleFont("Ubuntu"), "Arial", "sans-serif"], text_size='sm')) as ydcoza_face:
+    text_input = gr.Textbox(lines=2, label="Text Query")
+    examples = gr.Examples(examples=[
+        "I'm trying to figure out which agents are the busiest. Can you show me like the top few agents who have a lot on their plate? I'd like to see their names and maybe their contact info if we have it.", 
+        "I need to get an overview of our classes. Could you pull up a list that shows what each class is about and which client it's for? Oh, and it would be great to know how many students are in each class. Maybe order it so the biggest classes are at the top?",
+        "Can you give me an overview of all our classes? I'd like to see how many students are in each class and how diverse they are in terms of race. It would be helpful to see the class subject and location too. Don't leave out any classes, even if they have few or no students.",
+        "I'm curious about how our classes are doing. Can you show me a list of all the classes and their latest test results or evaluations? It would be helpful to see where each class is located too.",
+        "I heard some agents got moved around recently. Can you find out which agents have been switched to different classes? I'd like to know where they were before, where they are now, and maybe why they were moved."
+        ],example_labels=["01","02","03","04","05"], label="Demo Natural Language Queries",inputs=[text_input])
+    reformulated_output = gr.Textbox(lines=2, label="Optimised Query", elem_id='ydcoza_markdown_output_desc')
+    sql_output = gr.Code(label="Generated SQL", visible=False)
+    sql_result_output = gr.Dataframe(label="Query Results", elem_id='result_output', visible=False)  # Dataframe for SQL results
+    start_button = gr.Button("Submit Text Query", elem_id='ydcoza_gradio_button', interactive=False)
+
+    # Add reset button to reset the interface
+    reset_button = gr.Button("Reset Interface", elem_id='ydcoza_gradio_button_reset')
+    reset_button.click(
+        fn=reset_interface, 
+        inputs=[], 
+        outputs=[text_input, reformulated_output, sql_output, sql_result_output, start_button]
+    )
+    gr.HTML("""
+            <span class="ydcoza_gradio_banner">View The last 50 Queries generated in Table format.</span>
+            """)
+    saved_queries_output = gr.Dataframe(label="Last 50 Saved Queries", headers=["Query", "Optimised Query", "SQL", "Timestamp"], interactive=True, visible=False)
+    # Show the last 50 saved queries when button is clicked
+    show_saved_queries_button = gr.Button("View Queries", elem_id='ydcoza_gradio_button')
+    show_saved_queries_button.click(show_last_50_saved_queries, outputs=saved_queries_output).then(
+        lambda: gr.update(visible=True), outputs=saved_queries_output  # Make the saved queries visible
+    )
+    gr.HTML("""
+            <span class="ydcoza_gradio_banner">Download the generated Queries in .csv for you to explore.</span>
+            """)
+    csv_file_output = gr.File(label="Download CSV", visible=False)  # Initially hidden
+    download_csv_button = gr.Button("Download Queries", elem_id='ydcoza_gradio_button')
+    download_csv_button.click(export_saved_queries_to_csv, outputs=csv_file_output).then(
+        lambda: gr.update(visible=True), outputs=csv_file_output  # Make the file download visible
+    )
+    gr.HTML("""
+            <span class="ydcoza_gradio_banner">If you made changes to the database structure we need to import the latest DB Schema.</span>
+            """)
+    # Add a button to pull the latest schema and save it to schema.json
+    fetch_schema_button = gr.Button("Fetch Latest Schema", elem_id='ydcoza_gradio_button')
+    # fetch_schema_button.click(update_schema, outputs=[gr.Textbox(label="Schema Update Status")])
+    fetch_schema_button.click(update_schema)
+
+    # Setup the button click to trigger the process and show results
+    text_input.change(fn=update_button_state, inputs=text_input, outputs=start_button)
+    start_button.click(
+        fn=query_database, 
+        inputs=[text_input], 
+        outputs=[reformulated_output, sql_output, sql_result_output]
+    ).then(
+        continue_process, 
+        outputs=[sql_output, sql_result_output]
+    ).then(
+        lambda: gr.update(interactive=False), outputs=start_button  # Disable the submit button after submission
+    )
+
+# Launch the Gradio interface
+if __name__ == "__main__":
+    # ydcoza_face.launch(auth=auth_users, auth_message="Demo Login, Username: admin & Password: 1234")
+    # run gradio deploy in Terminal
+    ydcoza_face.launch()

db.py

@@ -0,0 +1,245 @@
+import os
+import json
+import psycopg2
+from psycopg2 import pool
+import sqlite3
+from dotenv import load_dotenv
+import logging
+import csv
+
+# Load environment variables
+load_dotenv()
+
+# Initialize logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+# PostgreSQL connection pool setup
+db_pool = psycopg2.pool.SimpleConnectionPool(
+    minconn=1,
+    maxconn=5,
+    user=os.getenv("db_user"),
+    password=os.getenv("db_password"),
+    host=os.getenv("db_host"),
+    dbname=os.getenv("db_name"),
+    port=os.getenv("db_port", 5432)
+)
+
+# SQLite DB path
+db_path = os.getenv("DB_PATH", "./query_logs.db")
+
+# PostgreSQL: Function to fetch schema and save to schema.json
+def fetch_and_save_schema():
+    try:
+        logging.info("Fetching schema from the database...")
+
+        conn = db_pool.getconn()
+        cursor = conn.cursor()
+
+        # Query to retrieve all table names and their comments
+        cursor.execute("""
+        SELECT table_name, obj_description(('public.' || table_name)::regclass) as table_comment
+        FROM information_schema.tables
+        WHERE table_schema = 'public';
+        """)
+        tables = cursor.fetchall()
+
+        # Build the schema information in JSON format
+        schema_info = {}
+
+        for table_name, table_comment in tables:
+            schema_info[table_name] = {
+                "comment": table_comment,
+                "columns": [],
+                "foreign_keys": []
+            }
+
+            # Fetch column details and comments for each table
+            cursor.execute(f"""
+            SELECT 
+                c.column_name, 
+                c.data_type, 
+                col_description(('public.' || c.table_name)::regclass, ordinal_position) as column_comment
+            FROM information_schema.columns c
+            WHERE c.table_name = '{table_name}';
+            """)
+            columns = cursor.fetchall()
+
+            for column_name, data_type, column_comment in columns:
+                schema_info[table_name]["columns"].append({
+                    "name": column_name,
+                    "data_type": data_type,
+                    "comment": column_comment
+                })
+
+            # Fetch foreign key relationships for each table
+            cursor.execute(f"""
+            SELECT 
+                kcu.column_name, 
+                ccu.table_name AS foreign_table_name, 
+                ccu.column_name AS foreign_column_name
+            FROM information_schema.table_constraints AS tc
+            JOIN information_schema.key_column_usage AS kcu
+              ON tc.constraint_name = kcu.constraint_name
+              AND tc.table_schema = kcu.table_schema
+            JOIN information_schema.constraint_column_usage AS ccu
+              ON ccu.constraint_name = tc.constraint_name
+            WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name = '{table_name}';
+            """)
+            foreign_keys = cursor.fetchall()
+
+            for column_name, foreign_table_name, foreign_column_name in foreign_keys:
+                schema_info[table_name]["foreign_keys"].append({
+                    "column": column_name,
+                    "references": {
+                        "table": foreign_table_name,
+                        "column": foreign_column_name
+                    }
+                })
+
+        cursor.close()
+        db_pool.putconn(conn)
+
+        # Save the schema to a JSON file
+        with open("schema.json", "w") as schema_file:
+            json.dump(schema_info, schema_file, indent=2)
+
+        logging.info("Schema fetched and saved to schema.json.")
+        return schema_info
+    except Exception as e:
+        logging.error(f"Error fetching schema: {e}")
+        return {"error": str(e)}
+
+# PostgreSQL: Function to execute SQL query
+def execute_sql_query(sql_query):
+    try:
+        conn = db_pool.getconn()
+        cursor = conn.cursor()
+        cursor.execute(sql_query)  # Execute the query
+        result = cursor.fetchall()  # Fetch all results
+
+        # Get column names from the cursor description
+        column_names = [desc[0] for desc in cursor.description]
+
+        cursor.close()
+        db_pool.putconn(conn)
+
+        # Format the result as a list of lists for Gradio Dataframe
+        return [column_names] + result
+    except Exception as e:
+        logging.error(f"Error executing SQL query: {e}")
+        return str(e)
+
+# SQLite: Initialize the local SQLite database
+def initialize_local_db():
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+
+    # Create table if it doesn't exist
+    cursor.execute('''
+    CREATE TABLE IF NOT EXISTS query_logs (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        natural_language_query TEXT,
+        reformulated_query TEXT,
+        generated_sql TEXT,
+        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+    );
+    ''')
+    conn.commit()
+    cursor.close()
+    conn.close()
+
+# SQLite: Function to save the query to the local database
+def save_query_to_local_db(nl_query, reformulated_query, sql_query):
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+        insert_query = '''
+        INSERT INTO query_logs (natural_language_query, reformulated_query, generated_sql)
+        VALUES (?, ?, ?);
+        '''
+        cursor.execute(insert_query, (nl_query, reformulated_query, sql_query))
+        conn.commit()
+        cursor.close()
+        conn.close()
+    except Exception as e:
+        logging.error(f"Error saving query: {e}")
+
+# SQLite: Function to get the last 50 saved queries
+def get_last_50_saved_queries():
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+        query = "SELECT natural_language_query, reformulated_query, generated_sql, created_at FROM query_logs ORDER BY created_at DESC LIMIT 50;"
+        cursor.execute(query)
+        rows = cursor.fetchall()
+        cursor.close()
+        conn.close()
+        return rows
+    except Exception as e:
+        logging.error(f"Error retrieving saved queries: {e}")
+        return str(e)
+
+# SQLite: Function to export saved queries to a CSV file
+def export_saved_queries_to_csv(file_path="./saved_queries.csv"):
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+
+        # Fetch all saved queries
+        cursor.execute("SELECT natural_language_query, reformulated_query, generated_sql, created_at FROM query_logs ORDER BY created_at DESC;")
+        rows = cursor.fetchall()
+
+        # Write the results to a CSV file
+        with open(file_path, 'w', newline='') as csvfile:
+            csv_writer = csv.writer(csvfile)
+            csv_writer.writerow(['Natural Language Query', 'Reformulated Query', 'Generated SQL', 'Timestamp'])
+            csv_writer.writerows(rows)
+
+        cursor.close()
+        conn.close()
+
+        return file_path
+    except Exception as e:
+        logging.error(f"Error exporting queries to CSV: {e}")
+        return str(e)
+
+def show_last_50_saved_queries():
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+        query = "SELECT natural_language_query, reformulated_query, generated_sql, created_at FROM query_logs ORDER BY created_at DESC LIMIT 50;"
+        cursor.execute(query)
+        rows = cursor.fetchall()
+        cursor.close()
+        conn.close()
+        return rows
+    except Exception as e:
+        logging.error(f"Error retrieving saved queries: {e}")
+        return str(e)
+
+
+# Function to reset (drop all tables) and recreate the schema
+def reset_sqlite_db():
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+
+    # Drop the query_logs table
+    cursor.execute("DROP TABLE IF EXISTS query_logs;")
+    print("Dropped query_logs table")
+
+    # Recreate the query_logs table
+    cursor.execute('''CREATE TABLE query_logs (
+                        id INTEGER PRIMARY KEY AUTOINCREMENT,
+                        natural_language_query TEXT,
+                        reformulated_query TEXT,
+                        generated_sql TEXT,
+                        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+                      );''')
+    print("Recreated the query_logs table.")
+
+    conn.commit()
+    cursor.close()
+    conn.close()
+
+# Uncomment the following line to reset the SQLite database when you run this script
+# reset_sqlite_db()

db_logging.py

@@ -0,0 +1,161 @@
+import os
+import sqlite3
+import csv
+from dotenv import load_dotenv
+
+# Load environment variables from .env file
+load_dotenv()
+
+# SQLite DB path from .env for saving results
+db_path = os.getenv("db_path", "./query_logs.db")
+
+# Function to initialize the SQLite database
+def initialize_local_db():
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+
+    # Check if the reformulated_query column exists
+    cursor.execute("PRAGMA table_info(query_logs);")
+    columns = [column[1] for column in cursor.fetchall()]
+
+    # If the reformulated_query column doesn't exist, alter the table
+    if 'reformulated_query' not in columns:
+        print("Altering table to add reformulated_query column...")
+        cursor.execute('''
+        CREATE TABLE IF NOT EXISTS query_logs_new (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            natural_language_query TEXT,
+            reformulated_query TEXT,
+            generated_sql TEXT,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        );
+        ''')
+
+        # Copy data from old table to new table
+        cursor.execute('''
+        INSERT INTO query_logs_new (id, natural_language_query, generated_sql, created_at)
+        SELECT id, natural_language_query, generated_sql, created_at FROM query_logs;
+        ''')
+
+        # Drop the old table
+        cursor.execute("DROP TABLE query_logs;")
+
+        # Rename the new table
+        cursor.execute("ALTER TABLE query_logs_new RENAME TO query_logs;")
+
+    conn.commit()
+    cursor.close()
+    conn.close()
+
+# Function to reset (drop all tables) and recreate the schema
+def reset_sqlite_db():
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+
+    # Drop the query_logs table
+    cursor.execute("DROP TABLE IF EXISTS query_logs;")
+    print("Dropped query_logs table")
+
+    # Recreate the query_logs table
+    cursor.execute('''CREATE TABLE query_logs (
+                        id INTEGER PRIMARY KEY AUTOINCREMENT,
+                        natural_language_query TEXT,
+                        reformulated_query TEXT,
+                        generated_sql TEXT,
+                        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+                      );''')
+    print("Recreated the query_logs table.")
+
+    conn.commit()
+    cursor.close()
+    conn.close()
+
+# Function to save the natural language query, reformulated query, and generated SQL to the local SQLite DB
+def save_query_to_local_db(nl_query, reformulated_query, sql_query):
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+        insert_query = '''
+        INSERT INTO query_logs (natural_language_query, reformulated_query, generated_sql)
+        VALUES (?, ?, ?);
+        '''
+        cursor.execute(insert_query, (nl_query, reformulated_query, sql_query))
+        conn.commit()
+        cursor.close()
+        conn.close()
+    except Exception as e:
+        print(f"Error saving query: {e}")
+
+# Function to retrieve saved queries from the local SQLite DB with pagination and search
+def get_saved_queries(page=1, per_page=10, search_term=None):
+    try:
+        offset = (page - 1) * per_page
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+
+        query = "SELECT natural_language_query, reformulated_query, generated_sql, created_at FROM query_logs"
+        if search_term:
+            query += f" WHERE natural_language_query LIKE '%{search_term}%' OR generated_sql LIKE '%{search_term}%'"
+        query += f" ORDER BY created_at DESC LIMIT {per_page} OFFSET {offset};"
+
+        cursor.execute(query)
+        rows = cursor.fetchall()
+
+        cursor.close()
+        conn.close()
+
+        return rows
+    except Exception as e:
+        return str(e)
+
+# Manually clear all data (for testing purposes)
+def clear_data():
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+    cursor.execute("DELETE FROM query_logs")
+    conn.commit()
+    cursor.close()
+    conn.close()
+
+# Function to retrieve the last 50 saved queries from the local SQLite DB
+def get_last_50_saved_queries():
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+
+        query = "SELECT natural_language_query, reformulated_query, generated_sql, created_at FROM query_logs ORDER BY created_at DESC LIMIT 50;"
+        cursor.execute(query)
+        rows = cursor.fetchall()
+
+        cursor.close()
+        conn.close()
+
+        return rows
+    except Exception as e:
+        return str(e)
+
+# Function to export all saved queries to a CSV file
+def export_saved_queries_to_csv(file_path="./saved_queries.csv"):
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+
+        # Fetch all saved queries (remove LIMIT 50)
+        cursor.execute("SELECT natural_language_query, reformulated_query, generated_sql, created_at FROM query_logs ORDER BY created_at DESC;")
+        rows = cursor.fetchall()
+
+        # Write the results to a CSV file
+        with open(file_path, 'w', newline='') as csvfile:
+            csv_writer = csv.writer(csvfile)
+            csv_writer.writerow(['Natural Language Query', 'Reformulated Query', 'Generated SQL', 'Timestamp'])
+            csv_writer.writerows(rows)
+
+        cursor.close()
+        conn.close()
+
+        return file_path
+    except Exception as e:
+        return str(e)
+
+# Uncomment the following line to reset the SQLite database when you run this script
+# reset_sqlite_db()

db_setup.py

@@ -0,0 +1,31 @@
+import os
+from dotenv import load_dotenv
+import sqlite3
+
+# Load environment variables
+load_dotenv()
+
+# Get database path from environment variable
+db_path = os.getenv("DB_PATH", "./query_logs.db")  # Default to './query_logs.db' if not set
+
+# Function to initialize SQLite database
+def initialize_local_db():
+    conn = sqlite3.connect(db_path)  # Use the DB path from .env
+    cursor = conn.cursor()
+
+    # Create table if it doesn't exist
+    cursor.execute('''
+    CREATE TABLE IF NOT EXISTS query_logs (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        natural_language_query TEXT,
+        generated_sql TEXT,
+        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+    );
+    ''')
+
+    conn.commit()
+    cursor.close()
+    conn.close()
+
+# Initialize the database on startup
+initialize_local_db()

openai_integration.py

@@ -0,0 +1,194 @@
+import os
+import openai
+import json
+from dotenv import load_dotenv
+from db_logging import save_query_to_local_db
+import logging
+
+GPT_MODEL = "gpt-4o-mini"
+
+# Load environment variables from .env file
+load_dotenv()
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+# Initialize the OpenAI client using API key from .env
+client = openai.OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+
+# Function to load schema from schema.json
+def load_schema_from_json():
+    try:
+        with open("schema.json", "r") as schema_file:
+            schema_info = json.load(schema_file)
+        logging.info("Schema loaded from schema.json")
+        return schema_info
+    except Exception as e:
+        logging.error(f"Error loading schema from schema.json: {e}")
+        return {"error": str(e)}
+
+# Function to build the schema description with foreign keys
+def build_schema_description(schema_info):
+    schema_description = ""
+
+    for table_name, table_details in schema_info.items():
+        schema_description += f"Table {table_name} (Comment: {table_details['comment']}):\n"
+        for column in table_details["columns"]:
+            schema_description += f"  - {column['name']} ({column['data_type']}) (Comment: {column['comment']})\n"
+        if table_details.get("foreign_keys"):
+            schema_description += "  Foreign Keys:\n"
+            for fk in table_details["foreign_keys"]:
+                schema_description += f"    - {fk['column']} references {fk['references']['table']}({fk['references']['column']})\n"
+    return schema_description
+
+# Function to reformulate the natural language query using OpenAI API
+def reformulate_query(nl_query, schema_description):
+    try:
+        # Adding examples to guide the reformulation process
+        examples = """
+        Example 1:
+        Input: "I'm trying to figure out which agents are the busiest. Can you show me like the top few agents who have a lot on their plate?"
+        Reformulated: "Show me the top 5 agents with the highest workload, including their names, emails, and specializations."
+
+        Example 2:
+        Input: "I need to get an overview of our classes. Could you pull up a list that shows what each class is about and which client it's for?"
+        Reformulated: "List all classes with their subjects, client names, and the number of learners in each class, sorted by the number of learners in descending order."
+        """
+
+        prompt = (
+            f"Database Schema:\n{schema_description}\n\n"
+            f"Using the database schema, reformulate the following natural language query to be more precise and clear for generating an SQL query:\n{nl_query}\n\n"
+            f"Here are some examples of how to reformulate the natural language query:\n{examples}\n\n"
+            f"Output only the natural language query string using the provided schema, don't prefix with anything and don't use markdown"
+        )
+
+        logging.info("Sending reformulation request to OpenAI...")
+        response = client.chat.completions.create(
+            model=GPT_MODEL,
+            messages=[
+                {"role": "system", "content": "You are an assistant that helps reformulate natural language queries and improve on them."},
+                {"role": "user", "content": prompt}
+            ],
+            max_tokens=300,
+            temperature=0.7
+        )
+
+        reformulated_query = response.choices[0].message.content.strip()
+
+        logging.info(f"Reformulated Query: {reformulated_query}")
+        return reformulated_query
+    except Exception as e:
+        logging.error(f"Error reformulating query: {e}")
+        return str(e)
+
+# Function to generate SQL from reformulated query using OpenAI API
+def generate_sql_from_reformulated(reformulated_query, schema_description):
+    try:
+        # Adding examples to guide the SQL generation process
+        examples = """
+        Example 1:
+        Input: 
+        "Show me the top 5 agents with the highest workload, including their names, emails, and specializations."
+        SQL: 
+        SELECT name, email, specialization, current_workload
+        FROM agents
+        ORDER BY current_workload DESC
+        LIMIT 5;
+
+        Example 2:
+        Input: 
+        "List all classes with their subjects, client names, and the number of learners in each class, sorted by the number of learners in descending order."
+        SQL: 
+        SELECT c.id AS class_id, c.subject, cl.name AS client_name, COUNT(l.id) AS learner_count
+        FROM classes c
+        JOIN clients cl ON c.client_id = cl.id
+        LEFT JOIN learners l ON c.id = l.class_id
+        GROUP BY c.id, c.subject, cl.name
+        ORDER BY learner_count DESC;
+
+        Example 3:
+        Input: 
+        "List all agents' work history, including the class they worked on, their role, and the start and end dates of each task."
+        SQL: 
+        SELECT a.name AS agent_name, awh.class_id, awh.role, awh.start_date, awh.end_date
+        FROM agents a
+        JOIN agent_work_history awh ON a.id = awh.agent_id
+        ORDER BY awh.start_date DESC;
+
+        Example 4:
+        Input: 
+        "Get a list of all classes with their subjects, the number of learners, and the progression status of each learner in the class."
+        SQL: 
+        SELECT c.id AS class_id, c.subject, COUNT(l.id) AS learner_count, p.progression_level
+        FROM classes c
+        LEFT JOIN learners l ON c.id = l.class_id
+        LEFT JOIN progressions p ON l.id = p.learner_id
+        GROUP BY c.id, c.subject, p.progression_level
+        ORDER BY learner_count DESC;      
+        """
+
+        prompt = (
+            f"Database Schema:\n{schema_description}\n\n"
+            f"Natural language query:\n{reformulated_query}\n\n"
+            f"Examples:\n{examples}\n\n"
+            f"Convert the natural language query into an SQL query that matches the schema.\n\n"
+            f"Use the Examples as a guide on what I expect you to do.\n\n"
+            f"Only return the RAW SQL that can be directly executed, don't prefix it with sql and don't end the sql with Quotation marks etc"
+        )
+
+        logging.info("Sending SQL generation request to OpenAI...")
+        response = client.chat.completions.create(
+            model=GPT_MODEL,
+            messages=[
+                {"role": "system", "content": "You are an assistant that converts natural language to SQL using a databse shema."},
+                {"role": "user", "content": prompt}
+            ],
+            max_tokens=300,
+            temperature=0.7
+        )
+
+        sql_query = response.choices[0].message.content.strip()
+
+        logging.info(f"SQL Query generated: {sql_query}")
+        return sql_query
+    except Exception as e:
+        logging.error(f"Error generating SQL: {e}")
+        return str(e)
+
+# Main function to reformulate the query first and then generate SQL
+def generate_sql(nl_query):
+    try:
+        # Load the schema from schema.json
+        schema_info = load_schema_from_json()
+
+        if isinstance(schema_info, str) and schema_info.startswith("Error"):
+            return "Error fetching schema", ""
+
+        # Build the schema description once and reuse it
+        schema_description = build_schema_description(schema_info)
+        logging.error(f"Build Shema Description: {schema_description}")
+
+        # Reformulate the query
+        reformulated_query = reformulate_query(nl_query, schema_description)
+
+        if "Error" in reformulated_query:
+            return reformulated_query, ""
+
+        # Generate SQL based on the reformulated query
+        sql_query = generate_sql_from_reformulated(reformulated_query, schema_description)
+
+        if "Error" in sql_query:
+            return reformulated_query, sql_query
+
+        # Save the query to the local database
+        save_query_to_local_db(nl_query, reformulated_query, sql_query)
+
+        return reformulated_query, sql_query
+
+    except openai.error.OpenAIError as e:
+        logging.error(f"OpenAI API error: {e}")
+        return "Error during interaction with OpenAI API.", ""
+
+    except Exception as e:
+        logging.error(f"General error during SQL process: {e}")
+        return "General error during SQL processing.", ""

requirements.txt

@@ -0,0 +1,65 @@
+aiofiles==23.2.1
+annotated-types==0.7.0
+anyio==4.4.0
+asarPy==1.0.1
+certifi==2024.8.30
+charset-normalizer==3.3.2
+click==8.1.7
+contourpy==1.3.0
+cycler==0.12.1
+distro==1.9.0
+fastapi==0.114.2
+ffmpy==0.4.0
+filelock==3.16.0
+fonttools==4.53.1
+fsspec==2024.9.0
+gradio==4.44.0
+gradio_client==1.3.0
+h11==0.14.0
+httpcore==1.0.5
+httpx==0.27.2
+huggingface-hub==0.24.7
+idna==3.10
+importlib_resources==6.4.5
+Jinja2==3.1.4
+jiter==0.5.0
+kiwisolver==1.4.7
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+matplotlib==3.9.2
+mdurl==0.1.2
+numpy==2.1.1
+openai==1.45.1
+orjson==3.10.7
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+psycopg2-binary==2.9.9
+pycairo==1.27.0
+pydantic==2.9.1
+pydantic_core==2.23.3
+pydub==0.25.1
+Pygments==2.18.0
+PyGObject==3.50.0
+pyparsing==3.1.4
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-multipart==0.0.9
+pytz==2024.2
+PyYAML==6.0.2
+requests==2.32.3
+rich==13.8.1
+ruff==0.6.5
+semantic-version==2.10.0
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.1
+starlette==0.38.5
+tomlkit==0.12.0
+tqdm==4.66.5
+typer==0.12.5
+typing_extensions==4.12.2
+tzdata==2024.1
+urllib3==2.2.3
+uvicorn==0.30.6
+websockets==12.0

schema.json

@@ -0,0 +1,802 @@
+{
+  "agent_assignments": {
+    "comment": null,
+    "columns": [
+      {
+        "name": "updated_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "agent_id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "class_id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "task_id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "assignment_date",
+        "data_type": "date",
+        "comment": null
+      },
+      {
+        "name": "reassigned_agent_id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "reassignment_date",
+        "data_type": "date",
+        "comment": null
+      },
+      {
+        "name": "created_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "reassignment_reason",
+        "data_type": "text",
+        "comment": null
+      },
+      {
+        "name": "status",
+        "data_type": "character varying",
+        "comment": null
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "agent_id",
+        "references": {
+          "table": "agents",
+          "column": "id"
+        }
+      },
+      {
+        "column": "class_id",
+        "references": {
+          "table": "classes",
+          "column": "id"
+        }
+      },
+      {
+        "column": "task_id",
+        "references": {
+          "table": "tasks",
+          "column": "id"
+        }
+      }
+    ]
+  },
+  "agent_availability": {
+    "comment": null,
+    "columns": [
+      {
+        "name": "created_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "updated_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "available_date",
+        "data_type": "date",
+        "comment": null
+      },
+      {
+        "name": "id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "agent_id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "availability_status",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "reason",
+        "data_type": "text",
+        "comment": null
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "agent_id",
+        "references": {
+          "table": "agents",
+          "column": "id"
+        }
+      }
+    ]
+  },
+  "agent_work_history": {
+    "comment": null,
+    "columns": [
+      {
+        "name": "updated_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "agent_id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "class_id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "task_id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "start_date",
+        "data_type": "date",
+        "comment": null
+      },
+      {
+        "name": "end_date",
+        "data_type": "date",
+        "comment": null
+      },
+      {
+        "name": "created_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "reassignment_id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "role",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "performance_notes",
+        "data_type": "text",
+        "comment": null
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "agent_id",
+        "references": {
+          "table": "agents",
+          "column": "id"
+        }
+      },
+      {
+        "column": "class_id",
+        "references": {
+          "table": "classes",
+          "column": "id"
+        }
+      },
+      {
+        "column": "task_id",
+        "references": {
+          "table": "tasks",
+          "column": "id"
+        }
+      }
+    ]
+  },
+  "agents": {
+    "comment": null,
+    "columns": [
+      {
+        "name": "updated_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "experience",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "current_workload",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "created_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "location",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "status",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "name",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "email",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "phone",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "specialization",
+        "data_type": "character varying",
+        "comment": null
+      }
+    ],
+    "foreign_keys": []
+  },
+  "assessments": {
+    "comment": null,
+    "columns": [
+      {
+        "name": "id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "class_id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "assessment_date",
+        "data_type": "date",
+        "comment": null
+      },
+      {
+        "name": "created_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "updated_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "assessment_type",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "assessor_name",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "result",
+        "data_type": "character varying",
+        "comment": null
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "class_id",
+        "references": {
+          "table": "classes",
+          "column": "id"
+        }
+      }
+    ]
+  },
+  "classes": {
+    "comment": null,
+    "columns": [
+      {
+        "name": "id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "client_id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "start_date",
+        "data_type": "date",
+        "comment": null
+      },
+      {
+        "name": "end_date",
+        "data_type": "date",
+        "comment": null
+      },
+      {
+        "name": "created_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "updated_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "attendance_status",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "progression_status",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "subject",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "site",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "phase",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "marketer",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "status",
+        "data_type": "character varying",
+        "comment": null
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "client_id",
+        "references": {
+          "table": "clients",
+          "column": "id"
+        }
+      }
+    ]
+  },
+  "clients": {
+    "comment": null,
+    "columns": [
+      {
+        "name": "id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "created_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "updated_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "email",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "phone",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "address",
+        "data_type": "text",
+        "comment": null
+      },
+      {
+        "name": "status",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "name",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "contact_person",
+        "data_type": "character varying",
+        "comment": null
+      }
+    ],
+    "foreign_keys": []
+  },
+  "deliveries": {
+    "comment": null,
+    "columns": [
+      {
+        "name": "delivery_date",
+        "data_type": "date",
+        "comment": null
+      },
+      {
+        "name": "class_id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "created_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "updated_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "delivery_type",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "status",
+        "data_type": "character varying",
+        "comment": null
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "class_id",
+        "references": {
+          "table": "classes",
+          "column": "id"
+        }
+      }
+    ]
+  },
+  "events": {
+    "comment": null,
+    "columns": [
+      {
+        "name": "id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "created_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "updated_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "client_id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "class_id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "event_date",
+        "data_type": "date",
+        "comment": null
+      },
+      {
+        "name": "reminder_date",
+        "data_type": "date",
+        "comment": null
+      },
+      {
+        "name": "name",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "event_type",
+        "data_type": "character varying",
+        "comment": null
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "class_id",
+        "references": {
+          "table": "classes",
+          "column": "id"
+        }
+      },
+      {
+        "column": "client_id",
+        "references": {
+          "table": "clients",
+          "column": "id"
+        }
+      }
+    ]
+  },
+  "learners": {
+    "comment": null,
+    "columns": [
+      {
+        "name": "id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "class_id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "created_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "updated_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "id_number",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "name",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "gender",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "race",
+        "data_type": "character varying",
+        "comment": null
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "class_id",
+        "references": {
+          "table": "classes",
+          "column": "id"
+        }
+      }
+    ]
+  },
+  "progressions": {
+    "comment": null,
+    "columns": [
+      {
+        "name": "updated_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "class_id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "learner_id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "created_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "completion_date",
+        "data_type": "date",
+        "comment": null
+      },
+      {
+        "name": "progression_level",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "status",
+        "data_type": "character varying",
+        "comment": null
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "class_id",
+        "references": {
+          "table": "classes",
+          "column": "id"
+        }
+      },
+      {
+        "column": "learner_id",
+        "references": {
+          "table": "learners",
+          "column": "id"
+        }
+      }
+    ]
+  },
+  "reports": {
+    "comment": null,
+    "columns": [
+      {
+        "name": "related_client_id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "created_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "updated_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "related_class_id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "report_type",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "content",
+        "data_type": "text",
+        "comment": null
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "related_class_id",
+        "references": {
+          "table": "classes",
+          "column": "id"
+        }
+      },
+      {
+        "column": "related_client_id",
+        "references": {
+          "table": "clients",
+          "column": "id"
+        }
+      }
+    ]
+  },
+  "tasks": {
+    "comment": null,
+    "columns": [
+      {
+        "name": "id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "event_id",
+        "data_type": "integer",
+        "comment": null
+      },
+      {
+        "name": "due_date",
+        "data_type": "date",
+        "comment": null
+      },
+      {
+        "name": "created_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "updated_at",
+        "data_type": "timestamp without time zone",
+        "comment": null
+      },
+      {
+        "name": "description",
+        "data_type": "text",
+        "comment": null
+      },
+      {
+        "name": "assigned_to",
+        "data_type": "character varying",
+        "comment": null
+      },
+      {
+        "name": "status",
+        "data_type": "character varying",
+        "comment": null
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "event_id",
+        "references": {
+          "table": "events",
+          "column": "id"
+        }
+      }
+    ]
+  }
+}

schema.md

@@ -0,0 +1,195 @@
+## Request
+IMPORTANT ! Use the "Database Schema" below
+
+Here’s a list describing what I want you to asist me with:
+
+1. **Generate Demo Data**: Create demo data for a PostgreSQL database with at least 25 entries per table, considering the relationships between the tables.
+   
+2. **Focus on South African Context**: The demo data must reflect South African specifics, such as:
+   - South African names for agents, clients, and learners.
+   - South African locations (cities and towns) for agents, clients, and class sites.
+   - South African phone numbers (+27 format).
+   - Realistic South African ID numbers.
+
+3. **Tables to Include**: Populate demo data for the following 13 tables:
+   - **agents**
+   - **clients**
+   - **classes**
+   - **learners**
+   - **assessments**
+   - **progressions**
+   - **tasks**
+   - **deliveries**
+   - **events**
+   - **reports**
+   - **agent_assignments**
+   - **agent_availability**
+   - **agent_work_history**
+
+4. **SQL Format**: The data should be returned in SQL `INSERT INTO` statements with multiple rows, following a format like:
+   ```sql
+   INSERT INTO agents (name, email, phone, specialization, experience, current_workload, location, status, created_at, updated_at) VALUES
+   ('Thabo Mokoena', 'thabo.mokoena@example.co.za', '+27831234567', 'Math Tutor', 5, 3, 'Johannesburg', 'Active', NOW(), NOW()),
+   ('Naledi Khumalo', 'naledi.khumalo@example.co.za', '+27836543210', 'Science Instructor', 8, 2, 'Pretoria', 'Active', NOW(), NOW());
+   ```
+
+5. **Ensure Data Consistency**: The generated data should respect the foreign key relationships between the tables, ensuring that IDs referenced in one table (e.g., class_id) exist in the appropriate related table.
+
+6. **Provide the Data as a Downloadable SQL File**: Finally, package the SQL insertions for all 13 tables into a downloadable `.sql` file.
+
+
+
+## Database Schema:
+
+Table agent_availability (Comment: None):
+  - id (integer) (Comment: None)
+  - agent_id (integer) (Comment: None)
+  - available_date (date) (Comment: None)
+  - availability_status (character varying) (Comment: None)
+  - reason (text) (Comment: None)
+  - created_at (timestamp without time zone) (Comment: None)
+  - updated_at (timestamp without time zone) (Comment: None)
+Table agent_work_history (Comment: None):
+  - id (integer) (Comment: None)
+  - agent_id (integer) (Comment: None)
+  - class_id (integer) (Comment: None)
+  - task_id (integer) (Comment: None)
+  - start_date (date) (Comment: None)
+  - end_date (date) (Comment: None)
+  - role (character varying) (Comment: None)
+  - performance_notes (text) (Comment: None)
+  - reassignment_id (integer) (Comment: None)
+  - created_at (timestamp without time zone) (Comment: None)
+  - updated_at (timestamp without time zone) (Comment: None)
+  Foreign Keys:
+    - class_id references classes(id)
+    - task_id references tasks(id)
+Table agents (Comment: None):
+  - id (integer) (Comment: None)
+  - name (character varying) (Comment: None)
+  - email (character varying) (Comment: None)
+  - phone (character varying) (Comment: None)
+  - specialization (character varying) (Comment: None)
+  - experience (integer) (Comment: None)
+  - current_workload (integer) (Comment: None)
+  - location (character varying) (Comment: None)
+  - status (character varying) (Comment: None)
+  - created_at (timestamp without time zone) (Comment: None)
+  - updated_at (timestamp without time zone) (Comment: None)
+  - sms (character varying) (Comment: None)
+Table assessments (Comment: None):
+  - id (integer) (Comment: None)
+  - class_id (integer) (Comment: None)
+  - assessment_type (character varying) (Comment: None)
+  - assessment_date (date) (Comment: None)
+  - result (character varying) (Comment: None)
+  - assessor_name (character varying) (Comment: None)
+  - created_at (timestamp without time zone) (Comment: None)
+  - updated_at (timestamp without time zone) (Comment: None)
+  Foreign Keys:
+    - class_id references classes(id)
+Table classes (Comment: None):
+  - id (integer) (Comment: None)
+  - client_id (integer) (Comment: None)
+  - site (character varying) (Comment: None)
+  - start_date (date) (Comment: None)
+  - end_date (date) (Comment: None)
+  - status (character varying) (Comment: None)
+  - attendance_status (character varying) (Comment: None)
+  - progression_status (character varying) (Comment: None)
+  - subject (character varying) (Comment: None)
+  - phase (character varying) (Comment: None)
+  - marketer (character varying) (Comment: None)
+  - created_at (timestamp without time zone) (Comment: None)
+  - updated_at (timestamp without time zone) (Comment: None)
+Table clients (Comment: None):
+  - id (integer) (Comment: None)
+  - name (character varying) (Comment: None)
+  - contact_person (character varying) (Comment: None)
+  - email (character varying) (Comment: None)
+  - phone (character varying) (Comment: None)
+  - address (text) (Comment: None)
+  - status (character varying) (Comment: None)
+  - created_at (timestamp without time zone) (Comment: None)
+  - updated_at (timestamp without time zone) (Comment: None)
+Table deliveries (Comment: None):
+  - id (integer) (Comment: None)
+  - class_id (integer) (Comment: None)
+  - delivery_type (character varying) (Comment: None)
+  - delivery_date (date) (Comment: None)
+  - status (character varying) (Comment: None)
+  - created_at (timestamp without time zone) (Comment: None)
+  - updated_at (timestamp without time zone) (Comment: None)
+  Foreign Keys:
+    - class_id references classes(id)
+Table events (Comment: None):
+  - id (integer) (Comment: None)
+  - name (character varying) (Comment: None)
+  - client_id (integer) (Comment: None)
+  - class_id (integer) (Comment: None)
+  - event_type (character varying) (Comment: None)
+  - event_date (date) (Comment: None)
+  - reminder_date (date) (Comment: None)
+  - created_at (timestamp without time zone) (Comment: None)
+  - updated_at (timestamp without time zone) (Comment: None)
+  Foreign Keys:
+    - class_id references classes(id)
+Table learners (Comment: None):
+  - id (integer) (Comment: None)
+  - name (character varying) (Comment: None)
+  - gender (character varying) (Comment: None)
+  - race (character varying) (Comment: None)
+  - id_number (character varying) (Comment: None)
+  - class_id (integer) (Comment: None)
+  - created_at (timestamp without time zone) (Comment: None)
+  - updated_at (timestamp without time zone) (Comment: None)
+  Foreign Keys:
+    - class_id references classes(id)
+Table progressions (Comment: None):
+  - id (integer) (Comment: None)
+  - class_id (integer) (Comment: None)
+  - learner_id (integer) (Comment: None)
+  - progression_level (character varying) (Comment: None)
+  - completion_date (date) (Comment: None)
+  - status (character varying) (Comment: None)
+  - created_at (timestamp without time zone) (Comment: None)
+  - updated_at (timestamp without time zone) (Comment: None)
+  Foreign Keys:
+    - class_id references classes(id)
+    - learner_id references learners(id)
+Table reports (Comment: None):
+  - id (integer) (Comment: None)
+  - report_type (character varying) (Comment: None)
+  - related_class_id (integer) (Comment: None)
+  - related_client_id (integer) (Comment: None)
+  - content (text) (Comment: None)
+  - created_at (timestamp without time zone) (Comment: None)
+  - updated_at (timestamp without time zone) (Comment: None)
+  Foreign Keys:
+    - related_class_id references classes(id)
+Table tasks (Comment: None):
+  - id (integer) (Comment: None)
+  - event_id (integer) (Comment: None)
+  - description (text) (Comment: None)
+  - due_date (date) (Comment: None)
+  - status (character varying) (Comment: None)
+  - assigned_to (character varying) (Comment: None)
+  - created_at (timestamp without time zone) (Comment: None)
+  - updated_at (timestamp without time zone) (Comment: None)
+  Foreign Keys:
+    - event_id references events(id)
+Table agent_assignments (Comment: None):
+  - id (integer) (Comment: None)
+  - agent_id (integer) (Comment: None)
+  - class_id (integer) (Comment: None)
+  - task_id (integer) (Comment: None)
+  - assignment_date (date) (Comment: None)
+  - reassigned_agent_id (integer) (Comment: None)
+  - reassignment_date (date) (Comment: None)
+  - reassignment_reason (text) (Comment: None)
+  - status (character varying) (Comment: None)
+  - created_at (timestamp without time zone) (Comment: None)
+  - updated_at (timestamp without time zone) (Comment: None)
+  Foreign Keys:
+    - class_id references classes(id)
+    - task_id references tasks(id)