Upload 8 files

README.md

@@ -2,6 +2,5 @@
 title: ai_eee_sql_gen
 app_file: app.py
 sdk: gradio
-sdk_version: 4.44.0
-python_version: 3.11.9
----
+sdk_version: 4.43.0
+---

app.py

@@ -1,12 +1,21 @@
+import os
 import gradio as gr
 import logging
-
-from openai_integration import generate_sql
-from db import get_last_50_saved_queries, initialize_local_db, export_saved_queries_to_csv, execute_sql_query, fetch_and_save_schema, show_last_50_saved_queries  # Import the correct function
+import yaml
+from db import (
+    get_last_50_saved_queries,
+    initialize_local_db,
+    export_saved_queries_to_csv,
+    execute_sql_query,
+    fetch_and_save_schema,
+    show_last_50_saved_queries,
+    fetch_schema_info,  # Now this function exists in db.py
+)
+from openai_integration import generate_sql_single_call  # Import the updated function
 
 
 # Initialize logging
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname=s - %(message)s')
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
 # Call the function to ensure the table is created
 initialize_local_db()
@@ -16,16 +25,16 @@ def query_database(nl_query, progress=gr.Progress()):
     try:
         progress(0, desc="Starting Query Process")
 
-        # Generate SQL and reformulated query
-        progress(0.2, desc="Generating Reformulated Query")
-        reformulated_query, sql_query = generate_sql(nl_query)
+        # Generate SQL and reformulated query using the updated single call function
+        progress(0.5, desc="Generating Reformulated Query and SQL")
+        reformulated_query, sql_query, total_cost_per_call = generate_sql_single_call(nl_query)
 
         # Default empty result in case of SQL query failure
         execution_result = []
 
         # If we have a SQL query, attempt execution
         if sql_query and not sql_query.startswith("Error"):
-            progress(0.5, desc="Executing SQL Query")
+            progress(0.8, desc="Executing SQL Query")
             execution_result = execute_sql_query(sql_query)
             
             # Ensure execution_result is in a valid format for a DataFrame
@@ -35,11 +44,11 @@ def query_database(nl_query, progress=gr.Progress()):
             execution_result = [["No results available."]]
 
         progress(1, desc="Query Completed")
-        return reformulated_query, sql_query, execution_result
+        return reformulated_query, sql_query, execution_result, total_cost_per_call
 
     except Exception as e:
         logging.error(f"Error during query generation or execution: {e}")
-        return "Error during query processing.", "", [["No results available due to an error."]]
+        return "Error during query processing.", "", [["No results available due to an error."], ""]
 
 # Function to update the schema when requested
 def update_schema():
@@ -54,13 +63,13 @@ def update_schema():
         raise gr.Error("No schema data was returned. The schema is empty.", duration=3)
     
     # Case 3: Schema successfully fetched
-    return "Query executed successfully", gr.Info("DB Schema Updated ℹ️", duration=3)
-
+    return "Schema updated successfully", gr.Info("DB Schema Updated ℹ️", duration=3)
 
 # Function to make hidden components visible after the process
 def continue_process():
-    # Ensure both the SQL and result outputs are shown (since reformulated_output is always visible now)
-    return gr.update(visible=True), gr.update(visible=True)
+    # Ensure all three outputs (SQL, result, and cost) are shown
+    return gr.update(visible=True), gr.update(visible=True), gr.update(visible=True)
+
 
 # Function to reset the interface to its initial state
 def reset_interface():
@@ -73,21 +82,46 @@ def update_button_state(text):
     else:
         return gr.update(interactive=False)
 
+# Function to fetch table names from schema and format them for display
+def get_table_names():
+    schema_info = fetch_schema_info()
+    if not schema_info:
+        return []
+    
+    # Return both the original table name and the formatted name
+    return [
+        (table_name, ' '.join(word.capitalize() for word in table_name.split('_')))
+        for table_name in schema_info.keys()
+    ]
 
-# Assuming 'table_names' is a list of table names fetched from the schema, e.g., ['Reports', 'Tasks', 'Clients']
+# Fetch table names as a list of tuples (original_name, formatted_name)
+table_names = get_table_names()
 
-# Function to fetch table names from schema (you may already have this in your app)
-def get_table_names():
-    # Replace this with actual schema fetching logic if needed
-    return ["Reports", "Tasks", "Clients", "Agents", "Learners", "Classes", "Events", "Assesments", "Progressions", "Deliveries", "Agent Assignments", "Agent Availability", "Agent Work History"]
 
 # Function to update the query textbox when a button is clicked
 def insert_table_name(current_text, table_name):
     # Add the table name to the current text
     return current_text + " " + table_name
 
-# Fetch table names from schema
-table_names = get_table_names()
+
+
+# Function to load examples from YAML file
+def load_examples_from_yaml(file_path):
+    try:
+        with open(file_path, 'r') as file:
+            examples = yaml.safe_load(file)
+        return examples
+    except Exception as e:
+        logging.error(f"Error loading examples: {e}")
+        return []
+
+# Load examples from YAML
+EXAMPLES_FILE_PATH = os.path.join(os.path.dirname(__file__), 'examples.yaml')
+examples_list = load_examples_from_yaml(EXAMPLES_FILE_PATH)
+# Extract the inputs for Gradio examples
+example_inputs = [example['input'] for example in examples_list]
+# Create numbered labels for each example (1., 2., 3., etc.)
+example_labels = [f"{i+1}" for i in range(len(example_inputs))]
 
 
 # Gradio interface setup
@@ -100,18 +134,23 @@ with gr.Blocks(theme=gr.themes.Soft(font=[gr.themes.GoogleFont("Ubuntu"), "Arial
             """)
     # Dynamically create buttons for each table
     with gr.Row():
-        for table in table_names:
-            gr.Button(table, size="small", elem_classes="ydcoza-small-button").click(fn=lambda current_text, t=table: insert_table_name(current_text, t), inputs=text_input, outputs=text_input)
-
-    examples = gr.Examples(examples=[
-        "I'm trying to figure out which agents are the busiest. Can you show me like the top few agents who have a lot on their plate? I'd like to see their names and maybe their contact info if we have it.", 
-        "I need to get an overview of our classes. Could you pull up a list that shows what each class is about and which client it's for? Oh, and it would be great to know how many students are in each class. Maybe order it so the biggest classes are at the top?",
-        # "Can you give me an overview of all our classes? I'd like to see how many students are in each class and how diverse they are in terms of race. It would be helpful to see the class subject and location too. Don't leave out any classes, even if they have few or no students.",
-        "List all classes scheduled for client EduTrust.",
-        "I'm curious about how our classes are doing. Can you show me a list of all the classes and their latest test results or evaluations? It would be helpful to see where each class is located too.",
-        #"I heard some agents got moved around recently. Can you find out which agents have been switched to different classes? I'd like to know where they were before, where they are now, and maybe why they were moved."
-        "I'd like to review our agents' performance. Could you provide a summary of each agent's work history, including the classes they've taught, their roles, performance notes, and any reassignments? Highlight any instances where agents were reassigned and the reasons for the reassignment."
-        ],example_labels=["01","02","03","04","05"], label="Demo Natural Language Queries",inputs=[text_input])
+        # Create Gradio buttons with formatted label and insert original table name on click
+        for original_name, formatted_name in table_names:
+            gr.Button(formatted_name, size="small", elem_classes="ydcoza-small-button").click(
+                fn=lambda current_text, t=original_name: insert_table_name(current_text, t),
+                inputs=text_input,
+                outputs=text_input
+            )
+
+    # Create Gradio Examples component
+    examples = gr.Examples(
+        examples=example_inputs,   # The actual inputs from the YAML file
+        example_labels=example_labels,  # Numbered labels for buttons
+        label="Demo Natural Language Queries",
+        inputs=[text_input]
+    )
+
+
     reformulated_output = gr.Textbox(lines=2, label="Optimised Query", elem_id='ydcoza_markdown_output_desc')
     sql_output = gr.Code(label="Generated SQL", visible=False)
     sql_result_output = gr.Dataframe(label="Query Results", elem_id='result_output', visible=False)  # Dataframe for SQL results
@@ -127,7 +166,12 @@ with gr.Blocks(theme=gr.themes.Soft(font=[gr.themes.GoogleFont("Ubuntu"), "Arial
     gr.HTML("""
             <span class="ydcoza_gradio_banner">View The last 50 Queries generated in Table format.</span>
             """)
-    saved_queries_output = gr.Dataframe(label="Last 50 Saved Queries", headers=["Query", "Optimised Query", "SQL", "Timestamp"], interactive=True, visible=False)
+    saved_queries_output = gr.Dataframe(
+        label="Last 50 Saved Queries",
+        headers=["Query", "Optimised Query", "SQL", "Timestamp"],
+        interactive=True,
+        visible=False
+    )
     # Show the last 50 saved queries when button is clicked
     show_saved_queries_button = gr.Button("View Queries", elem_id='ydcoza_gradio_button')
     show_saved_queries_button.click(show_last_50_saved_queries, outputs=saved_queries_output).then(
@@ -146,25 +190,28 @@ with gr.Blocks(theme=gr.themes.Soft(font=[gr.themes.GoogleFont("Ubuntu"), "Arial
             """)
     # Add a button to pull the latest schema and save it to schema.json
     fetch_schema_button = gr.Button("Fetch Latest Schema", elem_id='ydcoza_gradio_button')
-    fetch_schema_button.click(update_schema, outputs=[gr.Textbox(label="Schema Update Status")])
     fetch_schema_button.click(update_schema)
 
+    # Output for the cost information (initially hidden)
+    with gr.Row():
+        html_output_cost = gr.HTML(elem_id='ydcoza_cost_output', visible=False)
+
+
     # Setup the button click to trigger the process and show results
     text_input.change(fn=update_button_state, inputs=text_input, outputs=start_button)
+
     start_button.click(
         fn=query_database, 
         inputs=[text_input], 
-        outputs=[reformulated_output, sql_output, sql_result_output]
+        outputs=[reformulated_output, sql_output, sql_result_output, html_output_cost]  # Include the cost output here
     ).then(
         continue_process, 
-        outputs=[sql_output, sql_result_output]
+        outputs=[sql_output, sql_result_output, html_output_cost]  # Ensure cost is also shown
     ).then(
-        lambda: gr.update(interactive=False), outputs=start_button  # Disable the submit button after submission
+        lambda: gr.update(interactive=False), outputs=start_button
     )
 
+
 # Launch the Gradio interface
 if __name__ == "__main__":
-    # ydcoza_face.launch(auth=auth_users, auth_message="Demo Login, Username: admin & Password: 1234")
-    # run gradio deploy in Terminal
     ydcoza_face.launch()
-    

db.py

@@ -242,4 +242,14 @@ def reset_sqlite_db():
     conn.close()
 
 # Uncomment the following line to reset the SQLite database when you run this script
-# reset_sqlite_db()
+# reset_sqlite_db()
+
+def fetch_schema_info():
+    try:
+        with open("schema.json", "r") as schema_file:
+            schema_info = json.load(schema_file)
+        logging.info("Schema loaded from schema.json")
+        return schema_info
+    except Exception as e:
+        logging.error(f"Error loading schema from schema.json: {e}")
+        return {}

examples.yaml

@@ -0,0 +1,74 @@
+- input: "I'm trying to figure out which agents are the busiest. Can you show me like the top few agents who have a lot on their plate?"
+  reformulated_query: "Show me the top 5 agents with the highest workload, including their names, emails, and specializations. Use Tables: agents"
+  sql_query: |
+    SELECT name, email, specialization, current_workload
+    FROM agents
+    ORDER BY current_workload DESC
+    LIMIT 5;
+
+- input: "I need to get an overview of our classes. Could you pull up a list that shows what each class is about and which client it's for?"
+  reformulated_query: "List all classes with their subjects, client names, and the number of learners in each class, sorted by the number of learners in descending order. Use Tables: clients, learners, classes"
+  sql_query: |
+    SELECT c.id AS class_id, c.subject, cl.name AS client_name, COUNT(l.id) AS learner_count
+    FROM classes c
+    JOIN clients cl ON c.client_id = cl.id
+    LEFT JOIN learners l ON c.id = l.class_id
+    GROUP BY c.id, c.subject, cl.name
+    ORDER BY learner_count DESC;
+
+- input: "Can you show me the list of agents who are available next week?"
+  reformulated_query: "List all agents with their names and contact information who are available next week. Use Tables: agents, agent_availability"
+  sql_query: |
+    SELECT a.name, a.email, a.phone
+    FROM agents a
+    JOIN agent_availability aa ON a.id = aa.agent_id
+    WHERE aa.available_date BETWEEN CURRENT_DATE + INTERVAL '7 days' AND CURRENT_DATE + INTERVAL '14 days'
+      AND aa.availability_status = 'available';
+
+- input: "Which clients have classes starting next month and how many learners are enrolled in each class?"
+  reformulated_query: "List all clients with classes starting next month, including the client name, class subject, start date, and the number of learners enrolled in each class. Use Tables: clients, classes, learners"
+  sql_query: |
+    SELECT cl.name AS client_name, c.subject AS class_subject, c.start_date, COUNT(l.id) AS learner_count
+    FROM clients cl
+      JOIN classes c ON cl.id = c.client_id
+      LEFT JOIN learners l ON c.id = l.class_id
+    WHERE c.start_date >= date_trunc('month', CURRENT_DATE) + INTERVAL '1 month'
+      AND c.start_date < date_trunc('month', CURRENT_DATE) + INTERVAL '2 months'
+    GROUP BY cl.name, c.subject, c.start_date
+    ORDER BY cl.name, c.start_date;
+
+- input: "Show me all the tasks that are overdue and assigned to 'John Doe'."
+  reformulated_query: "List all tasks assigned to 'John Doe' that have a due date before today and are not marked as 'completed', including the task description and due date. Use Tables: tasks"
+  sql_query: |
+    SELECT description, due_date
+    FROM tasks
+    WHERE assigned_to = 'John Doe'
+      AND due_date < CURRENT_DATE
+      AND status != 'completed';
+
+- input: "I want to see the progression levels of learners in the history class."
+  reformulated_query: "List all learners in the history class, including their names and progression levels. Use Tables: learners, progressions, classes"
+  sql_query: |
+    SELECT l.name AS learner_name, p.progression_level
+    FROM classes c
+    JOIN learners l ON c.id = l.class_id
+    JOIN progressions p ON l.id = p.learner_id
+    WHERE c.subject = 'Advanced Physics';
+
+- input: "Can you show me a list of agents who have been reassigned more than once, along with the classes they were reassigned to and the reasons?"
+  reformulated_query: "List all agents who have been reassigned more than once, including their names, the classes they were reassigned to, assignment dates, and the reasons for reassignment. Use Tables: agents, agent_assignments, classes"
+  sql_query: |
+    SELECT a.name AS agent_name, c.subject AS class_subject, aa.assignment_date, aa.reassignment_reason
+    FROM agents a
+    JOIN agent_assignments aa ON a.id = aa.agent_id
+    JOIN classes c ON aa.class_id = c.id
+    WHERE aa.reassigned_agent_id IS NOT NULL
+        AND a.id IN (
+            SELECT agent_id
+            FROM agent_assignments
+            WHERE reassigned_agent_id IS NOT NULL
+            GROUP BY agent_id
+            HAVING COUNT(*) > 1
+        )
+    ORDER BY a.name, aa.assignment_date;
+    

openai_integration.py

@@ -1,20 +1,31 @@
 import os
 import openai
 import json
+import yaml
 from dotenv import load_dotenv
 from db_logging import save_query_to_local_db
 import logging
-
-GPT_MODEL = "gpt-4o-mini"
-
-# Load environment variables from .env file
+#############LangSmith & OpenAI####################
+from langsmith import traceable
+from langsmith.wrappers import wrap_openai
+# Initialize OpenAI client
+client = wrap_openai(openai.Client())
+# Load environment variables
 load_dotenv()
+# Load the OpenAI API key
+openai.api_key = os.getenv("OPENAI_API_KEY")
+if not openai.api_key:
+    raise ValueError("Error: OPENAI_API_KEY not found in environment variables")
+GPT_MODEL = "gpt-4o-mini"
+##################################################
 
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
-# Initialize the OpenAI client using API key from .env
-client = openai.OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+# Function to calculate cost based on token usage
+def ydcoza_cost(tokens_out):
+    t_cost = tokens_out * 0.2 / 1000000  # Rough Estimate For gpt-4o $5.00 / 1M input tokens :  mini: $0.150 / 1M input tokens
+    return t_cost
 
 # Function to load schema from schema.json
 def load_schema_from_json():
@@ -41,149 +52,120 @@ def build_schema_description(schema_info):
                 schema_description += f"    - {fk['column']} references {fk['references']['table']}({fk['references']['column']})\n"
     return schema_description
 
-# Function to reformulate the natural language query using OpenAI API
-def reformulate_query(nl_query, schema_description):
-    try:
-        # Adding examples to guide the reformulation process
-        examples = """
-        Example 1:
-        Input: "I'm trying to figure out which agents are the busiest. Can you show me like the top few agents who have a lot on their plate?"
-        Reformulated: "Show me the top 5 agents with the highest workload, including their names, emails, and specializations. Use Tables: agents"
-
-        Example 2:
-        Input: "I need to get an overview of our classes. Could you pull up a list that shows what each class is about and which client it's for?"
-        Reformulated: "List all classes with their subjects, client names, and the number of learners in each class, sorted by the number of learners in descending order. Use Tables: clients, learners, classes"
-        """
-
-        prompt = (
-            f"Database Schema:\n{schema_description}\n\n"
-            f"Using the database schema, reformulate the following natural language query to be more precise and in line with the database schema:\n{nl_query}\n\n"
-            f"Here are some examples of how to reformulate the natural language query:\n{examples}\n\n"
-            f"Output the query string, list the DB tables that will be used seperated by commas, don't prefix with anything and don't use markdown or quotation marks."
-        )
-
-        logging.info("Sending reformulation request to OpenAI...")
-        response = client.chat.completions.create(
-            model=GPT_MODEL,
-            messages=[
-                {"role": "system", "content": "You are an assistant that helps reformulate natural language queries and improve on them."},
-                {"role": "user", "content": prompt}
-            ],
-            max_tokens=300,
-            temperature=0.7
-        )
 
-        reformulated_query = response.choices[0].message.content.strip()
-
-        logging.info(f"Reformulated Query: {reformulated_query}")
-        return reformulated_query
+# Function to load examples from YAML
+def load_examples_from_yaml(file_path):
+    try:
+        with open(file_path, 'r') as file:
+            examples = yaml.safe_load(file)
+        logging.info("Examples loaded from examples.yaml")
+        return examples
     except Exception as e:
-        logging.error(f"Error reformulating query: {e}")
-        return str(e)
-
-# Function to generate SQL from reformulated query using OpenAI API
-def generate_sql_from_reformulated(reformulated_query, schema_description):
+        logging.error(f"Error loading examples: {e}")
+        return []
+
+# Function to build examples string for the prompt
+def build_examples_string(examples_list):
+    examples_string = ""
+    for idx, example in enumerate(examples_list, start=1):
+        examples_string += f"Example {idx}:\n"
+        examples_string += f"Input:\n{example['input']}\n"
+        examples_string += f"Reformulated Query:\n{example['reformulated_query']}\n"
+        examples_string += f"SQL Query:\n{example['sql_query']}\n\n"
+    return examples_string
+
+# Load examples from YAML
+EXAMPLES_FILE_PATH = os.path.join(os.path.dirname(__file__), 'examples.yaml')
+examples_list = load_examples_from_yaml(EXAMPLES_FILE_PATH)
+examples = build_examples_string(examples_list)
+# logging.info(f"Examples From YAML: {examples}")
+
+def generate_sql_single_call(nl_query):
     try:
-        # Adding examples to guide the SQL generation process
-        examples = """
-        Example 1:
-        Input: 
-        "Show me the top 5 agents with the highest workload, including their names, emails, and specializations."
-        SQL: 
-        SELECT name, email, specialization, current_workload
-        FROM agents
-        ORDER BY current_workload DESC
-        LIMIT 5;
-
-        Example 2:
-        Input: 
-        "List all classes with their subjects, client names, and the number of learners in each class, sorted by the number of learners in descending order."
-        SQL: 
-        SELECT c.id AS class_id, c.subject, cl.name AS client_name, COUNT(l.id) AS learner_count
-        FROM classes c
-        JOIN clients cl ON c.client_id = cl.id
-        LEFT JOIN learners l ON c.id = l.class_id
-        GROUP BY c.id, c.subject, cl.name
-        ORDER BY learner_count DESC;
-
-        Example 3:
-        Input: 
-        "List all agents' work history, including the class they worked on, their role, and the start and end dates of each task."
-        SQL: 
-        SELECT a.name AS agent_name, awh.class_id, awh.role, awh.start_date, awh.end_date
-        FROM agents a
-        JOIN agent_work_history awh ON a.id = awh.agent_id
-        ORDER BY awh.start_date DESC;
-
-        Example 4:
-        Input: 
-        "Get a list of all classes with their subjects, the number of learners, and the progression status of each learner in the class."
-        SQL: 
-        SELECT c.id AS class_id, c.subject, COUNT(l.id) AS learner_count, p.progression_level
-        FROM classes c
-        LEFT JOIN learners l ON c.id = l.class_id
-        LEFT JOIN progressions p ON l.id = p.learner_id
-        GROUP BY c.id, c.subject, p.progression_level
-        ORDER BY learner_count DESC;      
-        """
+        # Load the schema from schema.json
+        schema_info = load_schema_from_json()
+
+        if isinstance(schema_info, str) and schema_info.startswith("Error"):
+            return "Error fetching schema", ""
 
+        # Build the schema description once and reuse it
+        schema_description = build_schema_description(schema_info)
+        logging.info(f"Schema Description: {schema_description}")
+
+        # Use the final prompt as shown above
         prompt = (
             f"Database Schema:\n{schema_description}\n\n"
-            f"Natural language query:\n{reformulated_query}\n\n"
-            f"Examples:\n{examples}\n\n"
-            f"Convert the natural language query into an SQL query that matches the schema.\n\n"
-            f"Use the Examples as a guide on what I expect you to do.\n\n"
-            f"Only return the RAW SQL that can be directly executed, don't prefix it with sql, don't use quotation marks and don't use markdown."
+            f"Your task is to:\n"
+            f"1. Reformulate the user's natural language query to align precisely with the database schema. Also, indicate the tables to use. These tables **MUST** be present in the schema provided.\n"
+            f"2. Generate the corresponding SQL query based on the reformulated query and the provided schema.\n\n"
+            f"User's Query:\n\"{nl_query}\"\n\n"
+            f"Examples:\n{examples}\n"
+            f"Response Format (in JSON):\n"
+            f"{{\n"
+            f'  "reformulated_query": "<your reformulated query>",\n'
+            f'  "sql_query": "<your SQL query>"\n'
+            f"}}\n\n"
+            f"Important Guidelines:\n"
+            f"- Use only the tables and columns provided in the schema.\n"
+            f"- Ensure the SQL query is syntactically correct.\n"
+            f"- Do not include any additional text or explanations.\n"
+            f"- Do not include any text outside the JSON format.\n"
+            f"- Ensure the JSON is valid and properly formatted.\n"
+            f"- Avoid assumptions about data not present in the schema.\n"
+            f"- Double-check your response for accuracy."
         )
-
-        logging.info("Sending SQL generation request to OpenAI...")
+        logging.info(f"Full Prompt: {prompt}")
+        logging.info("Sending combined request to OpenAI...")
         response = client.chat.completions.create(
             model=GPT_MODEL,
             messages=[
-                {"role": "system", "content": "You are an assistant that converts natural language to SQL using a provided databse shema."},
+                {
+                    "role": "system",
+                    "content": (
+                        "You are an expert data analyst and SQL specialist. "
+                        "Your role is to reformulate natural language queries to align with a given database schema "
+                        # "and then generate accurate SQL queries based on the reformulated queries."
+                        "and then create a syntactically correct PostgreSQL query to run based on the reformulated queries."
+                        "Ensure that you only use tables and columns present in the provided schema. "
+                        "Indicate the tables to use in your reformulated query."
+                        "Ensure that your responses are precise, concise, and follow the provided guidelines."
+                        "Provide your response in the specified JSON format."
+                    )
+                },
                 {"role": "user", "content": prompt}
             ],
-            max_tokens=300,
-            temperature=0.7
+            max_tokens=500,
+            temperature=0.3  # Adjusted temperature
         )
 
-        sql_query = response.choices[0].message.content.strip()
-
-        logging.info(f"SQL Query generated: {sql_query}")
-        return sql_query
-    except Exception as e:
-        logging.error(f"Error generating SQL: {e}")
-        return str(e)
-
-# Main function to reformulate the query first and then generate SQL
-def generate_sql(nl_query):
-    try:
-        # Load the schema from schema.json
-        schema_info = load_schema_from_json()
+        # Process the assistant's response
+        assistant_response = response.choices[0].message.content.strip()
+        logging.info(f"Assistant Response:\n{assistant_response}")
 
-        if isinstance(schema_info, str) and schema_info.startswith("Error"):
-            return "Error fetching schema", ""
 
-        # Build the schema description once and reuse it
-        schema_description = build_schema_description(schema_info)
-        logging.error(f"Build Shema Description: {schema_description}")
 
-        # Reformulate the query
-        reformulated_query = reformulate_query(nl_query, schema_description)
+        # Calculate Tokens
+        tokens_used = response.usage.total_tokens
+        cost_per_call = ydcoza_cost(tokens_used)
+        total_cost_per_call = f"<p class='cost_per_call'>Tokens Consumed: {tokens_used} ; Cost per call: ${cost_per_call:.6f}</p>"
 
-        if "Error" in reformulated_query:
-            return reformulated_query, ""
 
-        # Generate SQL based on the reformulated query
-        sql_query = generate_sql_from_reformulated(reformulated_query, schema_description)
+        # Parse the assistant's response
+        try:
+            response_json = json.loads(assistant_response)
+            reformulated_query = response_json.get("reformulated_query", "")
+            sql_query = response_json.get("sql_query", "")
+        except json.JSONDecodeError:
+            logging.error("Could not parse assistant response as JSON.")
+            return "Error parsing assistant response.", ""
 
-        if "Error" in sql_query:
-            return reformulated_query, sql_query
+        logging.info(f"Reformulated Query: {reformulated_query}")
+        logging.info(f"SQL Query generated: {sql_query}")
 
         # Save the query to the local database
         save_query_to_local_db(nl_query, reformulated_query, sql_query)
 
-        return reformulated_query, sql_query
+        return reformulated_query, sql_query, total_cost_per_call
 
     except openai.error.OpenAIError as e:
         logging.error(f"OpenAI API error: {e}")
@@ -192,3 +174,4 @@ def generate_sql(nl_query):
     except Exception as e:
         logging.error(f"General error during SQL process: {e}")
         return "General error during SQL processing.", ""
+

schema.json

@@ -1,675 +1,1878 @@
 {
-  "agent_assignments": {
-    "comment": null,
+  "agent_absences": {
+    "comment": "Records instances when agents are absent from classes",
+    "columns": [
+      {
+        "name": "absence_id",
+        "data_type": "integer",
+        "comment": "Unique internal absence ID"
+      },
+      {
+        "name": "agent_id",
+        "data_type": "integer",
+        "comment": "Reference to the absent agent"
+      },
+      {
+        "name": "class_id",
+        "data_type": "integer",
+        "comment": "Reference to the class affected by the absence"
+      },
+      {
+        "name": "absence_date",
+        "data_type": "date",
+        "comment": "Date of the agent's absence"
+      },
+      {
+        "name": "reported_at",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the absence was reported"
+      },
+      {
+        "name": "reason",
+        "data_type": "text",
+        "comment": "Reason for the agent's absence"
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "agent_id",
+        "references": {
+          "table": "agents",
+          "column": "agent_id"
+        }
+      },
+      {
+        "column": "class_id",
+        "references": {
+          "table": "classes",
+          "column": "class_id"
+        }
+      }
+    ]
+  },
+  "agent_notes": {
+    "comment": "Stores historical notes and remarks about agents",
+    "columns": [
+      {
+        "name": "note_id",
+        "data_type": "integer",
+        "comment": "Unique internal note ID"
+      },
+      {
+        "name": "agent_id",
+        "data_type": "integer",
+        "comment": "Reference to the agent"
+      },
+      {
+        "name": "note_date",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the note was created"
+      },
+      {
+        "name": "note",
+        "data_type": "text",
+        "comment": "Content of the note regarding the agent"
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "agent_id",
+        "references": {
+          "table": "agents",
+          "column": "agent_id"
+        }
+      }
+    ]
+  },
+  "agent_orders": {
+    "comment": "Stores order information related to agents and classes",
+    "columns": [
+      {
+        "name": "updated_at",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the order record was last updated"
+      },
+      {
+        "name": "agent_id",
+        "data_type": "integer",
+        "comment": "Reference to the agent"
+      },
+      {
+        "name": "class_id",
+        "data_type": "integer",
+        "comment": "Reference to the class"
+      },
+      {
+        "name": "created_at",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the order record was created"
+      },
+      {
+        "name": "order_id",
+        "data_type": "integer",
+        "comment": "Unique internal order ID"
+      },
+      {
+        "name": "class_time",
+        "data_type": "time without time zone",
+        "comment": "Time when the class is scheduled"
+      },
+      {
+        "name": "order_hours",
+        "data_type": "integer",
+        "comment": "Number of hours linked to the agent's order for a specific class"
+      },
+      {
+        "name": "order_number",
+        "data_type": "character varying",
+        "comment": "Valid order number associated with the agent"
+      },
+      {
+        "name": "class_days",
+        "data_type": "character varying",
+        "comment": "Days when the class is scheduled"
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "agent_id",
+        "references": {
+          "table": "agents",
+          "column": "agent_id"
+        }
+      },
+      {
+        "column": "class_id",
+        "references": {
+          "table": "classes",
+          "column": "class_id"
+        }
+      }
+    ]
+  },
+  "agent_products": {
+    "comment": "Associates agents with the products they are trained to teach",
+    "columns": [
+      {
+        "name": "agent_id",
+        "data_type": "integer",
+        "comment": "Reference to the agent"
+      },
+      {
+        "name": "product_id",
+        "data_type": "integer",
+        "comment": "Reference to the product the agent is trained in"
+      },
+      {
+        "name": "trained_start_date",
+        "data_type": "date",
+        "comment": "Start date when the agent began training in the product"
+      },
+      {
+        "name": "trained_end_date",
+        "data_type": "date",
+        "comment": "End date when the agent finished training in the product"
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "agent_id",
+        "references": {
+          "table": "agents",
+          "column": "agent_id"
+        }
+      },
+      {
+        "column": "product_id",
+        "references": {
+          "table": "products",
+          "column": "product_id"
+        }
+      }
+    ]
+  },
+  "agent_qa_visits": {
+    "comment": "Records QA visits involving agents and classes",
+    "columns": [
+      {
+        "name": "visit_id",
+        "data_type": "integer",
+        "comment": "Unique internal QA visit ID"
+      },
+      {
+        "name": "agent_id",
+        "data_type": "integer",
+        "comment": "Reference to the agent"
+      },
+      {
+        "name": "class_id",
+        "data_type": "integer",
+        "comment": "Reference to the class"
+      },
+      {
+        "name": "visit_date",
+        "data_type": "date",
+        "comment": "Date of the QA visit"
+      },
+      {
+        "name": "qa_report_id",
+        "data_type": "integer",
+        "comment": "Reference to the associated QA report"
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "agent_id",
+        "references": {
+          "table": "agents",
+          "column": "agent_id"
+        }
+      },
+      {
+        "column": "class_id",
+        "references": {
+          "table": "classes",
+          "column": "class_id"
+        }
+      },
+      {
+        "column": "qa_report_id",
+        "references": {
+          "table": "qa_reports",
+          "column": "qa_report_id"
+        }
+      }
+    ]
+  },
+  "agent_replacements": {
+    "comment": "Records instances of agent replacements in classes",
+    "columns": [
+      {
+        "name": "replacement_id",
+        "data_type": "integer",
+        "comment": "Unique internal replacement ID"
+      },
+      {
+        "name": "class_id",
+        "data_type": "integer",
+        "comment": "Reference to the class"
+      },
+      {
+        "name": "original_agent_id",
+        "data_type": "integer",
+        "comment": "Reference to the original agent"
+      },
+      {
+        "name": "replacement_agent_id",
+        "data_type": "integer",
+        "comment": "Reference to the replacement agent"
+      },
+      {
+        "name": "start_date",
+        "data_type": "date",
+        "comment": "Date when the replacement starts"
+      },
+      {
+        "name": "end_date",
+        "data_type": "date",
+        "comment": "Date when the replacement ends"
+      },
+      {
+        "name": "reason",
+        "data_type": "text",
+        "comment": "Reason for the agent's replacement"
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "class_id",
+        "references": {
+          "table": "classes",
+          "column": "class_id"
+        }
+      },
+      {
+        "column": "original_agent_id",
+        "references": {
+          "table": "agents",
+          "column": "agent_id"
+        }
+      },
+      {
+        "column": "replacement_agent_id",
+        "references": {
+          "table": "agents",
+          "column": "agent_id"
+        }
+      }
+    ]
+  },
+  "agents": {
+    "comment": "Stores information about agents (instructors or facilitators)",
+    "columns": [
+      {
+        "name": "agent_id",
+        "data_type": "integer",
+        "comment": "Unique internal agent ID"
+      },
+      {
+        "name": "residential_town_id",
+        "data_type": "integer",
+        "comment": "Reference to the town where the agent lives"
+      },
+      {
+        "name": "preferred_working_area_1",
+        "data_type": "integer",
+        "comment": "Agent's first preferred working area"
+      },
+      {
+        "name": "preferred_working_area_2",
+        "data_type": "integer",
+        "comment": "Agent's second preferred working area"
+      },
+      {
+        "name": "preferred_working_area_3",
+        "data_type": "integer",
+        "comment": "Agent's third preferred working area"
+      },
+      {
+        "name": "sace_registration_date",
+        "data_type": "date",
+        "comment": "Date when the agent's SACE registration became effective"
+      },
+      {
+        "name": "sace_expiry_date",
+        "data_type": "date",
+        "comment": "Expiry date of the agent's provisional SACE registration"
+      },
+      {
+        "name": "date_loaded",
+        "data_type": "date",
+        "comment": "Date when the agent was added to the system"
+      },
+      {
+        "name": "quantum_result_communications",
+        "data_type": "numeric",
+        "comment": "Agent's competence score in Communications (percentage)"
+      },
+      {
+        "name": "quantum_result_mathematics",
+        "data_type": "numeric",
+        "comment": "Agent's competence score in Mathematics (percentage)"
+      },
+      {
+        "name": "quantum_result_training",
+        "data_type": "numeric",
+        "comment": "Agent's competence score in Training/Facilitating (percentage)"
+      },
+      {
+        "name": "agent_training_date",
+        "data_type": "date",
+        "comment": "Date when the agent received induction training"
+      },
+      {
+        "name": "signed_agreement",
+        "data_type": "boolean",
+        "comment": "Indicates if the agent has a signed agreement (true) or not (false)"
+      },
+      {
+        "name": "signed_agreement_date",
+        "data_type": "date",
+        "comment": "Date when the agent signed the agreement"
+      },
+      {
+        "name": "created_at",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the agent record was created"
+      },
+      {
+        "name": "updated_at",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the agent record was last updated"
+      },
+      {
+        "name": "bank_name",
+        "data_type": "character varying",
+        "comment": "Name of the agent's bank"
+      },
+      {
+        "name": "highest_qualification",
+        "data_type": "character varying",
+        "comment": "Highest qualification the agent has achieved"
+      },
+      {
+        "name": "sace_registration_number",
+        "data_type": "character varying",
+        "comment": "Agent's SACE (South African Council for Educators) registration number"
+      },
+      {
+        "name": "first_name",
+        "data_type": "character varying",
+        "comment": "Agent's first name"
+      },
+      {
+        "name": "initials",
+        "data_type": "character varying",
+        "comment": "Agent's initials"
+      },
+      {
+        "name": "surname",
+        "data_type": "character varying",
+        "comment": "Agent's surname"
+      },
+      {
+        "name": "gender",
+        "data_type": "character varying",
+        "comment": "Agent's gender"
+      },
+      {
+        "name": "race",
+        "data_type": "character varying",
+        "comment": "Agent's race; options include 'African', 'Coloured', 'White', 'Indian'"
+      },
+      {
+        "name": "sa_id_no",
+        "data_type": "character varying",
+        "comment": "Agent's South African ID number"
+      },
+      {
+        "name": "passport_number",
+        "data_type": "character varying",
+        "comment": "Agent's passport number if they are a foreigner"
+      },
+      {
+        "name": "tel_number",
+        "data_type": "character varying",
+        "comment": "Agent's primary telephone number"
+      },
+      {
+        "name": "email_address",
+        "data_type": "character varying",
+        "comment": "Agent's email address"
+      },
+      {
+        "name": "residential_address_line",
+        "data_type": "character varying",
+        "comment": "Agent's residential street address"
+      },
+      {
+        "name": "residential_suburb",
+        "data_type": "character varying",
+        "comment": "Agent's residential suburb"
+      },
+      {
+        "name": "bank_branch_code",
+        "data_type": "character varying",
+        "comment": "Branch code of the agent's bank"
+      },
+      {
+        "name": "residential_postal_code",
+        "data_type": "character varying",
+        "comment": "Postal code of the agent's residential area"
+      },
+      {
+        "name": "bank_account_number",
+        "data_type": "character varying",
+        "comment": "Agent's bank account number"
+      },
+      {
+        "name": "agent_notes",
+        "data_type": "text",
+        "comment": "Notes regarding the agent's performance, issues, or other relevant information"
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "preferred_working_area_1",
+        "references": {
+          "table": "locations",
+          "column": "location_id"
+        }
+      },
+      {
+        "column": "preferred_working_area_2",
+        "references": {
+          "table": "locations",
+          "column": "location_id"
+        }
+      },
+      {
+        "column": "preferred_working_area_3",
+        "references": {
+          "table": "locations",
+          "column": "location_id"
+        }
+      },
+      {
+        "column": "residential_town_id",
+        "references": {
+          "table": "locations",
+          "column": "location_id"
+        }
+      }
+    ]
+  },
+  "attendance_records": {
+    "comment": "Associates learners with their attendance status on specific dates",
+    "columns": [
+      {
+        "name": "register_id",
+        "data_type": "integer",
+        "comment": "Reference to the attendance register"
+      },
+      {
+        "name": "learner_id",
+        "data_type": "integer",
+        "comment": "Reference to the learner"
+      },
+      {
+        "name": "status",
+        "data_type": "character varying",
+        "comment": "Attendance status of the learner (e.g., 'Present', 'Absent')"
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "learner_id",
+        "references": {
+          "table": "learners",
+          "column": "learner_id"
+        }
+      },
+      {
+        "column": "register_id",
+        "references": {
+          "table": "attendance_registers",
+          "column": "register_id"
+        }
+      }
+    ]
+  },
+  "attendance_registers": {
+    "comment": "Records attendance registers for classes",
+    "columns": [
+      {
+        "name": "register_id",
+        "data_type": "integer",
+        "comment": "Unique internal attendance register ID"
+      },
+      {
+        "name": "class_id",
+        "data_type": "integer",
+        "comment": "Reference to the class"
+      },
+      {
+        "name": "date",
+        "data_type": "date",
+        "comment": "Date of the attendance"
+      },
+      {
+        "name": "agent_id",
+        "data_type": "integer",
+        "comment": "Reference to the agent who conducted the attendance"
+      },
+      {
+        "name": "created_at",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the attendance register was created"
+      },
+      {
+        "name": "updated_at",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the attendance register was last updated"
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "agent_id",
+        "references": {
+          "table": "agents",
+          "column": "agent_id"
+        }
+      },
+      {
+        "column": "class_id",
+        "references": {
+          "table": "classes",
+          "column": "class_id"
+        }
+      }
+    ]
+  },
+  "class_agents": {
+    "comment": "Associates agents with classes they facilitate, including their roles and durations",
+    "columns": [
+      {
+        "name": "class_id",
+        "data_type": "integer",
+        "comment": "Reference to the class"
+      },
+      {
+        "name": "agent_id",
+        "data_type": "integer",
+        "comment": "Reference to the agent facilitating the class"
+      },
+      {
+        "name": "start_date",
+        "data_type": "date",
+        "comment": "Date when the agent started facilitating the class"
+      },
+      {
+        "name": "end_date",
+        "data_type": "date",
+        "comment": "Date when the agent stopped facilitating the class"
+      },
+      {
+        "name": "role",
+        "data_type": "character varying",
+        "comment": "Role of the agent in the class (e.g., 'Original', 'Backup', 'Replacement')"
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "agent_id",
+        "references": {
+          "table": "agents",
+          "column": "agent_id"
+        }
+      },
+      {
+        "column": "class_id",
+        "references": {
+          "table": "classes",
+          "column": "class_id"
+        }
+      }
+    ]
+  },
+  "class_notes": {
+    "comment": "Stores historical notes and remarks about classes",
+    "columns": [
+      {
+        "name": "note_id",
+        "data_type": "integer",
+        "comment": "Unique internal note ID"
+      },
+      {
+        "name": "class_id",
+        "data_type": "integer",
+        "comment": "Reference to the class"
+      },
+      {
+        "name": "note_date",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the note was created"
+      },
+      {
+        "name": "note",
+        "data_type": "text",
+        "comment": "Content of the note regarding the class"
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "class_id",
+        "references": {
+          "table": "classes",
+          "column": "class_id"
+        }
+      }
+    ]
+  },
+  "class_schedules": {
+    "comment": "Stores scheduling information for classes",
+    "columns": [
+      {
+        "name": "schedule_id",
+        "data_type": "integer",
+        "comment": "Unique internal schedule ID"
+      },
+      {
+        "name": "class_id",
+        "data_type": "integer",
+        "comment": "Reference to the class"
+      },
+      {
+        "name": "start_time",
+        "data_type": "time without time zone",
+        "comment": "Class start time"
+      },
+      {
+        "name": "end_time",
+        "data_type": "time without time zone",
+        "comment": "Class end time"
+      },
+      {
+        "name": "day_of_week",
+        "data_type": "character varying",
+        "comment": "Day of the week when the class occurs (e.g., 'Monday')"
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "class_id",
+        "references": {
+          "table": "classes",
+          "column": "class_id"
+        }
+      }
+    ]
+  },
+  "class_subjects": {
+    "comment": "Associates classes with the subjects or products being taught",
+    "columns": [
+      {
+        "name": "class_id",
+        "data_type": "integer",
+        "comment": "Reference to the class"
+      },
+      {
+        "name": "product_id",
+        "data_type": "integer",
+        "comment": "Reference to the subject or product taught in the class"
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "class_id",
+        "references": {
+          "table": "classes",
+          "column": "class_id"
+        }
+      },
+      {
+        "column": "product_id",
+        "references": {
+          "table": "products",
+          "column": "product_id"
+        }
+      }
+    ]
+  },
+  "classes": {
+    "comment": "Stores information about classes, including scheduling and associations",
     "columns": [
       {
         "name": "updated_at",
         "data_type": "timestamp without time zone",
-        "comment": null
+        "comment": "Timestamp when the class record was last updated"
+      },
+      {
+        "name": "client_id",
+        "data_type": "integer",
+        "comment": "Reference to the client associated with the class"
+      },
+      {
+        "name": "stop_date",
+        "data_type": "date",
+        "comment": "Date when the class stopped"
+      },
+      {
+        "name": "restart_date",
+        "data_type": "date",
+        "comment": "Date when the class restarted"
+      },
+      {
+        "name": "seta_funded",
+        "data_type": "boolean",
+        "comment": "Indicates if the project is SETA funded (true) or not (false)"
+      },
+      {
+        "name": "exam_class",
+        "data_type": "boolean",
+        "comment": "Indicates if this is an exam project (true) or not (false)"
+      },
+      {
+        "name": "project_supervisor_id",
+        "data_type": "integer",
+        "comment": "Reference to the project supervisor managing the class"
+      },
+      {
+        "name": "delivery_date",
+        "data_type": "date",
+        "comment": "Date when materials or resources must be delivered to the class"
+      },
+      {
+        "name": "collection_date",
+        "data_type": "date",
+        "comment": "Date when portfolios or materials must be collected from the class"
+      },
+      {
+        "name": "created_at",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the class record was created"
+      },
+      {
+        "name": "class_id",
+        "data_type": "integer",
+        "comment": "Unique internal class ID"
+      },
+      {
+        "name": "class_town_id",
+        "data_type": "integer",
+        "comment": "Reference to the town where the class takes place"
+      },
+      {
+        "name": "original_start_date",
+        "data_type": "date",
+        "comment": "Original start date of the class"
+      },
+      {
+        "name": "class_site_name",
+        "data_type": "character varying",
+        "comment": "Name of the class or site"
+      },
+      {
+        "name": "class_address_line",
+        "data_type": "character varying",
+        "comment": "Street address where the class takes place"
+      },
+      {
+        "name": "class_suburb",
+        "data_type": "character varying",
+        "comment": "Suburb where the class takes place"
+      },
+      {
+        "name": "class_notes",
+        "data_type": "text",
+        "comment": "Notes about the class; important information to remember"
+      },
+      {
+        "name": "class_postal_code",
+        "data_type": "character varying",
+        "comment": "Postal code of the class location"
+      },
+      {
+        "name": "class_type",
+        "data_type": "character varying",
+        "comment": "Type of class; determines the 'rules' (e.g., 'Employed', 'Community')"
+      },
+      {
+        "name": "class_status",
+        "data_type": "character varying",
+        "comment": "Status of the class (e.g., 'New', 'Restarted', 'Stopped')"
+      },
+      {
+        "name": "exam_type",
+        "data_type": "character varying",
+        "comment": "Type of exam associated with the class"
+      },
+      {
+        "name": "seta",
+        "data_type": "character varying",
+        "comment": "Name of the SETA (Sector Education and Training Authority) the client belongs to"
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "class_town_id",
+        "references": {
+          "table": "locations",
+          "column": "location_id"
+        }
+      },
+      {
+        "column": "client_id",
+        "references": {
+          "table": "clients",
+          "column": "client_id"
+        }
+      },
+      {
+        "column": "project_supervisor_id",
+        "references": {
+          "table": "users",
+          "column": "user_id"
+        }
+      }
+    ]
+  },
+  "client_communications": {
+    "comment": "Stores records of communications with clients",
+    "columns": [
+      {
+        "name": "client_id",
+        "data_type": "integer",
+        "comment": "Reference to the client"
+      },
+      {
+        "name": "communication_date",
+        "data_type": "timestamp without time zone",
+        "comment": "Date and time when the communication occurred"
+      },
+      {
+        "name": "user_id",
+        "data_type": "integer",
+        "comment": "Reference to the user who communicated with the client"
+      },
+      {
+        "name": "communication_id",
+        "data_type": "integer",
+        "comment": "Unique internal communication ID"
+      },
+      {
+        "name": "subject",
+        "data_type": "character varying",
+        "comment": "Subject of the communication"
+      },
+      {
+        "name": "communication_type",
+        "data_type": "character varying",
+        "comment": "Type of communication (e.g., 'Email', 'Phone Call')"
+      },
+      {
+        "name": "content",
+        "data_type": "text",
+        "comment": "Content or summary of the communication"
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "client_id",
+        "references": {
+          "table": "clients",
+          "column": "client_id"
+        }
       },
       {
-        "name": "agent_id",
+        "column": "user_id",
+        "references": {
+          "table": "users",
+          "column": "user_id"
+        }
+      }
+    ]
+  },
+  "client_contact_persons": {
+    "comment": "Stores contact person information for clients",
+    "columns": [
+      {
+        "name": "contact_id",
         "data_type": "integer",
-        "comment": null
+        "comment": "Unique internal contact person ID"
       },
       {
-        "name": "class_id",
+        "name": "client_id",
         "data_type": "integer",
-        "comment": null
+        "comment": "Reference to the client"
+      },
+      {
+        "name": "first_name",
+        "data_type": "character varying",
+        "comment": "First name of the contact person"
+      },
+      {
+        "name": "surname",
+        "data_type": "character varying",
+        "comment": "Surname of the contact person"
+      },
+      {
+        "name": "email",
+        "data_type": "character varying",
+        "comment": "Email address of the contact person"
+      },
+      {
+        "name": "cellphone_number",
+        "data_type": "character varying",
+        "comment": "Cellphone number of the contact person"
+      },
+      {
+        "name": "tel_number",
+        "data_type": "character varying",
+        "comment": "Landline number of the contact person"
       },
       {
-        "name": "task_id",
+        "name": "position",
+        "data_type": "character varying",
+        "comment": "Position or role of the contact person at the client company"
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "client_id",
+        "references": {
+          "table": "clients",
+          "column": "client_id"
+        }
+      }
+    ]
+  },
+  "clients": {
+    "comment": "Stores information about clients (companies or organizations)",
+    "columns": [
+      {
+        "name": "client_id",
         "data_type": "integer",
-        "comment": null
+        "comment": "Unique internal client ID"
       },
       {
-        "name": "assignment_date",
-        "data_type": "date",
-        "comment": null
+        "name": "branch_of",
+        "data_type": "integer",
+        "comment": "Reference to the parent client if this client is a branch"
       },
       {
-        "name": "reassigned_agent_id",
+        "name": "town_id",
         "data_type": "integer",
-        "comment": null
+        "comment": "Reference to the town where the client is located"
+      },
+      {
+        "name": "financial_year_end",
+        "data_type": "date",
+        "comment": "Date of the client's financial year-end"
       },
       {
-        "name": "reassignment_date",
+        "name": "bbbee_verification_date",
         "data_type": "date",
-        "comment": null
+        "comment": "Date of the client's BBBEE verification"
       },
       {
         "name": "created_at",
         "data_type": "timestamp without time zone",
-        "comment": null
+        "comment": "Timestamp when the client record was created"
       },
       {
-        "name": "id",
-        "data_type": "integer",
-        "comment": null
+        "name": "updated_at",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the client record was last updated"
       },
       {
-        "name": "reassignment_reason",
-        "data_type": "text",
-        "comment": null
+        "name": "postal_code",
+        "data_type": "character varying",
+        "comment": "Postal code of the client's location"
       },
       {
-        "name": "status",
+        "name": "client_name",
+        "data_type": "character varying",
+        "comment": "Name of the client company or organization"
+      },
+      {
+        "name": "seta",
+        "data_type": "character varying",
+        "comment": "SETA the client belongs to"
+      },
+      {
+        "name": "company_registration_number",
+        "data_type": "character varying",
+        "comment": "Company registration number of the client"
+      },
+      {
+        "name": "address_line",
+        "data_type": "character varying",
+        "comment": "Client's street address"
+      },
+      {
+        "name": "suburb",
+        "data_type": "character varying",
+        "comment": "Suburb where the client is located"
+      },
+      {
+        "name": "client_status",
         "data_type": "character varying",
-        "comment": null
+        "comment": "Current status of the client (e.g., 'Active Client', 'Lost Client')"
       }
     ],
     "foreign_keys": [
       {
-        "column": "agent_id",
-        "references": {
-          "table": "agents",
-          "column": "id"
-        }
-      },
-      {
-        "column": "class_id",
+        "column": "branch_of",
         "references": {
-          "table": "classes",
-          "column": "id"
+          "table": "clients",
+          "column": "client_id"
         }
       },
       {
-        "column": "task_id",
+        "column": "town_id",
         "references": {
-          "table": "tasks",
-          "column": "id"
+          "table": "locations",
+          "column": "location_id"
         }
       }
     ]
   },
-  "agent_availability": {
-    "comment": null,
+  "collections": {
+    "comment": "Records collections made from classes",
     "columns": [
       {
         "name": "created_at",
         "data_type": "timestamp without time zone",
-        "comment": null
+        "comment": "Timestamp when the collection record was created"
       },
       {
         "name": "updated_at",
         "data_type": "timestamp without time zone",
-        "comment": null
+        "comment": "Timestamp when the collection record was last updated"
       },
       {
-        "name": "available_date",
+        "name": "collection_date",
         "data_type": "date",
-        "comment": null
+        "comment": "Date when the collection is scheduled or occurred"
       },
       {
-        "name": "id",
+        "name": "collection_id",
         "data_type": "integer",
-        "comment": null
+        "comment": "Unique internal collection ID"
       },
       {
-        "name": "agent_id",
+        "name": "class_id",
         "data_type": "integer",
-        "comment": null
+        "comment": "Reference to the class"
       },
       {
-        "name": "availability_status",
-        "data_type": "character varying",
-        "comment": null
+        "name": "items",
+        "data_type": "text",
+        "comment": "Items collected from the class"
       },
       {
-        "name": "reason",
-        "data_type": "text",
-        "comment": null
+        "name": "status",
+        "data_type": "character varying",
+        "comment": "Collection status (e.g., 'Pending', 'Collected')"
       }
     ],
     "foreign_keys": [
       {
-        "column": "agent_id",
+        "column": "class_id",
         "references": {
-          "table": "agents",
-          "column": "id"
+          "table": "classes",
+          "column": "class_id"
         }
       }
     ]
   },
-  "agent_work_history": {
-    "comment": null,
+  "deliveries": {
+    "comment": "Records deliveries made to classes",
     "columns": [
+      {
+        "name": "created_at",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the delivery record was created"
+      },
       {
         "name": "updated_at",
         "data_type": "timestamp without time zone",
-        "comment": null
+        "comment": "Timestamp when the delivery record was last updated"
       },
       {
-        "name": "agent_id",
+        "name": "delivery_date",
+        "data_type": "date",
+        "comment": "Date when the delivery is scheduled or occurred"
+      },
+      {
+        "name": "delivery_id",
         "data_type": "integer",
-        "comment": null
+        "comment": "Unique internal delivery ID"
       },
       {
         "name": "class_id",
         "data_type": "integer",
-        "comment": null
+        "comment": "Reference to the class"
+      },
+      {
+        "name": "items",
+        "data_type": "text",
+        "comment": "Items included in the delivery"
       },
       {
-        "name": "task_id",
+        "name": "status",
+        "data_type": "character varying",
+        "comment": "Delivery status (e.g., 'Pending', 'Delivered')"
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "class_id",
+        "references": {
+          "table": "classes",
+          "column": "class_id"
+        }
+      }
+    ]
+  },
+  "employers": {
+    "comment": "Stores information about employers or sponsors of learners",
+    "columns": [
+      {
+        "name": "employer_id",
         "data_type": "integer",
-        "comment": null
+        "comment": "Unique internal employer ID"
       },
       {
-        "name": "start_date",
-        "data_type": "date",
-        "comment": null
+        "name": "created_at",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the employer record was created"
       },
       {
-        "name": "end_date",
-        "data_type": "date",
-        "comment": null
+        "name": "updated_at",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the employer record was last updated"
+      },
+      {
+        "name": "employer_name",
+        "data_type": "character varying",
+        "comment": "Name of the employer or sponsoring organization"
+      }
+    ],
+    "foreign_keys": []
+  },
+  "exam_results": {
+    "comment": "Stores detailed exam results for learners",
+    "columns": [
+      {
+        "name": "updated_at",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the exam result was last updated"
+      },
+      {
+        "name": "exam_id",
+        "data_type": "integer",
+        "comment": "Reference to the exam"
+      },
+      {
+        "name": "learner_id",
+        "data_type": "integer",
+        "comment": "Reference to the learner"
       },
       {
         "name": "created_at",
         "data_type": "timestamp without time zone",
-        "comment": null
+        "comment": "Timestamp when the exam result was created"
       },
       {
-        "name": "id",
+        "name": "result_id",
         "data_type": "integer",
-        "comment": null
+        "comment": "Unique internal exam result ID"
       },
       {
-        "name": "reassignment_id",
+        "name": "mock_exam_number",
         "data_type": "integer",
-        "comment": null
+        "comment": "Number of the mock exam (e.g., 1, 2, 3)"
       },
       {
-        "name": "role",
+        "name": "score",
+        "data_type": "numeric",
+        "comment": "Learner's score in the exam"
+      },
+      {
+        "name": "exam_date",
+        "data_type": "date",
+        "comment": "Date when the exam was taken"
+      },
+      {
+        "name": "subject",
         "data_type": "character varying",
-        "comment": null
+        "comment": "Subject of the exam"
       },
       {
-        "name": "performance_notes",
-        "data_type": "text",
-        "comment": null
+        "name": "result",
+        "data_type": "character varying",
+        "comment": "Exam result (e.g., 'Pass', 'Fail')"
       }
     ],
     "foreign_keys": [
       {
-        "column": "agent_id",
-        "references": {
-          "table": "agents",
-          "column": "id"
-        }
-      },
-      {
-        "column": "class_id",
+        "column": "exam_id",
         "references": {
-          "table": "classes",
-          "column": "id"
+          "table": "exams",
+          "column": "exam_id"
         }
       },
       {
-        "column": "task_id",
+        "column": "learner_id",
         "references": {
-          "table": "tasks",
-          "column": "id"
+          "table": "learners",
+          "column": "learner_id"
         }
       }
     ]
   },
-  "agents": {
-    "comment": null,
+  "exams": {
+    "comment": "Stores exam results for learners",
     "columns": [
       {
-        "name": "updated_at",
-        "data_type": "timestamp without time zone",
-        "comment": null
+        "name": "exam_id",
+        "data_type": "integer",
+        "comment": "Unique internal exam ID"
       },
       {
-        "name": "experience",
+        "name": "learner_id",
         "data_type": "integer",
-        "comment": null
+        "comment": "Reference to the learner"
       },
       {
-        "name": "current_workload",
+        "name": "product_id",
         "data_type": "integer",
-        "comment": null
+        "comment": "Reference to the product or subject"
+      },
+      {
+        "name": "exam_date",
+        "data_type": "date",
+        "comment": "Date when the exam was taken"
       },
       {
         "name": "created_at",
         "data_type": "timestamp without time zone",
-        "comment": null
+        "comment": "Timestamp when the exam record was created"
       },
       {
-        "name": "id",
-        "data_type": "integer",
-        "comment": null
+        "name": "updated_at",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the exam record was last updated"
       },
       {
-        "name": "location",
+        "name": "score",
+        "data_type": "numeric",
+        "comment": "Learner's score in the exam"
+      },
+      {
+        "name": "exam_type",
         "data_type": "character varying",
-        "comment": null
+        "comment": "Type of exam (e.g., 'Mock', 'Final')"
       },
       {
-        "name": "status",
+        "name": "result",
+        "data_type": "character varying",
+        "comment": "Exam result (e.g., 'Pass', 'Fail')"
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "learner_id",
+        "references": {
+          "table": "learners",
+          "column": "learner_id"
+        }
+      },
+      {
+        "column": "product_id",
+        "references": {
+          "table": "products",
+          "column": "product_id"
+        }
+      }
+    ]
+  },
+  "files": {
+    "comment": "Stores references to files associated with various entities",
+    "columns": [
+      {
+        "name": "file_id",
+        "data_type": "integer",
+        "comment": "Unique internal file ID"
+      },
+      {
+        "name": "owner_id",
+        "data_type": "integer",
+        "comment": "ID of the owner entity"
+      },
+      {
+        "name": "uploaded_at",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the file was uploaded"
+      },
+      {
+        "name": "owner_type",
         "data_type": "character varying",
-        "comment": null
+        "comment": "Type of entity that owns the file (e.g., 'Learner', 'Class', 'Agent')"
       },
       {
-        "name": "name",
+        "name": "file_path",
         "data_type": "character varying",
-        "comment": null
+        "comment": "File path or URL to the stored file"
       },
       {
-        "name": "email",
+        "name": "file_type",
         "data_type": "character varying",
-        "comment": null
+        "comment": "Type of file (e.g., 'Scanned Portfolio', 'QA Report')"
+      }
+    ],
+    "foreign_keys": []
+  },
+  "history": {
+    "comment": "Records historical changes and actions performed on entities",
+    "columns": [
+      {
+        "name": "action_date",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the action occurred"
+      },
+      {
+        "name": "user_id",
+        "data_type": "integer",
+        "comment": "Reference to the user who performed the action"
+      },
+      {
+        "name": "entity_id",
+        "data_type": "integer",
+        "comment": "ID of the entity"
+      },
+      {
+        "name": "history_id",
+        "data_type": "integer",
+        "comment": "Unique internal history ID"
       },
       {
-        "name": "phone",
+        "name": "changes",
+        "data_type": "jsonb",
+        "comment": "Details of the changes made, stored in JSON format"
+      },
+      {
+        "name": "action",
         "data_type": "character varying",
-        "comment": null
+        "comment": "Type of action performed (e.g., 'Created', 'Updated', 'Deleted')"
       },
       {
-        "name": "specialization",
+        "name": "entity_type",
         "data_type": "character varying",
-        "comment": null
+        "comment": "Type of entity the history record refers to (e.g., 'Learner', 'Agent', 'Class')"
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "user_id",
+        "references": {
+          "table": "users",
+          "column": "user_id"
+        }
+      }
+    ]
+  },
+  "learner_products": {
+    "comment": "Associates learners with the products they are enrolled in",
+    "columns": [
+      {
+        "name": "learner_id",
+        "data_type": "integer",
+        "comment": "Reference to the learner"
+      },
+      {
+        "name": "product_id",
+        "data_type": "integer",
+        "comment": "Reference to the product the learner is enrolled in"
+      },
+      {
+        "name": "start_date",
+        "data_type": "date",
+        "comment": "Start date of the learner's enrollment in the product"
+      },
+      {
+        "name": "end_date",
+        "data_type": "date",
+        "comment": "End date of the learner's enrollment in the product"
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "learner_id",
+        "references": {
+          "table": "learners",
+          "column": "learner_id"
+        }
+      },
+      {
+        "column": "product_id",
+        "references": {
+          "table": "products",
+          "column": "product_id"
+        }
       }
-    ],
-    "foreign_keys": []
+    ]
   },
-  "assessments": {
-    "comment": null,
+  "learner_progressions": {
+    "comment": "Tracks the progression of learners between products",
     "columns": [
       {
-        "name": "id",
+        "name": "progression_id",
         "data_type": "integer",
-        "comment": null
+        "comment": "Unique internal progression ID"
       },
       {
-        "name": "class_id",
+        "name": "learner_id",
         "data_type": "integer",
-        "comment": null
-      },
-      {
-        "name": "assessment_date",
-        "data_type": "date",
-        "comment": null
-      },
-      {
-        "name": "created_at",
-        "data_type": "timestamp without time zone",
-        "comment": null
+        "comment": "Reference to the learner"
       },
       {
-        "name": "updated_at",
-        "data_type": "timestamp without time zone",
-        "comment": null
+        "name": "from_product_id",
+        "data_type": "integer",
+        "comment": "Reference to the initial product"
       },
       {
-        "name": "assessment_type",
-        "data_type": "character varying",
-        "comment": null
+        "name": "to_product_id",
+        "data_type": "integer",
+        "comment": "Reference to the new product after progression"
       },
       {
-        "name": "assessor_name",
-        "data_type": "character varying",
-        "comment": null
+        "name": "progression_date",
+        "data_type": "date",
+        "comment": "Date when the learner progressed to the new product"
       },
       {
-        "name": "result",
-        "data_type": "character varying",
-        "comment": null
+        "name": "notes",
+        "data_type": "text",
+        "comment": "Additional notes regarding the progression"
       }
     ],
     "foreign_keys": [
       {
-        "column": "class_id",
+        "column": "from_product_id",
         "references": {
-          "table": "classes",
-          "column": "id"
+          "table": "products",
+          "column": "product_id"
+        }
+      },
+      {
+        "column": "learner_id",
+        "references": {
+          "table": "learners",
+          "column": "learner_id"
+        }
+      },
+      {
+        "column": "to_product_id",
+        "references": {
+          "table": "products",
+          "column": "product_id"
         }
       }
     ]
   },
-  "classes": {
-    "comment": null,
+  "learners": {
+    "comment": "Stores personal, educational, and assessment information about learners",
     "columns": [
       {
-        "name": "id",
+        "name": "updated_at",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the learner record was last updated"
+      },
+      {
+        "name": "city_town_id",
         "data_type": "integer",
-        "comment": null
+        "comment": "Reference to the city or town where the learner lives"
       },
       {
-        "name": "client_id",
+        "name": "province_region_id",
         "data_type": "integer",
-        "comment": null
+        "comment": "Reference to the province/region where the learner lives"
       },
       {
-        "name": "start_date",
+        "name": "date_loaded",
         "data_type": "date",
-        "comment": null
+        "comment": "Date when the learner was added to the system"
       },
       {
-        "name": "end_date",
+        "name": "placement_assessment_date",
         "data_type": "date",
-        "comment": null
+        "comment": "Date when the learner took the placement assessment"
       },
       {
-        "name": "created_at",
-        "data_type": "timestamp without time zone",
-        "comment": null
+        "name": "employment_status",
+        "data_type": "boolean",
+        "comment": "Indicates if the learner is employed (true) or not (false)"
       },
       {
-        "name": "updated_at",
-        "data_type": "timestamp without time zone",
-        "comment": null
+        "name": "employer_id",
+        "data_type": "integer",
+        "comment": "Reference to the learner's employer or sponsor"
       },
       {
-        "name": "attendance_status",
-        "data_type": "character varying",
-        "comment": null
+        "name": "disability_status",
+        "data_type": "boolean",
+        "comment": "Indicates if the learner has a disability (true) or not (false)"
       },
       {
-        "name": "progression_status",
-        "data_type": "character varying",
-        "comment": null
+        "name": "created_at",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the learner record was created"
       },
       {
-        "name": "subject",
-        "data_type": "character varying",
-        "comment": null
+        "name": "learner_id",
+        "data_type": "integer",
+        "comment": "Unique internal learner ID"
       },
       {
-        "name": "site",
+        "name": "email_address",
         "data_type": "character varying",
-        "comment": null
+        "comment": "Learner's email address"
       },
       {
-        "name": "phase",
+        "name": "address_line_1",
         "data_type": "character varying",
-        "comment": null
+        "comment": "First line of learner's physical address"
       },
       {
-        "name": "marketer",
+        "name": "address_line_2",
         "data_type": "character varying",
-        "comment": null
+        "comment": "Second line of learner's physical address"
       },
       {
-        "name": "status",
+        "name": "scanned_portfolio",
         "data_type": "character varying",
-        "comment": null
-      }
-    ],
-    "foreign_keys": [
-      {
-        "column": "client_id",
-        "references": {
-          "table": "clients",
-          "column": "id"
-        }
-      }
-    ]
-  },
-  "clients": {
-    "comment": null,
-    "columns": [
-      {
-        "name": "id",
-        "data_type": "integer",
-        "comment": null
+        "comment": "File path or URL to the learner's scanned portfolio in PDF format"
       },
       {
-        "name": "created_at",
-        "data_type": "timestamp without time zone",
-        "comment": null
+        "name": "placement_level_communications",
+        "data_type": "character varying",
+        "comment": "Learner's initial placement level in Communications (e.g., 'CL1b', 'CL1', 'CL2')"
       },
       {
-        "name": "updated_at",
-        "data_type": "timestamp without time zone",
-        "comment": null
+        "name": "postal_code",
+        "data_type": "character varying",
+        "comment": "Postal code of the learner's area"
       },
       {
-        "name": "email",
+        "name": "highest_qualification",
         "data_type": "character varying",
-        "comment": null
+        "comment": "Highest qualification the learner has achieved"
       },
       {
-        "name": "phone",
+        "name": "placement_level_mathematics",
         "data_type": "character varying",
-        "comment": null
+        "comment": "Learner's initial placement level in Mathematics (e.g., 'ML1', 'ML2', 'ML3')"
       },
       {
-        "name": "address",
-        "data_type": "text",
-        "comment": null
+        "name": "assessment_status",
+        "data_type": "character varying",
+        "comment": "Assessment status; indicates if the learner was assessed ('Assessed', 'Not Assessed')"
       },
       {
-        "name": "status",
+        "name": "first_name",
         "data_type": "character varying",
-        "comment": null
+        "comment": "Learner's first name"
       },
       {
-        "name": "name",
+        "name": "initials",
         "data_type": "character varying",
-        "comment": null
+        "comment": "Learner's initials"
       },
       {
-        "name": "contact_person",
+        "name": "surname",
         "data_type": "character varying",
-        "comment": null
-      }
-    ],
-    "foreign_keys": []
-  },
-  "deliveries": {
-    "comment": null,
-    "columns": [
-      {
-        "name": "delivery_date",
-        "data_type": "date",
-        "comment": null
+        "comment": "Learner's surname"
       },
       {
-        "name": "class_id",
-        "data_type": "integer",
-        "comment": null
+        "name": "gender",
+        "data_type": "character varying",
+        "comment": "Learner's gender"
       },
       {
-        "name": "created_at",
-        "data_type": "timestamp without time zone",
-        "comment": null
+        "name": "race",
+        "data_type": "character varying",
+        "comment": "Learner's race; options include 'African', 'Coloured', 'White', 'Indian'"
       },
       {
-        "name": "updated_at",
-        "data_type": "timestamp without time zone",
-        "comment": null
+        "name": "sa_id_no",
+        "data_type": "character varying",
+        "comment": "Learner's South African ID number"
       },
       {
-        "name": "id",
-        "data_type": "integer",
-        "comment": null
+        "name": "passport_number",
+        "data_type": "character varying",
+        "comment": "Learner's passport number if they are a foreigner"
       },
       {
-        "name": "delivery_type",
+        "name": "tel_number",
         "data_type": "character varying",
-        "comment": null
+        "comment": "Learner's primary telephone number"
       },
       {
-        "name": "status",
+        "name": "alternative_tel_number",
         "data_type": "character varying",
-        "comment": null
+        "comment": "Learner's alternative contact number"
       }
     ],
     "foreign_keys": [
       {
-        "column": "class_id",
+        "column": "city_town_id",
         "references": {
-          "table": "classes",
-          "column": "id"
+          "table": "locations",
+          "column": "location_id"
+        }
+      },
+      {
+        "column": "employer_id",
+        "references": {
+          "table": "employers",
+          "column": "employer_id"
+        }
+      },
+      {
+        "column": "province_region_id",
+        "references": {
+          "table": "locations",
+          "column": "location_id"
         }
       }
     ]
   },
-  "events": {
-    "comment": null,
+  "locations": {
+    "comment": "Stores geographical location data for addresses",
     "columns": [
       {
-        "name": "id",
+        "name": "location_id",
         "data_type": "integer",
-        "comment": null
+        "comment": "Unique internal location ID"
       },
       {
-        "name": "created_at",
-        "data_type": "timestamp without time zone",
-        "comment": null
+        "name": "longitude",
+        "data_type": "numeric",
+        "comment": "Geographical longitude coordinate"
       },
       {
-        "name": "updated_at",
-        "data_type": "timestamp without time zone",
-        "comment": null
+        "name": "latitude",
+        "data_type": "numeric",
+        "comment": "Geographical latitude coordinate"
       },
       {
-        "name": "client_id",
-        "data_type": "integer",
-        "comment": null
+        "name": "created_at",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the location record was created"
       },
       {
-        "name": "class_id",
-        "data_type": "integer",
-        "comment": null
+        "name": "updated_at",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the location record was last updated"
       },
       {
-        "name": "event_date",
-        "data_type": "date",
-        "comment": null
+        "name": "suburb",
+        "data_type": "character varying",
+        "comment": "Suburb name"
       },
       {
-        "name": "reminder_date",
-        "data_type": "date",
-        "comment": null
+        "name": "town",
+        "data_type": "character varying",
+        "comment": "Town name"
       },
       {
-        "name": "name",
+        "name": "province",
         "data_type": "character varying",
-        "comment": null
+        "comment": "Province name"
       },
       {
-        "name": "event_type",
+        "name": "postal_code",
         "data_type": "character varying",
-        "comment": null
+        "comment": "Postal code"
       }
     ],
-    "foreign_keys": [
+    "foreign_keys": []
+  },
+  "products": {
+    "comment": "Stores information about educational products or courses",
+    "columns": [
       {
-        "column": "class_id",
-        "references": {
-          "table": "classes",
-          "column": "id"
-        }
+        "name": "product_id",
+        "data_type": "integer",
+        "comment": "Unique internal product ID"
       },
       {
-        "column": "client_id",
-        "references": {
-          "table": "clients",
-          "column": "id"
-        }
-      }
-    ]
-  },
-  "learners": {
-    "comment": null,
-    "columns": [
+        "name": "product_duration",
+        "data_type": "integer",
+        "comment": "Total duration of the product in hours"
+      },
       {
-        "name": "id",
+        "name": "learning_area_duration",
         "data_type": "integer",
-        "comment": null
+        "comment": "Duration of each learning area in hours"
       },
       {
-        "name": "class_id",
+        "name": "parent_product_id",
         "data_type": "integer",
-        "comment": null
+        "comment": "Reference to a parent product for hierarchical structuring"
       },
       {
         "name": "created_at",
         "data_type": "timestamp without time zone",
-        "comment": null
+        "comment": "Timestamp when the product record was created"
       },
       {
         "name": "updated_at",
         "data_type": "timestamp without time zone",
-        "comment": null
+        "comment": "Timestamp when the product record was last updated"
       },
       {
-        "name": "id_number",
-        "data_type": "character varying",
-        "comment": null
+        "name": "product_notes",
+        "data_type": "text",
+        "comment": "Notes or additional information about the product"
       },
       {
-        "name": "name",
+        "name": "product_name",
         "data_type": "character varying",
-        "comment": null
+        "comment": "Name of the product or course"
       },
       {
-        "name": "gender",
-        "data_type": "character varying",
-        "comment": null
+        "name": "product_rules",
+        "data_type": "text",
+        "comment": "Rules or guidelines associated with the product"
       },
       {
-        "name": "race",
+        "name": "learning_area",
         "data_type": "character varying",
-        "comment": null
+        "comment": "Learning areas covered by the product (e.g., 'Communication', 'Numeracy')"
+      },
+      {
+        "name": "product_flags",
+        "data_type": "text",
+        "comment": "Flags or alerts for the product (e.g., attendance thresholds)"
+      },
+      {
+        "name": "reporting_structure",
+        "data_type": "text",
+        "comment": "Structure of progress reports for the product"
       }
     ],
     "foreign_keys": [
       {
-        "column": "class_id",
+        "column": "parent_product_id",
         "references": {
-          "table": "classes",
-          "column": "id"
+          "table": "products",
+          "column": "product_id"
         }
       }
     ]
   },
-  "progressions": {
-    "comment": null,
+  "progress_reports": {
+    "comment": "Stores progress reports for learners in specific classes and products",
     "columns": [
       {
         "name": "updated_at",
         "data_type": "timestamp without time zone",
-        "comment": null
+        "comment": "Timestamp when the progress report was last updated"
       },
       {
         "name": "class_id",
         "data_type": "integer",
-        "comment": null
+        "comment": "Reference to the class"
       },
       {
         "name": "learner_id",
         "data_type": "integer",
-        "comment": null
+        "comment": "Reference to the learner"
       },
       {
-        "name": "created_at",
-        "data_type": "timestamp without time zone",
-        "comment": null
+        "name": "product_id",
+        "data_type": "integer",
+        "comment": "Reference to the product or subject"
       },
       {
-        "name": "id",
-        "data_type": "integer",
-        "comment": null
+        "name": "progress_percentage",
+        "data_type": "numeric",
+        "comment": "Learner's progress percentage in the product"
       },
       {
-        "name": "completion_date",
+        "name": "report_date",
         "data_type": "date",
-        "comment": null
+        "comment": "Date when the progress report was generated"
       },
       {
-        "name": "progression_level",
-        "data_type": "character varying",
-        "comment": null
+        "name": "report_id",
+        "data_type": "integer",
+        "comment": "Unique internal progress report ID"
       },
       {
-        "name": "status",
-        "data_type": "character varying",
-        "comment": null
+        "name": "created_at",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the progress report was created"
+      },
+      {
+        "name": "remarks",
+        "data_type": "text",
+        "comment": "Additional remarks or comments"
       }
     ],
     "foreign_keys": [
@@ -677,126 +1880,185 @@
         "column": "class_id",
         "references": {
           "table": "classes",
-          "column": "id"
+          "column": "class_id"
         }
       },
       {
         "column": "learner_id",
         "references": {
           "table": "learners",
-          "column": "id"
+          "column": "learner_id"
+        }
+      },
+      {
+        "column": "product_id",
+        "references": {
+          "table": "products",
+          "column": "product_id"
         }
       }
     ]
   },
-  "reports": {
-    "comment": null,
+  "qa_reports": {
+    "comment": "Stores QA (Quality Assurance) reports for classes and agents",
     "columns": [
       {
-        "name": "related_client_id",
+        "name": "qa_report_id",
         "data_type": "integer",
-        "comment": null
+        "comment": "Unique internal QA report ID"
       },
       {
-        "name": "created_at",
-        "data_type": "timestamp without time zone",
-        "comment": null
+        "name": "class_id",
+        "data_type": "integer",
+        "comment": "Reference to the class"
       },
       {
-        "name": "updated_at",
-        "data_type": "timestamp without time zone",
-        "comment": null
+        "name": "agent_id",
+        "data_type": "integer",
+        "comment": "Reference to the agent"
       },
       {
-        "name": "related_class_id",
-        "data_type": "integer",
-        "comment": null
+        "name": "report_date",
+        "data_type": "date",
+        "comment": "Date when the QA report was created"
       },
       {
-        "name": "id",
-        "data_type": "integer",
-        "comment": null
+        "name": "created_at",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the QA report was created"
+      },
+      {
+        "name": "updated_at",
+        "data_type": "timestamp without time zone",
+        "comment": "Timestamp when the QA report was last updated"
       },
       {
-        "name": "report_type",
+        "name": "report_file",
         "data_type": "character varying",
-        "comment": null
+        "comment": "File path or URL to the QA report"
       },
       {
-        "name": "content",
+        "name": "notes",
         "data_type": "text",
-        "comment": null
+        "comment": "Additional notes or observations from the QA report"
       }
     ],
     "foreign_keys": [
       {
-        "column": "related_class_id",
+        "column": "agent_id",
         "references": {
-          "table": "classes",
-          "column": "id"
+          "table": "agents",
+          "column": "agent_id"
         }
       },
       {
-        "column": "related_client_id",
+        "column": "class_id",
         "references": {
-          "table": "clients",
-          "column": "id"
+          "table": "classes",
+          "column": "class_id"
         }
       }
     ]
   },
-  "tasks": {
-    "comment": null,
+  "user_permissions": {
+    "comment": "Grants specific permissions to users",
     "columns": [
       {
-        "name": "id",
+        "name": "permission_id",
         "data_type": "integer",
-        "comment": null
+        "comment": "Unique internal permission ID"
       },
       {
-        "name": "event_id",
+        "name": "user_id",
         "data_type": "integer",
-        "comment": null
+        "comment": "Reference to the user"
       },
       {
-        "name": "due_date",
-        "data_type": "date",
-        "comment": null
+        "name": "permission",
+        "data_type": "character varying",
+        "comment": "Specific permission granted to the user"
+      }
+    ],
+    "foreign_keys": [
+      {
+        "column": "user_id",
+        "references": {
+          "table": "users",
+          "column": "user_id"
+        }
+      }
+    ]
+  },
+  "user_roles": {
+    "comment": "Defines roles and associated permissions for users",
+    "columns": [
+      {
+        "name": "role_id",
+        "data_type": "integer",
+        "comment": "Unique internal role ID"
+      },
+      {
+        "name": "permissions",
+        "data_type": "jsonb",
+        "comment": "Permissions associated with the role, stored in JSON format"
+      },
+      {
+        "name": "role_name",
+        "data_type": "character varying",
+        "comment": "Name of the role (e.g., 'Admin', 'Project Supervisor')"
+      }
+    ],
+    "foreign_keys": []
+  },
+  "users": {
+    "comment": "Stores system user information",
+    "columns": [
+      {
+        "name": "user_id",
+        "data_type": "integer",
+        "comment": "Unique internal user ID"
       },
       {
         "name": "created_at",
         "data_type": "timestamp without time zone",
-        "comment": null
+        "comment": "Timestamp when the user record was created"
       },
       {
         "name": "updated_at",
         "data_type": "timestamp without time zone",
-        "comment": null
+        "comment": "Timestamp when the user record was last updated"
       },
       {
-        "name": "description",
-        "data_type": "text",
-        "comment": null
+        "name": "email",
+        "data_type": "character varying",
+        "comment": "User's email address"
       },
       {
-        "name": "assigned_to",
+        "name": "cellphone_number",
         "data_type": "character varying",
-        "comment": null
+        "comment": "User's cellphone number"
       },
       {
-        "name": "status",
+        "name": "role",
         "data_type": "character varying",
-        "comment": null
-      }
-    ],
-    "foreign_keys": [
+        "comment": "User's role in the system, e.g., 'Admin', 'Project Supervisor'"
+      },
       {
-        "column": "event_id",
-        "references": {
-          "table": "events",
-          "column": "id"
-        }
+        "name": "password_hash",
+        "data_type": "character varying",
+        "comment": "Hashed password for user authentication"
+      },
+      {
+        "name": "first_name",
+        "data_type": "character varying",
+        "comment": "User's first name"
+      },
+      {
+        "name": "surname",
+        "data_type": "character varying",
+        "comment": "User's surname"
       }
-    ]
+    ],
+    "foreign_keys": []
   }
 }