api code changes for schema reader

backend/app/api/endpoints.py

@@ -23,7 +23,7 @@ class QueryRequest(BaseModel):
 from fastapi import HTTPException
 
 @router.post("/process-text")
-def ask_query(req: QueryRequest):
+def process_text(req: QueryRequest):
     try:
         sql = generate_sql_query(req.question)
 
@@ -34,6 +34,7 @@ def ask_query(req: QueryRequest):
 
         # Use utility method to check if the string is a valid SQL query
         formattedSqlQuery =  UtilityClass.is_valid_sql_query(sql)
+        print(f"Formatted SQL Query: {formattedSqlQuery}")
         if not formattedSqlQuery:
             return {"message": str(sql) if sql else "No SQL query could be generated for your question."}
 
@@ -50,13 +51,73 @@ def ask_query(req: QueryRequest):
 
         # Prepare heading and records JSON using LLM
         result_json = UtilityClass.prepare_llm_heading_and_records(req.question, df)
+        
+        # Parse the heading if it's a JSON string containing heading and summary
+        heading_text = result_json["heading"]
+        summary_text = ""
+        
+        if isinstance(heading_text, str):
+            try:
+                import json
+                # Check if it's wrapped in markdown code block
+                if heading_text.strip().startswith('```json') and heading_text.strip().endswith('```'):
+                    # Extract JSON from markdown code block
+                    json_content = heading_text.strip()
+                    # Remove ```json from start and ``` from end
+                    json_content = json_content[7:-3].strip()  # Remove ```json and ```
+                    heading_data = json.loads(json_content)
+                else:
+                    # Try to parse directly
+                    heading_data = json.loads(heading_text)
+                    
+                if isinstance(heading_data, dict):
+                    heading_text = heading_data.get("heading", heading_text)
+                    summary_text = heading_data.get("summary", summary_text)
+            except (json.JSONDecodeError, ValueError):
+                # If not JSON, use as-is
+                pass
 
         return {
             "sql": sql,
             "rows": result_json["records"],
-            "heading": result_json["heading"],
+            "heading": heading_text,  # Send just the heading text
+            "summary": summary_text,  # Send just the summary text
             "chart": chart
         }
     except Exception as e:
         raise HTTPException(status_code=400, detail=f"An error occurred: {str(e)}")
 
+
+@router.get("/health")
+def health_check():
+    """
+    Health check endpoint to verify API and system status.
+    
+    Returns:
+        dict: Health status information including system checks and timestamp
+    """
+    try:
+        health_status = UtilityClass.get_health_status()
+        
+        # Return appropriate HTTP status based on health
+        if health_status["status"] == "healthy":
+            return health_status
+        else:
+            # Return 503 Service Unavailable if any critical component is unhealthy
+            from fastapi import Response
+            import json
+            return Response(
+                content=json.dumps(health_status),
+                media_type="application/json",
+                status_code=503
+            )
+            
+    except Exception as e:
+        # Return 503 if health check itself fails
+        from datetime import datetime
+        error_response = {
+            "status": "unhealthy",
+            "message": f"Health check failed: {str(e)}",
+            "timestamp": datetime.utcnow().isoformat()
+        }
+        raise HTTPException(status_code=503, detail=error_response)

backend/app/db/db_connector.py

@@ -24,16 +24,23 @@ def get_connection():
     if not os.path.isfile(db_path):
         raise FileNotFoundError(f"Database file not found at {db_path}. Set DB_PATH env variable or .env to the correct location.")
     try:
-        conn = sqlite3.connect(db_path, timeout=30, check_same_thread=False)
+        # Remove timeout restriction to allow long-running queries
+        # Complex AI-generated queries may take time and should not be interrupted
+        conn = sqlite3.connect(db_path, check_same_thread=False)
         conn.row_factory = sqlite3.Row  # access columns by name
         return conn
     except sqlite3.OperationalError as e:
         raise Exception(f"Error connecting to DB at {db_path}: {e}")
 
-# Create engine
+# Create engine with no timeout restrictions for complex queries
 engine = create_engine(
     DATABASE_URL,
-    connect_args={"check_same_thread": False}  # Needed for SQLite threading
+    connect_args={
+        "check_same_thread": False,  # Needed for SQLite threading
+        "timeout": 0  # No timeout - wait indefinitely for query completion
+    },
+    pool_timeout=None,  # No pool timeout
+    pool_recycle=-1    # No connection recycling timeout
 )
 
 # Session factory

backend/app/db/schema_reader.py

@@ -24,27 +24,3 @@ def get_schema():
         cursor.close()
         conn.close()
 
-def get_schema2():
-    """Fetch database schema information (tables & columns)."""
-    engine = get_db_engine()
-    inspector = inspect(engine)
-
-    schema = {}
-    for table_name in inspector.get_table_names():
-        columns = [
-            {"name": col["name"], "type": str(col["type"])}
-            for col in inspector.get_columns(table_name)
-        ]
-        schema[table_name] = columns
-
-    return schema
-
-# def get_schema():
-#     engine = get_db_engine()
-#     inspector = inspect(engine)
-
-#     schema_info = {}
-#     for table in inspector.get_table_names():
-#         columns = [col["name"] for col in inspector.get_columns(table)]
-#         schema_info[table] = columns
-#     return schema_info

backend/app/services/query_executor.py

@@ -23,7 +23,12 @@ def run_and_handle_sql_query(sql_query: str, user_question: str):
     try:
         rows = execute_sql_query(sql_query)
         if not rows:
-            llm = ChatOpenAI(model="gpt-4o-mini", temperature=0, api_key=settings.OPENAI_API_KEY)
+            llm = ChatOpenAI(
+                model="gpt-4o-mini", 
+                temperature=0, 
+                api_key=settings.OPENAI_API_KEY,
+                request_timeout=None  # No timeout for API requests
+            )
             no_result_prompt = (
                 f"The following SQL query was generated for the user's question, but it returned no results. "
                 f"User question: {user_question}\nSQL query: {sql_query}\n"

backend/app/services/sql_generator.py

@@ -6,55 +6,116 @@ from langchain.prompts import PromptTemplate, ChatPromptTemplate
 from app.db.schema_reader import get_schema
 from app.db.db_connector import get_db_engine
 from app.db.db_connector import get_connection
+import re
 
-# llm = ChatOpenAI(model="gpt-4", temperature=0, api_key=settings.OPENAI_API_KEY)
+# llm = ChatOpenAI(model="gpt-4o-mini", temperature=0, api_key=settings.OPENAI_API_KEY)
 llm = ChatOpenAI(model="gpt-4o-mini", temperature=0, api_key=settings.OPENAI_API_KEY)
 
+def detect_foreign_key_relationships_only(schema):
+    """Only detect foreign key relationships without any assumptions about data types"""
+    relationships = []
+    
+    for table_name, columns in schema.items():
+        for column in columns:
+            # Look for foreign key patterns: *_id columns
+            if column.endswith('_id') and column != 'id':
+                # Extract potential referenced table name
+                ref_name = column[:-3]  # Remove '_id'
+                
+                # Check for exact match or plural/singular variations
+                possible_refs = [ref_name, ref_name + 's', ref_name[:-1] if ref_name.endswith('s') else ref_name + 's']
+                
+                for possible_ref in possible_refs:
+                    if possible_ref in schema:
+                        relationships.append({
+                            'from_table': table_name,
+                            'from_column': column,
+                            'to_table': possible_ref,
+                            'to_column': 'id'
+                        })
+                        break
+    
+    return relationships
+
+def generate_pure_schema_context(schema):
+    """Generate schema context without any assumptions - let AI figure out everything"""
+    
+    # Just provide the raw schema structure
+    context = "DATABASE SCHEMA:\n\n"
+    
+    # List all tables and their columns
+    context += "TABLES AND COLUMNS:\n"
+    for table_name, columns in schema.items():
+        context += f"- {table_name}: {', '.join(columns)}\n"
+    
+    # Detect only foreign key relationships (no assumptions about data types)
+    relationships = detect_foreign_key_relationships_only(schema)
+    
+    # Show detected relationships
+    if relationships:
+        context += "\nDETECTED FOREIGN KEY RELATIONSHIPS:\n"
+        for rel in relationships:
+            context += f"- {rel['from_table']}.{rel['from_column']} → {rel['to_table']}.{rel['to_column']}\n"
+    
+    return context
+
 
 def generate_sql_query(user_question: str):
     schema = get_schema()
-    schema_str = "\n".join(
-        [f"{table}: {', '.join(cols)}" for table, cols in schema.items()]
-    )
+    
+    # Generate pure schema context - no assumptions, let AI figure it out
+    schema_context = generate_pure_schema_context(schema)
 
     prompt = ChatPromptTemplate.from_template("""
-    You are an expert SQLite query generator.
-    Use ONLY the following tables and columns:
-
-    {schema}
-
-    Generate a correct SQLite query (no explanations, only SQL).
-    Do not add single quotes or any quotes around table names or column names.
-    If the user asks for data, generate a SELECT query and that outcome should have only 
-    sqlite compatible, don't include any other text.
-    If you cannot answer, say: "I cannot generate this query."
+    You are an expert SQLite query generator with advanced analytical capabilities.
+    
+    {schema_context}
+
+    INTELLIGENT ANALYSIS INSTRUCTIONS:
+    1. EXAMINE the schema above - look at table names and column names to understand what data each table contains
+    2. ANALYZE the column names to infer what type of information is stored (names, addresses, amounts, dates, etc.)
+    3. USE the detected foreign key relationships to understand how tables connect
+    4. For queries needing data from multiple tables, follow the relationship paths and use JOINs
+    5. INFER from context - if user asks about location and you see city/address columns, use those tables
+    6. THINK logically about which tables would contain the requested information
+
+    QUERY GENERATION STRATEGY:
+    - Read the user's question carefully
+    - Identify what data they want (customers, transactions, locations, etc.)
+    - Look at the schema to find which tables likely contain that data
+    - Check if multiple tables are needed and use the foreign key relationships
+    - Generate appropriate SQL with JOINs if data spans multiple tables
+
+    RESPONSE FORMAT:
+    - Return ONLY valid SQLite syntax
+    - No explanations, comments, or extra text
+    - Use proper JOIN syntax when needed
+    - If you cannot determine how to query the available schema, respond: "I cannot generate this query."
+
+    Example thinking process:
+    - User asks "customers in New York" → Look for tables with customer info AND location info → Use relationships to JOIN them
+    - User asks "transactions over $1000" → Look for tables with transaction/amount data
+    - User asks "branch managers" → Look for tables with employee/staff info and management roles
 
     User question: {question}
     """)
 
-    messages = prompt.format_messages(schema=schema_str, question=user_question)
+    messages = prompt.format_messages(schema_context=schema_context, question=user_question)
     sql_query = llm.invoke(messages).content.strip()
 
     # If the LLM says it cannot generate a query, treat as normal chat
     if sql_query.lower().startswith("i cannot generate this query"):
         # Use the LLM as a chatbot for normal conversation
-        chat_prompt = f"You are a helpful assistant. Respond conversationally to: {user_question}"
+        chat_prompt = (
+            "You are an AI assistant whose primary role is to generate SQL queries from natural language questions. "
+            "If the user's question is not related to SQL or databases, respond conversationally as a helpful assistant."
+            f"\n\nUser question: {user_question}"
+        )
         chat_response = llm.invoke([{"role": "user", "content": chat_prompt}]).content.strip()
         return {"chat_message": chat_response}
 
     return sql_query
 
-def generate_sql_query2(natural_language_query: str) -> str:
-    """Generate SQL query from natural language using LangChain."""
-    engine = get_db_engine()
-    db = SQLDatabase(engine)
 
-    # You can swap ChatOpenAI with LLaMA wrapper
-    llm = ChatOpenAI(model="gpt-4o-mini", temperature=0, api_key=settings.OPENAI_API_KEY)
-
-    sql_chain = create_sql_query_chain(llm, db)
-
-    sql_query = sql_chain.invoke({"question": natural_language_query})
-    return sql_query
 
 

backend/app/services/utility.py

@@ -1,11 +1,10 @@
-import dropbox
+# import dropbox
 import openai
 from config.settings import settings
 from langchain_openai import ChatOpenAI
 import sqlparse
 import re
 import os
-import tempfile
 class UtilityClass:
     """
     Classifies user input as 'sql' (SQL/data context) or 'chat' (normal conversation) using an LLM.
@@ -52,35 +51,12 @@ class UtilityClass:
         heading = llm.invoke([{"role": "user", "content": heading_prompt}]).content.strip()
         return {"heading": heading, "records": rows}
     
-    @staticmethod
-    def is_valid_sql_query33(sql: str) -> bool:
-        """
-        Returns True if the string is a valid SQL statement (not just a keyword in text).
-        Uses sqlparse to check for a valid statement structure.
-        Accepts queries starting with 'sql ' followed by a valid SQL statement.
-        """
-        if not sql or not isinstance(sql, str):
-            return False
-        sql_strip = sql.strip()
-        # Remove leading 'sql' if present
-        if sql_strip.lower().startswith('sql'):
-            sql_strip = sql_strip[4:].lstrip()
-        parsed = sqlparse.parse(sql_strip)
-        if not parsed or not parsed[0].tokens:
-            return False
-        stmt = parsed[0]
-        first_token = stmt.token_first(skip_cm=True, skip_ws=True)
-        if first_token is None:
-            return False
-        # Accept only if the first token is a SQL keyword and there is more than one token
-        return first_token.ttype in sqlparse.tokens.Keyword.DML and len(stmt.tokens) > 1
-    
     @staticmethod
     def is_valid_sql_query(text: str):
         if not text or not text.strip():
             return False
 
-        # Remove markdown formatting like ```sql ... ``` 
+        # Remove markdown formatting like ```sql ... ```
         text = re.sub(r"```sql|```", "", text, flags=re.IGNORECASE).strip()
 
         # Collapse multiple spaces & line breaks into a single space
@@ -91,10 +67,7 @@ class UtilityClass:
             r"^SELECT\s+.+\s+FROM\s+.+",          # SELECT ... FROM ...
             r"^INSERT\s+INTO\s+.+\s+VALUES\s*\(", # INSERT INTO ... VALUES (...)
             r"^UPDATE\s+.+\s+SET\s+.+",           # UPDATE ... SET ...
-            r"^DELETE\s+FROM\s+.+",               # DELETE FROM ...
             r"^CREATE\s+(TABLE|DATABASE)\s+.+",   # CREATE TABLE/DATABASE ...
-            r"^DROP\s+(TABLE|DATABASE)\s+.+",     # DROP TABLE/DATABASE ...
-            r"^ALTER\s+TABLE\s+.+",               # ALTER TABLE ...
             r"^WITH\s+.+\s+AS\s*\(.+\)"           # WITH ... AS (...)
         ]
 
@@ -106,23 +79,81 @@ class UtilityClass:
         return False
 
     @staticmethod
-    def download_sqlite_db_from_dropbox(dropbox_token: str, dropbox_db_path: str, local_db_path: str = None) -> str:
+    def get_health_status() -> dict:
         """
-        Downloads a SQLite DB file from Dropbox and returns the local file path.
-        :param dropbox_token: Dropbox API access token.
-        :param dropbox_db_path: Path to the DB file in Dropbox (e.g., '/folder/mydb.db').
-        :param local_db_path: Local path to save the file. If None, saves to './downloaded_db.db'.
-        :return: Local file path of the downloaded DB.
+        Get the health status of the application and its dependencies.
+        
+        Returns:
+            dict: Health status information including API status, database connectivity, and timestamp
         """
-        if local_db_path is None: 
-            print("Is /tmp writable?", os.access('/tmp', os.W_OK))           
-            print("Is /mnt writable?", os.access('/mnt', os.W_OK))           
-            print("Is /space writable?", os.access('/space', os.W_OK))           
-            temp_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "space")
-            os.makedirs(temp_dir, exist_ok=True)
-            local_db_path = os.path.join(temp_dir, "downloaded_db.db")
-        dbx = dropbox.Dropbox(dropbox_token)
-        with open(local_db_path, 'wb') as f:
-            metadata, res = dbx.files_download(path=dropbox_db_path)
-            f.write(res.content)
-        return local_db_path
+        import time
+        from datetime import datetime
+        from app.db.db_connector import get_connection
+        
+        health_data = {
+            "status": "healthy",
+            "timestamp": datetime.utcnow().isoformat(),
+            "version": "1.0.0",
+            "checks": {}
+        }
+        
+        # Check database connectivity
+        try:
+            with get_connection() as conn:
+                cursor = conn.cursor()
+                cursor.execute("SELECT 1")
+                result = cursor.fetchone()
+                if result:
+                    health_data["checks"]["database"] = {
+                        "status": "healthy",
+                        "message": "Database connection successful"
+                    }
+                else:
+                    health_data["checks"]["database"] = {
+                        "status": "unhealthy",
+                        "message": "Database query failed"
+                    }
+                    health_data["status"] = "unhealthy"
+        except Exception as e:
+            health_data["checks"]["database"] = {
+                "status": "unhealthy",
+                "message": f"Database connection failed: {str(e)}"
+            }
+            health_data["status"] = "unhealthy"
+        
+        # Check OpenAI API (if configured)
+        try:
+            if hasattr(settings, 'OPENAI_API_KEY') and settings.OPENAI_API_KEY:
+                health_data["checks"]["openai_api"] = {
+                    "status": "configured",
+                    "message": "OpenAI API key is configured"
+                }
+            else:
+                health_data["checks"]["openai_api"] = {
+                    "status": "warning",
+                    "message": "OpenAI API key not configured"
+                }
+        except Exception as e:
+            health_data["checks"]["openai_api"] = {
+                "status": "error", 
+                "message": f"OpenAI API check failed: {str(e)}"
+            }
+        
+        return health_data
+
+    # @staticmethod
+    # def download_sqlite_db_from_dropbox(dropbox_token: str, dropbox_db_path: str, local_db_path: str = None) -> str:
+    #     """
+    #     Downloads a SQLite DB file from Dropbox and returns the local file path.
+    #     :param dropbox_token: Dropbox API access token.
+    #     :param dropbox_db_path: Path to the DB file in Dropbox (e.g., '/folder/mydb.db').
+    #     :param local_db_path: Local path to save the file. If None, saves to './downloaded_db.db'.
+    #     :return: Local file path of the downloaded DB.
+    #     """
+    #     if local_db_path is None:
+    #         local_db_path = os.path.join(os.getcwd(), "downloaded_db.db")
+    #     dbx = dropbox.Dropbox(dropbox_token)
+    #     with open(local_db_path, 'wb') as f:
+    #         metadata, res = dbx.files_download(path=dropbox_db_path)
+    #         f.write(res.content)
+    #     return local_db_path

backend/requirements.txt

@@ -12,4 +12,8 @@ faiss-cpu          # optional for embeddings
 pandas
 matplotlib
 python-dotenv
-dropbox
+pytest==8.4.1
+pytest-cov==6.2.1
+pytest-mock==3.14.1
+coverage==7.10.6
+# dropbox