feat: implement advanced hybrid rule-based + LLM NLP-to-SQL engine

database.py

@@ -163,89 +163,480 @@ def validate_sql(sql):
     return upper.startswith("SELECT")
 
 
+# ================= NLP TO SQL ================= #
+
+# ================= HELPER FUNCTIONS ================= #
+
+def clean_sql(sql_str):
+    """Clean and normalize SQL output"""
+    
+    sql = sql_str.strip()
+    sql = sql.replace("```sql", "").replace("```", "")
+    sql = sql.strip()
+    
+    if not sql.endswith(";"):
+        sql += ";"
+    
+    return sql
+
+
 # ================= NLP TO SQL ================= #
 
 def ask_llm(user_query):
+    """
+    Advanced NLP-to-SQL Generator
+    Hybrid Rule-Based + LLM Approach for Vehicle Intelligence
+    """
+
+    import re
 
     if client is None:
-        return "SELECT 1;"  # Dummy query if no client
+        return "SELECT * FROM vehicle_logs LIMIT 10;"
 
-    prompt = f"""
-You are an expert PostgreSQL SQL generator.
+    q = user_query.lower().strip()
 
-ONLY use this table:
+    # =========================================================
+    # RULE-BASED FAST PATHS (VERY IMPORTANT)
+    # =========================================================
 
+    # ----- PLATE TRACKING (Generic) -----
+    plate_match = re.search(r'([A-Z]{2}\d{1,2}[A-Z]{1,3}\d{3,4})', user_query.upper())
+
+    if plate_match:
+        plate = plate_match.group(1)
+
+        if any(k in q for k in ["location", "route", "travel", "movement", "where", "pass", "track", "history"]):
+            return clean_sql(f"""
+            SELECT
+                timestamp,
+                plate,
+                state,
+                vehicle_type,
+                location,
+                camera_id
+            FROM vehicle_logs
+            WHERE plate = '{plate}'
+            ORDER BY timestamp DESC
+            LIMIT 100;
+            """)
+
+        if any(k in q for k in ["count", "how many", "detections"]):
+            return clean_sql(f"""
+            SELECT
+                plate,
+                COUNT(*) as detection_count,
+                COUNT(DISTINCT location) as unique_locations,
+                COUNT(DISTINCT date) as days_detected
+            FROM vehicle_logs
+            WHERE plate = '{plate}'
+            GROUP BY plate;
+            """)
+
+        return clean_sql(f"""
+        SELECT *
+        FROM vehicle_logs
+        WHERE plate = '{plate}'
+        ORDER BY timestamp DESC
+        LIMIT 50;
+        """)
+
+    # ----- STATE SEARCH (Generic) -----
+    states_map = {
+        "tn": "TN", "tamil": "TN", "tamil nadu": "TN",
+        "ka": "KA", "karnataka": "KA",
+        "kl": "KL", "kerala": "KL",
+        "ap": "AP", "andhra": "AP",
+        "ts": "TS", "telangana": "TS",
+        "mh": "MH", "maharashtra": "MH",
+        "dl": "DL", "delhi": "DL",
+        "gj": "GJ", "gujarat": "GJ",
+        "rj": "RJ", "rajasthan": "RJ",
+        "up": "UP", "uttar": "UP",
+        "wb": "WB", "bengal": "WB",
+        "hr": "HR", "haryana": "HR",
+        "pb": "PB", "punjab": "PB"
+    }
+
+    for key, state_code in states_map.items():
+        if key in q:
+            if "count" in q:
+                return clean_sql(f"""
+                SELECT
+                    state,
+                    COUNT(*) as total_vehicles,
+                    COUNT(DISTINCT plate) as unique_plates,
+                    COUNT(DISTINCT location) as locations_active
+                FROM vehicle_logs
+                WHERE state = '{state_code}'
+                GROUP BY state;
+                """)
+            
+            if "distribution" in q or "breakdown" in q:
+                return clean_sql(f"""
+                SELECT
+                    vehicle_type,
+                    COUNT(*) as count
+                FROM vehicle_logs
+                WHERE state = '{state_code}'
+                GROUP BY vehicle_type
+                ORDER BY count DESC;
+                """)
+
+            return clean_sql(f"""
+            SELECT *
+            FROM vehicle_logs
+            WHERE state = '{state_code}'
+            ORDER BY timestamp DESC
+            LIMIT 50;
+            """)
+
+    # ----- LOCATION SEARCH (Generic) -----
+    locations = [
+        "adyar", "guindy", "velachery", "besant", "thiruvanmiyur",
+        "tnagar", "mylapore", "annanagar", "koyambedu", "nungambakkam",
+        "kotturpuram", "porur", "indiranagar", "whitefield", "koramangala",
+        "bangalore", "hyderabad", "trivandrum", "kochi", "pune", "mumbai"
+    ]
+
+    for loc in locations:
+        if loc in q:
+            if "count" in q:
+                return clean_sql(f"""
+                SELECT
+                    location,
+                    COUNT(*) as detection_count,
+                    COUNT(DISTINCT plate) as unique_vehicles
+                FROM vehicle_logs
+                WHERE LOWER(location) LIKE '%{loc}%'
+                GROUP BY location
+                ORDER BY detection_count DESC;
+                """)
+
+            return clean_sql(f"""
+            SELECT
+                timestamp,
+                plate,
+                state,
+                vehicle_type,
+                location
+            FROM vehicle_logs
+            WHERE LOWER(location) LIKE '%{loc}%'
+            ORDER BY timestamp DESC
+            LIMIT 100;
+            """)
+
+    # ----- VEHICLE TYPE SEARCH (Generic) -----
+    vehicle_types = {
+        "suv": "SUV", "sedan": "Sedan", "hatchback": "Hatchback",
+        "truck": "Truck", "bus": "Bus", "bike": "Bike",
+        "motorcycle": "Bike", "auto": "Auto", "taxi": "Taxi",
+        "car": "Car", "van": "Van", "tempo": "Tempo"
+    }
+
+    for vtype_key, vtype_val in vehicle_types.items():
+        if vtype_key in q:
+            if "count" in q:
+                return clean_sql(f"""
+                SELECT
+                    vehicle_type,
+                    COUNT(*) as count,
+                    ROUND(AVG(vehicle_conf), 2) as avg_confidence
+                FROM vehicle_logs
+                WHERE LOWER(vehicle_type) LIKE '%{vtype_val.lower()}%'
+                GROUP BY vehicle_type;
+                """)
+
+            return clean_sql(f"""
+            SELECT *
+            FROM vehicle_logs
+            WHERE LOWER(vehicle_type) LIKE '%{vtype_val.lower()}%'
+            ORDER BY timestamp DESC
+            LIMIT 50;
+            """)
+
+    # ----- DATE SEARCH -----
+    date_match = re.search(r'(\d{4}-\d{2}-\d{2})', q)
+
+    if date_match:
+        date_value = date_match.group(1)
+
+        if "count" in q:
+            return clean_sql(f"""
+            SELECT
+                date,
+                COUNT(*) as total_detections,
+                COUNT(DISTINCT plate) as unique_vehicles,
+                COUNT(DISTINCT location) as unique_locations
+            FROM vehicle_logs
+            WHERE date = '{date_value}'
+            GROUP BY date;
+            """)
+
+        return clean_sql(f"""
+        SELECT *
+        FROM vehicle_logs
+        WHERE date = '{date_value}'
+        ORDER BY timestamp DESC
+        LIMIT 100;
+        """)
+
+    # ----- TIME-BASED QUERIES -----
+    if "morning" in q:
+        return clean_sql("""
+        SELECT *
+        FROM vehicle_logs
+        WHERE hour BETWEEN 6 AND 11
+        ORDER BY timestamp DESC
+        LIMIT 100;
+        """)
+
+    if "afternoon" in q:
+        return clean_sql("""
+        SELECT *
+        FROM vehicle_logs
+        WHERE hour BETWEEN 12 AND 17
+        ORDER BY timestamp DESC
+        LIMIT 100;
+        """)
+
+    if "evening" in q or "night" in q:
+        return clean_sql("""
+        SELECT *
+        FROM vehicle_logs
+        WHERE hour BETWEEN 18 AND 23 OR hour BETWEEN 0 AND 5
+        ORDER BY timestamp DESC
+        LIMIT 100;
+        """)
+
+    if "busiest hour" in q or "peak hour" in q:
+        return clean_sql("""
+        SELECT
+            hour,
+            COUNT(*) as traffic_volume
+        FROM vehicle_logs
+        GROUP BY hour
+        ORDER BY traffic_volume DESC
+        LIMIT 1;
+        """)
+
+    if "hourly traffic" in q or "traffic by hour" in q:
+        return clean_sql("""
+        SELECT
+            hour,
+            COUNT(*) as traffic_count,
+            COUNT(DISTINCT plate) as unique_vehicles
+        FROM vehicle_logs
+        GROUP BY hour
+        ORDER BY hour;
+        """)
+
+    # ----- ANALYTICS QUERIES -----
+    if "top plates" in q or "most detected" in q or "repeated plates" in q:
+        return clean_sql("""
+        SELECT
+            plate,
+            COUNT(*) as detections,
+            COUNT(DISTINCT location) as locations,
+            COUNT(DISTINCT date) as days
+        FROM vehicle_logs
+        GROUP BY plate
+        ORDER BY detections DESC
+        LIMIT 20;
+        """)
+
+    if "suspicious" in q or "high frequency" in q or "unusual" in q:
+        return clean_sql("""
+        SELECT
+            plate,
+            COUNT(*) as detection_count,
+            COUNT(DISTINCT location) as unique_locations,
+            ROUND(AVG(vehicle_conf), 2) as avg_confidence
+        FROM vehicle_logs
+        GROUP BY plate
+        HAVING COUNT(*) > 10
+        ORDER BY detection_count DESC
+        LIMIT 50;
+        """)
+
+    if "vehicle type" in q and ("count" in q or "distribution" in q or "breakdown" in q):
+        return clean_sql("""
+        SELECT
+            vehicle_type,
+            COUNT(*) as count,
+            ROUND(100.0 * COUNT(*) / (SELECT COUNT(*) FROM vehicle_logs), 2) as percentage,
+            ROUND(AVG(vehicle_conf), 2) as avg_confidence
+        FROM vehicle_logs
+        GROUP BY vehicle_type
+        ORDER BY count DESC;
+        """)
+
+    if "state distribution" in q or ("count" in q and "state" in q):
+        return clean_sql("""
+        SELECT
+            state,
+            COUNT(*) as count,
+            COUNT(DISTINCT plate) as unique_plates,
+            ROUND(100.0 * COUNT(*) / (SELECT COUNT(*) FROM vehicle_logs), 2) as percentage
+        FROM vehicle_logs
+        GROUP BY state
+        ORDER BY count DESC;
+        """)
+
+    if "latest" in q or "recent" in q or "last detection" in q:
+        return clean_sql("""
+        SELECT *
+        FROM vehicle_logs
+        ORDER BY timestamp DESC
+        LIMIT 50;
+        """)
+
+    if "total vehicle" in q or "total count" in q:
+        return clean_sql("""
+        SELECT
+            COUNT(*) as total_detections,
+            COUNT(DISTINCT plate) as unique_vehicles,
+            COUNT(DISTINCT state) as states_active,
+            COUNT(DISTINCT location) as locations_active
+        FROM vehicle_logs;
+        """)
+
+    if "camera" in q or "detection point" in q:
+        return clean_sql("""
+        SELECT
+            camera_id,
+            location,
+            COUNT(*) as detections,
+            COUNT(DISTINCT plate) as unique_vehicles
+        FROM vehicle_logs
+        WHERE camera_id IS NOT NULL
+        GROUP BY camera_id, location
+        ORDER BY detections DESC
+        LIMIT 20;
+        """)
+
+    # ----- ADVANCED COMBINATION QUERIES -----
+    if "passed through" in q or "traveled through" in q:
+        return clean_sql("""
+        SELECT
+            plate,
+            location,
+            COUNT(*) as times_detected,
+            MIN(timestamp) as first_detection,
+            MAX(timestamp) as last_detection
+        FROM vehicle_logs
+        WHERE location IS NOT NULL
+        GROUP BY plate, location
+        ORDER BY plate, MIN(timestamp) DESC
+        LIMIT 100;
+        """)
+
+    # =========================================================
+    # FALLBACK: ADVANCED LLM
+    # =========================================================
+
+    prompt = f"""
+You are an expert PostgreSQL SQL generator for vehicle intelligence.
+
+DATABASE SCHEMA:
 vehicle_logs(
-    id,
     timestamp,
     plate,
     state,
     vehicle_type,
     vehicle_conf,
+    camera_id,
+    location,
     date,
     hour,
     day
 )
 
-Rules:
-- ONLY SELECT queries
-- NEVER hallucinate columns
-- state column EXISTS
-- LIMIT 50 max
-- Output ONLY SQL
-
-Examples:
-
-Question:
-Show TN vehicles
+VALID COLUMNS:
+timestamp - Detection timestamp
+plate - License plate number
+state - State code (TN, KA, KL, AP, TS, MH, DL, GJ, RJ, UP, WB, HR, PB)
+vehicle_type - Car, SUV, Truck, Bus, Bike, Auto, Taxi, Van, etc.
+vehicle_conf - Detection confidence (0.0-1.0)
+camera_id - Camera identifier
+location - Detection location/area name
+date - Detection date (YYYY-MM-DD)
+hour - Hour of day (0-23)
+day - Day of week (Monday-Sunday)
+
+STRICT RULES:
+1. ONLY SELECT queries
+2. NEVER JOIN tables
+3. NEVER use subqueries (except COUNT aggregates)
+4. ONLY vehicle_logs table
+5. ALWAYS use LIMIT 50 or LIMIT 100
+6. NEVER use DELETE, UPDATE, DROP, ALTER, CREATE, TRUNCATE
+7. NEVER invent columns or tables
+8. Return SQL ONLY (no explanation)
+9. No markdown formatting
+10. Always end with semicolon
+
+EXAMPLES:
+
+Q: Show TN vehicles
+A: SELECT * FROM vehicle_logs WHERE state='TN' ORDER BY timestamp DESC LIMIT 50;
+
+Q: Show all vehicles from Adyar
+A: SELECT * FROM vehicle_logs WHERE LOWER(location) LIKE '%adyar%' ORDER BY timestamp DESC LIMIT 50;
+
+Q: Show suspicious vehicles
+A: SELECT plate, COUNT(*) as count FROM vehicle_logs GROUP BY plate HAVING COUNT(*) > 10 ORDER BY count DESC LIMIT 50;
+
+Q: Show vehicle type distribution
+A: SELECT vehicle_type, COUNT(*) as count FROM vehicle_logs GROUP BY vehicle_type ORDER BY count DESC;
+
+Q: Show hourly traffic
+A: SELECT hour, COUNT(*) as count FROM vehicle_logs GROUP BY hour ORDER BY hour;
+
+Q: Show latest detections
+A: SELECT * FROM vehicle_logs ORDER BY timestamp DESC LIMIT 50;
+
+Q: Track vehicle TN63AB1234
+A: SELECT * FROM vehicle_logs WHERE plate='TN63AB1234' ORDER BY timestamp DESC LIMIT 100;
+
+USER QUESTION:
+{user_query}
 
 SQL:
-SELECT * FROM vehicle_logs
-WHERE state='TN'
-ORDER BY timestamp DESC
-LIMIT 50;
-
-Question:
-Top repeated plates
+"""
 
-SQL:
-SELECT plate, COUNT(*) AS count
-FROM vehicle_logs
-GROUP BY plate
-ORDER BY count DESC
-LIMIT 10;
+    try:
+        response = client.text_generation(
+            prompt,
+            max_new_tokens=150,
+            temperature=0.05,
+            repetition_penalty=1.2
+        )
 
-Question:
-Hourly traffic
+        sql_query = clean_sql(response.strip())
 
-SQL:
-SELECT hour, COUNT(*) AS count
-FROM vehicle_logs
-GROUP BY hour
-ORDER BY hour;
+        # =====================================================
+        # EXTRA SAFETY VALIDATION
+        # =====================================================
 
-Question:
-{user_query}
+        sql_upper = sql_query.upper()
 
-SQL:
-"""
+        blocked = ["DROP", "DELETE", "UPDATE", "INSERT", "ALTER", "CREATE", "TRUNCATE", "JOIN", "UNION"]
 
-    response = client.text_generation(
-        prompt,
-        max_new_tokens=120,
-        temperature=0.1
-    )
+        for b in blocked:
+            if b in sql_upper:
+                return f"SELECT * FROM vehicle_logs LIMIT 10;"  # Fallback
 
-    sql = response.strip()
+        if "VEHICLE_LOGS" not in sql_upper:
+            return f"SELECT * FROM vehicle_logs LIMIT 10;"  # Fallback
 
-    sql = sql.replace("```sql", "")
-    sql = sql.replace("```", "")
-    sql = sql.strip()
+        if not sql_upper.startswith("SELECT"):
+            return f"SELECT * FROM vehicle_logs LIMIT 10;"  # Fallback
 
-    if ";" not in sql:
-        sql += ";"
+        return sql_query
 
-    return sql
+    except Exception as e:
+        print(f"⚠️ LLM Error: {e}")
+        return f"SELECT * FROM vehicle_logs LIMIT 10;"  # Fallback
 
 
 # ================= QUERY ================= #