Update app.py

app.py

@@ -15,6 +15,7 @@ import json
 import time
 import random
 import sqlite3
+import threading
 from dataclasses import dataclass, asdict
 from datetime import datetime, timezone
 from typing import List, Dict, Any, Tuple, Optional
@@ -45,7 +46,7 @@ except Exception:
 DB_DIR = "/data" if os.path.exists("/data") else "."
 DB_PATH = os.path.join(DB_DIR, "sql_trainer_dynamic.db")
 EXPORT_DIR = "."
-ADMIN_KEY = os.getenv("OPENAI_API_KEY", "demo")
+ADMIN_KEY = os.getenv("ADMIN_KEY", "demo")
 RANDOM_SEED = int(os.getenv("RANDOM_SEED", "7"))
 random.seed(RANDOM_SEED)
 SYS_RAND = random.SystemRandom()
@@ -65,10 +66,11 @@ def _to_pil(fig) -> Image.Image:
 
 def draw_dynamic_erd(schema: Dict[str, Any]) -> Image.Image:
     """
+    Draw a simple ERD for the current randomized schema.
     schema = {
       "domain": "bookstore",
       "tables": [
-          {"name":"authors","columns":[{"name":"author_id","type":"INTEGER",...}, ...],
+          {"name":"authors","columns":[{"name":"author_id","type":"INTEGER"}, ...],
            "pk":["author_id"], "fks":[{"columns":["author_id"],"ref_table":"...","ref_columns":["..."]}],
            "rows":[{...}, {...}]}
       ]
@@ -111,46 +113,56 @@ def draw_dynamic_erd(schema: Dict[str, Any]) -> Image.Image:
     ax.text(0.5, 0.06, f"Domain: {schema.get('domain','unknown')}", fontsize=9, ha="center")
     return _to_pil(fig)
 
-# -------------------- SQLite helpers --------------------
+# -------------------- SQLite connection + locking --------------------
+DB_LOCK = threading.RLock()
+
 def connect_db():
-    con = sqlite3.connect(DB_PATH)
+    """
+    Single shared connection that is allowed to be used across threads.
+    All operations (reads + writes) are serialized via DB_LOCK.
+    WAL mode improves read concurrency.
+    """
+    con = sqlite3.connect(DB_PATH, check_same_thread=False)
+    con.execute("PRAGMA journal_mode=WAL;")
+    con.execute("PRAGMA synchronous=NORMAL;")
     con.execute("PRAGMA foreign_keys = ON;")
     return con
 
 CONN = connect_db()
 
 def init_progress_tables(con: sqlite3.Connection):
-    cur = con.cursor()
-    cur.execute("""
-        CREATE TABLE IF NOT EXISTS users (
-            user_id TEXT PRIMARY KEY,
-            name TEXT,
-            created_at TEXT
-        )
-    """)
-    cur.execute("""
-        CREATE TABLE IF NOT EXISTS attempts (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            user_id TEXT,
-            question_id TEXT,
-            category TEXT,
-            correct INTEGER,
-            sql_text TEXT,
-            timestamp TEXT,
-            time_taken REAL,
-            difficulty INTEGER,
-            source TEXT,
-            notes TEXT
-        )
-    """)
-    cur.execute("""
-        CREATE TABLE IF NOT EXISTS session_meta (
-            id INTEGER PRIMARY KEY CHECK (id=1),
-            domain TEXT,
-            schema_json TEXT
-        )
-    """)
-    con.commit()
+    with DB_LOCK:
+        cur = con.cursor()
+        cur.execute("""
+            CREATE TABLE IF NOT EXISTS users (
+                user_id TEXT PRIMARY KEY,
+                name TEXT,
+                created_at TEXT
+            )
+        """)
+        cur.execute("""
+            CREATE TABLE IF NOT EXISTS attempts (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                user_id TEXT,
+                question_id TEXT,
+                category TEXT,
+                correct INTEGER,
+                sql_text TEXT,
+                timestamp TEXT,
+                time_taken REAL,
+                difficulty INTEGER,
+                source TEXT,
+                notes TEXT
+            )
+        """)
+        cur.execute("""
+            CREATE TABLE IF NOT EXISTS session_meta (
+                id INTEGER PRIMARY KEY CHECK (id=1),
+                domain TEXT,
+                schema_json TEXT
+            )
+        """)
+        con.commit()
 
 init_progress_tables(CONN)
 
@@ -411,58 +423,60 @@ def llm_generate_domain_and_questions() -> Optional[Dict[str,Any]]:
 
 # -------------------- Schema install & question handling --------------------
 def drop_existing_domain_tables(con: sqlite3.Connection, keep_internal=True):
-    cur = con.cursor()
-    cur.execute("SELECT name, type FROM sqlite_master WHERE type IN ('table','view')")
-    items = cur.fetchall()
-    for name, typ in items:
-        if keep_internal and name in ("users","attempts","session_meta"):
-            continue
-        try:
-            cur.execute(f"DROP {typ.upper()} IF EXISTS {name}")
-        except Exception:
-            pass
-    con.commit()
+    with DB_LOCK:
+        cur = con.cursor()
+        cur.execute("SELECT name, type FROM sqlite_master WHERE type IN ('table','view')")
+        items = cur.fetchall()
+        for name, typ in items:
+            if keep_internal and name in ("users","attempts","session_meta"):
+                continue
+            try:
+                cur.execute(f"DROP {typ.upper()} IF EXISTS {name}")
+            except Exception:
+                pass
+        con.commit()
 
 def install_schema(con: sqlite3.Connection, schema: Dict[str,Any]):
     drop_existing_domain_tables(con, keep_internal=True)
-    cur = con.cursor()
-    # Create tables first
-    for t in schema.get("tables", []):
-        cols_sql = []
-        pk = t.get("pk", [])
-        for c in t.get("columns", []):
-            cname = c["name"]
-            ctype = c.get("type","TEXT")
-            cols_sql.append(f"{cname} {ctype}")
-        if pk:
-            cols_sql.append(f"PRIMARY KEY ({', '.join(pk)})")
-        create_sql = f"CREATE TABLE {t['name']} ({', '.join(cols_sql)})"
-        cur.execute(create_sql)
-    # Insert rows
-    for t in schema.get("tables", []):
-        if not t.get("rows"):
-            continue
-        cols = [c["name"] for c in t.get("columns", [])]
-        qmarks = ",".join(["?"]*len(cols))
-        insert_sql = f"INSERT INTO {t['name']} ({', '.join(cols)}) VALUES ({qmarks})"
-        # rows can be objects or arrays
-        for r in t["rows"]:
-            if isinstance(r, dict):
-                vals = [r.get(col, None) for col in cols]
-            elif isinstance(r, list) or isinstance(r, tuple):
-                vals = list(r) + [None]*(len(cols)-len(r))
-                vals = vals[:len(cols)]
-            else:
+    with DB_LOCK:
+        cur = con.cursor()
+        # Create tables first
+        for t in schema.get("tables", []):
+            cols_sql = []
+            pk = t.get("pk", [])
+            for c in t.get("columns", []):
+                cname = c["name"]
+                ctype = c.get("type","TEXT")
+                cols_sql.append(f"{cname} {ctype}")
+            if pk:
+                cols_sql.append(f"PRIMARY KEY ({', '.join(pk)})")
+            create_sql = f"CREATE TABLE {t['name']} ({', '.join(cols_sql)})"
+            cur.execute(create_sql)
+        # Insert rows
+        for t in schema.get("tables", []):
+            if not t.get("rows"):
                 continue
-            cur.execute(insert_sql, vals)
-    con.commit()
-    # Persist schema JSON
-    cur.execute("INSERT OR REPLACE INTO session_meta(id, domain, schema_json) VALUES (1, ?, ?)",
-                (schema.get("domain","unknown"), json.dumps(schema)))
-    con.commit()
+            cols = [c["name"] for c in t.get("columns", [])]
+            qmarks = ",".join(["?"]*len(cols))
+            insert_sql = f"INSERT INTO {t['name']} ({', '.join(cols)}) VALUES ({qmarks})"
+            for r in t["rows"]:
+                if isinstance(r, dict):
+                    vals = [r.get(col, None) for col in cols]
+                elif isinstance(r, (list, tuple)):
+                    vals = list(r) + [None]*(len(cols)-len(r))
+                    vals = vals[:len(cols)]
+                else:
+                    continue
+                cur.execute(insert_sql, vals)
+        con.commit()
+        # Persist schema JSON
+        cur.execute("INSERT OR REPLACE INTO session_meta(id, domain, schema_json) VALUES (1, ?, ?)",
+                    (schema.get("domain","unknown"), json.dumps(schema)))
+        con.commit()
 
 def run_df(con: sqlite3.Connection, sql: str) -> pd.DataFrame:
-    return pd.read_sql_query(sql, con)
+    with DB_LOCK:
+        return pd.read_sql_query(sql, con)
 
 def rewrite_select_into(sql: str) -> Tuple[str, Optional[str]]:
     s = sql.strip().strip(";")
@@ -491,19 +505,20 @@ def detect_cartesian(con: sqlite3.Connection, sql: str, df_result: pd.DataFrame)
     missing_on = (" join " in low) and (" on " not in low) and (" using " not in low) and (" natural " not in low)
     if comma_from or missing_on:
         try:
-            cur = con.cursor()
-            if comma_from:
-                t1, t2 = comma_from.groups()
-            else:
-                m = re.search(r"\bfrom\b\s+([a-z_]\w*)", low)
-                j = re.search(r"\bjoin\b\s+([a-z_]\w*)", low)
-                if not m or not j:
-                    return "Possible cartesian product: no join condition detected."
-                t1, t2 = m.group(1), j.group(1)
-            cur.execute(f"SELECT COUNT(*) FROM {t1}")
-            n1 = cur.fetchone()[0]
-            cur.execute(f"SELECT COUNT(*) FROM {t2}")
-            n2 = cur.fetchone()[0]
+            with DB_LOCK:
+                cur = con.cursor()
+                if comma_from:
+                    t1, t2 = comma_from.groups()
+                else:
+                    m = re.search(r"\bfrom\b\s+([a-z_]\w*)", low)
+                    j = re.search(r"\bjoin\b\s+([a-z_]\w*)", low)
+                    if not m or not j:
+                        return "Possible cartesian product: no join condition detected."
+                    t1, t2 = m.group(1), j.group(1)
+                cur.execute(f"SELECT COUNT(*) FROM {t1}")
+                n1 = cur.fetchone()[0]
+                cur.execute(f"SELECT COUNT(*) FROM {t2}")
+                n2 = cur.fetchone()[0]
             prod = n1 * n2
             if len(df_result) == prod and prod > 0:
                 return f"Result row count equals {n1}×{n2}={prod}. Likely cartesian product (missing join)."
@@ -580,14 +595,15 @@ CURRENT_SCHEMA, CURRENT_QS = install_new_domain()
 
 # -------------------- Progress + mastery --------------------
 def upsert_user(con: sqlite3.Connection, user_id: str, name: str):
-    cur = con.cursor()
-    cur.execute("SELECT user_id FROM users WHERE user_id = ?", (user_id,))
-    if cur.fetchone() is None:
-        cur.execute("INSERT INTO users (user_id, name, created_at) VALUES (?, ?, ?)",
-                    (user_id, name, datetime.now(timezone.utc).isoformat()))
-    else:
-        cur.execute("UPDATE users SET name=? WHERE user_id=?", (name, user_id))
-    con.commit()
+    with DB_LOCK:
+        cur = con.cursor()
+        cur.execute("SELECT user_id FROM users WHERE user_id = ?", (user_id,))
+        if cur.fetchone() is None:
+            cur.execute("INSERT INTO users (user_id, name, created_at) VALUES (?, ?, ?)",
+                        (user_id, name, datetime.now(timezone.utc).isoformat()))
+        else:
+            cur.execute("UPDATE users SET name=? WHERE user_id=?", (name, user_id))
+        con.commit()
 
 CATEGORIES_ORDER = [
     "SELECT *", "SELECT columns", "WHERE", "Aliases",
@@ -605,7 +621,8 @@ def topic_stats(df_attempts: pd.DataFrame) -> pd.DataFrame:
     return pd.DataFrame(rows)
 
 def fetch_attempts(con: sqlite3.Connection, user_id: str) -> pd.DataFrame:
-    return pd.read_sql_query("SELECT * FROM attempts WHERE user_id=? ORDER BY id DESC", con, params=(user_id,))
+    with DB_LOCK:
+        return pd.read_sql_query("SELECT * FROM attempts WHERE user_id=? ORDER BY id DESC", con, params=(user_id,))
 
 def pick_next_question(user_id: str) -> Dict[str,Any]:
     df = fetch_attempts(CONN, user_id)
@@ -637,30 +654,31 @@ def exec_student_sql(sql_text: str) -> Tuple[Optional[pd.DataFrame], Optional[st
             warn = detect_cartesian(CONN, sql_rew, df)
             return df, None, warn, note
         else:
-            cur = CONN.cursor()
-            cur.execute(sql_rew)
-            CONN.commit()
-            # Preview newly created objects
-            if low.startswith("create view"):
-                m = re.match(r"(?is)^\s*create\s+view\s+(if\s+not\s+exists\s+)?([a-z_]\w*)\s+as\s+(select.*)$", low)
-                name = m.group(2) if m else None
-                if name:
-                    try:
-                        df = run_df(CONN, f"SELECT * FROM {name}")
-                        return df, None, None, note
-                    except Exception:
-                        return None, "View created but could not be queried.", None, note
-            if low.startswith("create table"):
-                tbl = created_tbl
-                if not tbl:
-                    m = re.match(r"(?is)^\s*create\s+table\s+(if\s+not\s+exists\s+)?([a-z_]\w*)\s+as\s+select.*$", low)
-                    tbl = m.group(2) if m else None
-                if tbl:
-                    try:
-                        df = run_df(CONN, f"SELECT * FROM {tbl}")
-                        return df, None, None, note
-                    except Exception:
-                        return None, "Table created but could not be queried.", None, note
+            with DB_LOCK:
+                cur = CONN.cursor()
+                cur.execute(sql_rew)
+                CONN.commit()
+                # Preview newly created objects
+                if low.startswith("create view"):
+                    m = re.match(r"(?is)^\s*create\s+view\s+(if\s+not\s+exists\s+)?([a-z_]\w*)\s+as\s+(select.*)$", low)
+                    name = m.group(2) if m else None
+                    if name:
+                        try:
+                            df = pd.read_sql_query(f"SELECT * FROM {name}", CONN)
+                            return df, None, None, note
+                        except Exception:
+                            return None, "View created but could not be queried.", None, note
+                if low.startswith("create table"):
+                    tbl = created_tbl
+                    if not tbl:
+                        m = re.match(r"(?is)^\s*create\s+table\s+(if\s+not\s+exists\s+)?([a-z_]\w*)\s+as\s+select.*$", low)
+                        tbl = m.group(2) if m else None
+                    if tbl:
+                        try:
+                            df = pd.read_sql_query(f"SELECT * FROM {tbl}", CONN)
+                            return df, None, None, note
+                        except Exception:
+                            return None, "Table created but could not be queried.", None, note
             return pd.DataFrame(), None, None, note
     except Exception as e:
         # Tailored messages
@@ -689,19 +707,21 @@ def answer_df(answer_sql: List[str]) -> Optional[pd.DataFrame]:
                 # temp preview
                 m = re.match(r"(?is)^\s*create\s+view\s+(if\s+not\s+exists\s+)?([a-z_]\w*)\s+as\s+select.*$", low)
                 view_name = m.group(2) if m else "vw_tmp"
-                cur = CONN.cursor()
-                cur.execute(f"DROP VIEW IF EXISTS {view_name}")
-                cur.execute(sql)
-                CONN.commit()
+                with DB_LOCK:
+                    cur = CONN.cursor()
+                    cur.execute(f"DROP VIEW IF EXISTS {view_name}")
+                    cur.execute(sql)
+                    CONN.commit()
                 return run_df(CONN, f"SELECT * FROM {view_name}")
             if low.startswith("create table"):
                 m = re.match(r"(?is)^\s*create\s+table\s+(if\s+not\s+exists\s+)?([a-z_]\w*)\s+as\s+select.*$", low)
                 tbl = m.group(2) if m else None
-                cur = CONN.cursor()
-                if tbl:
-                    cur.execute(f"DROP TABLE IF EXISTS {tbl}")
-                cur.execute(sql)
-                CONN.commit()
+                with DB_LOCK:
+                    cur = CONN.cursor()
+                    if tbl:
+                        cur.execute(f"DROP TABLE IF EXISTS {tbl}")
+                    cur.execute(sql)
+                    CONN.commit()
                 if tbl:
                     return run_df(CONN, f"SELECT * FROM {tbl}")
         except Exception:
@@ -710,7 +730,7 @@ def answer_df(answer_sql: List[str]) -> Optional[pd.DataFrame]:
 
 def validate_answer(q: Dict[str,Any], student_sql: str, df_student: Optional[pd.DataFrame]) -> Tuple[bool, str]:
     df_expected = answer_df(q["answer_sql"])
-    # If we can't build a canonical DF (e.g., DDL side effect), we accept any successful execution as correct
+    # If we can't build a canonical DF (e.g., DDL side effect), accept any successful execution as correct
     if df_expected is None:
         return (df_student is not None), f"**Explanation:** Your statement executed successfully for this task."
     if df_student is None:
@@ -719,13 +739,14 @@ def validate_answer(q: Dict[str,Any], student_sql: str, df_student: Optional[pd.
 
 def log_attempt(user_id: str, qid: str, category: str, correct: bool, sql_text: str,
                 time_taken: float, difficulty: int, source: str, notes: str):
-    cur = CONN.cursor()
-    cur.execute("""
-        INSERT INTO attempts (user_id, question_id, category, correct, sql_text, timestamp, time_taken, difficulty, source, notes)
-        VALUES (?,?,?,?,?,?,?,?,?,?)
-    """, (user_id, qid, category, int(correct), sql_text, datetime.now(timezone.utc).isoformat(),
-          time_taken, difficulty, source, notes))
-    CONN.commit()
+    with DB_LOCK:
+        cur = CONN.cursor()
+        cur.execute("""
+            INSERT INTO attempts (user_id, question_id, category, correct, sql_text, timestamp, time_taken, difficulty, source, notes)
+            VALUES (?,?,?,?,?,?,?,?,?,?)
+        """, (user_id, qid, category, int(correct), sql_text, datetime.now(timezone.utc).isoformat(),
+              time_taken, difficulty, source, notes))
+        CONN.commit()
 
 # -------------------- UI callbacks --------------------
 def start_session(name: str, session: dict):