Update app.py

app.py

@@ -2,8 +2,8 @@
 # - OpenAI randomizes a domain and questions (fallback dataset if unavailable).
 # - 3–4 related tables with seed rows installed in SQLite.
 # - Students practice SELECT, WHERE, JOINs (INNER/LEFT), aliases, views, CTAS/SELECT INTO.
-# - Validator now enforces columns only when the prompt asks for them; otherwise it focuses on rows.
-# - ERD shows all FK edges in light gray and dynamically HIGHLIGHTS edges implied by the student’s JOINs.
+# - Validator enforces columns only when the prompt asks; otherwise focuses on rows.
+# - ERD shows all FK edges in light gray and dynamically HIGHLIGHTS edges implied by JOINs.
 
 import os
 import re
@@ -12,7 +12,6 @@ import time
 import random
 import sqlite3
 import threading
-from dataclasses import dataclass
 from datetime import datetime, timezone
 from typing import List, Dict, Any, Tuple, Optional, Set
 
@@ -69,25 +68,21 @@ def draw_dynamic_erd(
     highlight_edges uses (src_table, dst_table) with dst_table = referenced table.
     """
     highlight_tables = set(highlight_tables or [])
-    # normalize edges so (A,B) & (B,A) match regardless of direction
     def _norm_edge(a, b): return tuple(sorted([a, b]))
     H = set(_norm_edge(*e) for e in (highlight_edges or set()))
 
     tables = schema.get("tables", [])
+    fig, ax = plt.subplots(figsize=PLOT_FIGSIZE); ax.axis("off")
     if not tables:
-        fig, ax = plt.subplots(figsize=PLOT_FIGSIZE); ax.axis("off")
         ax.text(0.5, 0.5, "No tables to diagram.", ha="center", va="center")
         return _fig_to_pil(fig)
 
-    # Layout tables horizontally
     n = len(tables)
-    fig, ax = plt.subplots(figsize=PLOT_FIGSIZE); ax.axis("off")
     margin = 0.03
     width = (1 - margin * (n + 1)) / max(n, 1)
     height = 0.70
     y = 0.20
 
-    # Build quick FK lookup: [(src_table, dst_table)]
     fk_edges = []
     for t in tables:
         for fk in t.get("fks", []) or []:
@@ -95,20 +90,16 @@ def draw_dynamic_erd(
             if dst:
                 fk_edges.append((t["name"], dst))
 
-    # Draw table boxes + columns
     boxes: Dict[str, Tuple[float,float,float,float]] = {}
     for i, t in enumerate(tables):
         tx = margin + i * (width + margin)
         boxes[t["name"]] = (tx, y, width, height)
-
-        # Highlight table border if used in current SQL
         lw = 2.0 if t["name"] in highlight_tables else 1.2
         ax.add_patch(Rectangle((tx, y), width, height, fill=False, lw=lw))
         ax.text(tx + 0.01, y + height - 0.04, t["name"], fontsize=10, ha="left", va="top", weight="bold")
 
         yy = y + height - 0.09
         pkset = set(t.get("pk", []) or [])
-        # For FK annotation by column
         fk_map: Dict[str, List[Tuple[str, str]]] = {}
         for fk in t.get("fks", []) or []:
             ref_tbl = fk.get("ref_table", "")
@@ -126,7 +117,6 @@ def draw_dynamic_erd(
             ax.text(tx + 0.016, yy, f"{nm}{tag}", fontsize=9, ha="left", va="top")
             yy -= 0.055
 
-    # Draw FK edges: light gray
     for (src, dst) in fk_edges:
         if src not in boxes or dst not in boxes: 
             continue
@@ -137,7 +127,6 @@ def draw_dynamic_erd(
                     xytext=(x1 + w1/2.0, y1),
                     arrowprops=dict(arrowstyle="->", lw=1.0, color="#cccccc"))
 
-    # Overlay highlighted edges: bold, darker
     for (src, dst) in fk_edges:
         if _norm_edge(src, dst) in H:
             (x1, y1, w1, h1) = boxes[src]
@@ -150,22 +139,15 @@ def draw_dynamic_erd(
     ax.text(0.5, 0.06, f"Domain: {schema.get('domain','unknown')}", fontsize=9, ha="center")
     return _fig_to_pil(fig)
 
-# Parse JOINs from SQL and turn them into tables/edges to highlight on ERD
 JOIN_TBL_RE = re.compile(r"\b(?:from|join)\s+([a-z_]\w*)(?:\s+(?:as\s+)?([a-z_]\w*))?", re.IGNORECASE)
 EQ_ON_RE = re.compile(r"([a-z_]\w*)\.[a-z_]\w*\s*=\s*([a-z_]\w*)\.[a-z_]\w*", re.IGNORECASE)
 USING_RE = re.compile(r"\bjoin\s+([a-z_]\w*)(?:\s+(?:as\s+)?([a-z_]\w*))?\s+using\s*\(", re.IGNORECASE)
 
 def sql_highlights(sql: str, schema: Dict[str, Any]) -> Tuple[Set[str], Set[Tuple[str, str]]]:
-    """
-    Returns (highlight_tables, highlight_edges) based on the student's SQL.
-    - Tables: any table appearing after FROM or JOIN (by name or alias).
-    - Edges: pairs inferred from `a.col = b.col` or JOIN ... USING (...).
-    """
     if not sql:
         return set(), set()
 
     low = " ".join(sql.strip().split())
-    # Alias map alias->table and list of tables in join order
     alias_to_table: Dict[str, str] = {}
     join_order: List[str] = []
 
@@ -175,7 +157,6 @@ def sql_highlights(sql: str, schema: Dict[str, Any]) -> Tuple[Set[str], Set[Tupl
         alias_to_table[alias] = table
         join_order.append(alias)
 
-    # Edges from explicit equality ON clauses
     edges: Set[Tuple[str, str]] = set()
     for a1, a2 in EQ_ON_RE.findall(low):
         t1 = alias_to_table.get(a1, a1)
@@ -183,7 +164,6 @@ def sql_highlights(sql: str, schema: Dict[str, Any]) -> Tuple[Set[str], Set[Tupl
         if t1 != t2:
             edges.add((t1, t2))
 
-    # Heuristic for USING(): connect the previous alias with the current JOIN alias
     if USING_RE.search(low) and len(join_order) >= 2:
         for i in range(1, len(join_order)):
             t_left = alias_to_table.get(join_order[i-1], join_order[i-1])
@@ -191,14 +171,10 @@ def sql_highlights(sql: str, schema: Dict[str, Any]) -> Tuple[Set[str], Set[Tupl
             if t_left != t_right:
                 edges.add((t_left, t_right))
 
-    # Highlight tables used (map aliases back to table names)
     used_tables = {alias_to_table.get(a, a) for a in join_order}
-
-    # Normalize edges to actual table names present in schema
     schema_tables = {t["name"] for t in schema.get("tables", [])}
     edges = { (a, b) for (a, b) in edges if a in schema_tables and b in schema_tables }
     used_tables = { t for t in used_tables if t in schema_tables }
-
     return used_tables, edges
 
 # -------------------- SQLite + locking --------------------
@@ -405,7 +381,6 @@ def _loose_json_parse(s: str) -> Optional[dict]:
             return None
     return None
 
-# Canonicalization of LLM output (questions & tables)
 _SQL_FENCE = re.compile(r"```sql(.*?)```", re.IGNORECASE | re.DOTALL)
 _CODE_FENCE = re.compile(r"```(.*?)```", re.DOTALL)
 
@@ -500,7 +475,7 @@ def _canon_tables(tables: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
         })
     return out
 
-def llm_generate_domain_and_questions(prev_domain: Optional[str]) -> Tuple[Optional[Dict[str,Any]], Optional[str], Optional[str], Dict[str,int]]:
+def llm_generate_domain_and_questions(prev_domain: Optional[str]):
     if not OPENAI_AVAILABLE or not os.getenv("OPENAI_API_KEY"):
         return None, "OpenAI client not available or OPENAI_API_KEY missing.", None, {"accepted_questions":0,"dropped_questions":0}
     errors = []
@@ -605,8 +580,6 @@ def install_schema_and_prepare_questions(prev_domain: Optional[str]):
     install_schema(CONN, schema)
     if not questions:
         questions = FALLBACK_QUESTIONS
-        info = {"source":"openai+fallback-questions","model":info.get("model"),
-                "error":"LLM returned 0 usable questions; used fallback bank.","accepted":0,"dropped":0}
     return schema, questions, info
 
 # -------------------- Session globals --------------------
@@ -647,7 +620,7 @@ def pick_next_question(user_id: str) -> Dict[str,Any]:
     pool = CURRENT_QS if CURRENT_QS else FALLBACK_QUESTIONS
     df = fetch_attempts(CONN, user_id)
     stats = topic_stats(df)
-    stats = stats.sort_values(by=["accuracy","attempts"], ascending=[True, True])
+    stats = stats.sort_values(by=["accuracy","attempts"], ascending=[True, True]) if not stats.empty else stats
     weakest = stats.iloc[0]["category"] if not stats.empty else CATEGORIES_ORDER[0]
     cands = [q for q in pool if str(q.get("category","")).strip() == weakest] or pool
     return dict(random.choice(cands))
@@ -699,16 +672,14 @@ def detect_cartesian(con: sqlite3.Connection, sql: str, df_result: pd.DataFrame)
             return "Possible cartesian product: no join condition detected."
     return None
 
-# Column enforcement policy — only when the prompt asks for it
 def should_enforce_columns(q: Dict[str, Any]) -> bool:
     cat = (q.get("category") or "").strip()
     if cat in ("SELECT columns", "Aggregation", "VIEW", "CTAS / SELECT INTO"):
         return True
     prompt = (q.get("prompt_md") or "").lower()
-    # Signals that the projection is specified in the prompt
-    if re.search(r"`[^`]+`", q.get("prompt_md") or ""):  # backticked names
+    if re.search(r"`[^`]+`", q.get("prompt_md") or ""):
         return True
-    if re.search(r"\((?:show|return|display)[^)]+\)", prompt):  # e.g., "(show title, price)"
+    if re.search(r"\((?:show|return|display)[^)]+\)", prompt):
         return True
     if re.search(r"\b(show|return|display|select)\b[^.]{0,100}\b(columns?|fields?|name|title|price)\b", prompt):
         return True
@@ -734,7 +705,6 @@ def results_equal_or_superset(df_student: pd.DataFrame, df_expected: pd.DataFram
     return False, None
 
 def results_equal_rowcount_only(df_student: pd.DataFrame, df_expected: pd.DataFrame) -> bool:
-    # When projection isn't specified, match on row count only.
     return df_student.shape[0] == df_expected.shape[0]
 
 def aliases_present(sql: str, required_aliases: List[str]) -> bool:
@@ -744,7 +714,7 @@ def aliases_present(sql: str, required_aliases: List[str]) -> bool:
             return False
     return True
 
-def exec_student_sql(sql_text: str) -> Tuple[Optional[pd.DataFrame], Optional[str], Optional[str], Optional[str]]:
+def exec_student_sql(sql_text: str):
     if not sql_text or not sql_text.strip():
         return None, "Enter a SQL statement.", None, None
     sql_raw = sql_text.strip().rstrip(";")
@@ -940,28 +910,15 @@ def show_hint(session: dict):
     }.get(cat, "Identify keys from the schema and join on them.")
     return gr.update(value=f"**Hint:** {hint}", visible=True)
 
-def export_progress(user_name: str):
-    slug = "-".join((user_name or "").lower().split())
-    if not slug: return None
-    user_id = slug[:64]
-    with DB_LOCK:
-        df = pd.read_sql_query("SELECT * FROM attempts WHERE user_id=? ORDER BY id DESC", CONN, params=(user_id,))
-    os.makedirs(EXPORT_DIR, exist_ok=True)
-    path = os.path.abspath(os.path.join(EXPORT_DIR, f"{user_id}_progress.csv"))
-    (pd.DataFrame([{"info":"No attempts yet."}]) if df.empty else df).to_csv(path, index=False)
-    return path
-
 def _domain_status_md():
-    if CURRENT_INFO.get("source","") in ("openai","openai+fallback-questions"):
-        note = " (LLM domain ok; used fallback questions)" if CURRENT_INFO.get("source") == "openai+fallback-questions" else ""
+    if CURRENT_INFO.get("source","openai"):
         accepted = CURRENT_INFO.get("accepted",0); dropped = CURRENT_INFO.get("dropped",0)
-        return (f"✅ **Domain via OpenAI** `{CURRENT_INFO.get('model','?')}` → **{CURRENT_SCHEMA.get('domain','?')}**{note}. "
+        return (f"✅ **Domain via OpenAI** `{CURRENT_INFO.get('model','?')}` → **{CURRENT_SCHEMA.get('domain','?')}**. "
                 f"Accepted questions: {accepted}, dropped: {dropped}.  \n"
                 f"Tables: {', '.join(t['name'] for t in CURRENT_SCHEMA.get('tables', []))}.")
     err = CURRENT_INFO.get("error",""); err_short = (err[:160] + "…") if len(err) > 160 else err
     return f"⚠️ **OpenAI randomization unavailable** → using fallback **{CURRENT_SCHEMA.get('domain','?')}**.\n\n> Reason: {err_short}"
 
-# ----- UPDATED: regenerate also refreshes tbl list and, if session active, seeds a new question + shows input
 def list_tables_for_preview():
     df = run_df(CONN, """
         SELECT name FROM sqlite_master
@@ -972,6 +929,7 @@ def list_tables_for_preview():
     """)
     return df["name"].tolist() if not df.empty else ["(no tables)"]
 
+# >>> FIX: Always reseed a question on randomize (creates a guest session if needed)
 def regenerate_domain(session: dict):
     global CURRENT_SCHEMA, CURRENT_QS, CURRENT_INFO
     prev = CURRENT_SCHEMA.get("domain") if CURRENT_SCHEMA else None
@@ -979,20 +937,33 @@ def regenerate_domain(session: dict):
     erd = draw_dynamic_erd(CURRENT_SCHEMA)
     status = _domain_status_md()
 
-    # Refresh the preview dropdown
-    choices = list_tables_for_preview()
-    dd_update = gr.update(choices=choices, value=(choices[0] if choices and choices[0]!="(no tables)" else None))
-
-    # If a session is active, show the first question immediately for the new domain
-    prompt_update = gr.update()
-    input_update  = gr.update()
-    if session and session.get("user_id"):
-        q = pick_next_question(session["user_id"])
-        session["qid"] = q["id"]; session["q"] = q; session["start_ts"] = time.time()
-        prompt_update = gr.update(value=f"**Question {q['id']}**\n\n{q['prompt_md']}", visible=True)
-        input_update  = gr.update(value="", visible=True)
-
-    return status, erd, prompt_update, input_update, dd_update, session
+    # Ensure a session (guest if needed)
+    if not session or not session.get("user_id"):
+        user_id = f"guest-{int(time.time())}"
+        upsert_user(CONN, user_id, "Guest")
+        session = {"user_id": user_id, "name": "Guest", "qid": None, "start_ts": time.time(), "q": None}
+
+    # Seed next question for this session
+    q = pick_next_question(session["user_id"])
+    session.update({"qid": q["id"], "q": q, "start_ts": time.time()})
+
+    # Fresh mastery and cleared result preview
+    stats = topic_stats(fetch_attempts(CONN, session["user_id"]))
+    empty_df = pd.DataFrame()
+
+    # Refresh dropdown
+    dd_update = gr.update(choices=list_tables_for_preview(), value=None)
+
+    return (
+        gr.update(value=status, visible=True),                 # regen_fb
+        erd,                                                   # er_image
+        gr.update(value=f"**Question {q['id']}**\n\n{q['prompt_md']}", visible=True),  # prompt_md
+        gr.update(value="", visible=True),                     # sql_input
+        dd_update,                                             # tbl_dd
+        stats,                                                 # mastery_df
+        empty_df,                                              # result_df
+        session                                                # session_state
+    )
 
 def preview_table(tbl: str):
     try:
@@ -1059,7 +1030,6 @@ with gr.Blocks(title="Adaptive SQL Trainer — Randomized Domains") as demo:
             gr.Markdown("### Result Preview")
             result_df = gr.Dataframe(value=pd.DataFrame(), interactive=False)
 
-    # Wire events
     start_btn.click(
         start_session,
         inputs=[name_box, session_state],
@@ -1086,18 +1056,15 @@ with gr.Blocks(title="Adaptive SQL Trainer — Randomized Domains") as demo:
         outputs=[feedback_md],
     )
     export_btn.click(
-        export_progress,
+        lambda user: os.path.abspath(os.path.join(EXPORT_DIR, f"{'-'.join((user or '').lower().split())[:64]}_progress.csv")),
         inputs=[export_name],
         outputs=[export_file],
     )
-
-    # UPDATED: one callback handles regeneration, dropdown refresh, and (if session) reseeding the next question
-    regen_btn.click(
+    regen_btn.click(  # one callback: reseed question, refresh dropdown, clear previews
         regenerate_domain,
         inputs=[session_state],
-        outputs=[regen_fb, er_image, prompt_md, sql_input, tbl_dd, session_state],
+        outputs=[regen_fb, er_image, prompt_md, sql_input, tbl_dd, mastery_df, result_df, session_state],
     )
-
     tbl_btn.click(
         preview_table,
         inputs=[tbl_dd],