P-Karthik-Mohan commited on
Commit
949c1e6
·
1 Parent(s): ee70dbf

Restore clean files , removed gradio, added task 4 and 5 :)

Browse files
Files changed (2) hide show
  1. inference.py +81 -80
  2. main.py +194 -117
inference.py CHANGED
@@ -1,35 +1,26 @@
1
  """
2
  inference.py — Baseline AI agent for SQL Analyst OpenEnv
3
- ---------------------------------------------------------
4
- Uses the OpenAI client (pointed at any compatible LLM via API_BASE_URL)
5
- to solve all 3 tasks by interacting with the running FastAPI environment.
6
  """
7
  import os
8
  import sys
 
9
  import time
10
  import requests
11
  from openai import OpenAI
12
- from dotenv import load_dotenv
13
- load_dotenv()
14
- # ── Configuration ─────────────────────────────────────────────────────────────
15
 
16
- ENV_BASE_URL = "https://p-karthik-mohan-sql-analyst-env.hf.space" # live HF Space
17
- MAX_ATTEMPTS = 5 # max SQL attempts per task
18
- TASK_IDS = [1, 2, 3] # tasks to solve
19
 
20
- API_BASE_URL = "https://api.groq.com/openai/v1"
21
- MODEL_NAME = "llama-3.1-8b-instant"
22
- HF_TOKEN = os.environ.get("HF_TOKEN")
23
-
24
- # ── OpenAI Client ─────────────────────────────────────────────────────────────
25
 
26
  client = OpenAI(
27
  base_url=API_BASE_URL,
28
  api_key=HF_TOKEN if HF_TOKEN else "no-key-needed",
29
  )
30
 
31
- # ── Environment API helpers ───────────────────────────────────────────────────
32
-
33
  def env_reset(task_id: int) -> dict:
34
  r = requests.post(f"{ENV_BASE_URL}/reset", json={"task_id": task_id})
35
  r.raise_for_status()
@@ -40,46 +31,35 @@ def env_step(sql: str) -> dict:
40
  r.raise_for_status()
41
  return r.json()
42
 
43
- def env_state() -> dict:
44
- r = requests.get(f"{ENV_BASE_URL}/state")
45
- r.raise_for_status()
46
- return r.json()
47
-
48
  def wait_for_server(retries: int = 10, delay: float = 2.0):
49
- """Wait until the FastAPI server is ready."""
50
  print("Waiting for environment server...")
51
  for i in range(retries):
52
  try:
53
- r = requests.get(f"{ENV_BASE_URL}/docs")
54
  if r.status_code == 200:
55
- print("Server is up!")
56
  return
57
- except requests.ConnectionError:
58
  pass
 
59
  time.sleep(delay)
60
  print("ERROR: Server did not start in time.")
61
  sys.exit(1)
62
 
63
- # ── LLM SQL Generator ─────────────────────────────────────────────────────────
64
-
65
  def build_system_prompt() -> str:
66
  return """You are an expert SQL analyst. Your job is to write correct SQLite queries.
67
 
68
  Rules:
69
  - Only write SELECT or WITH (CTE) statements. Never INSERT, UPDATE, DELETE, or DROP.
70
  - Always match the exact column names specified in the task.
71
- - Use proper SQLite syntax. RANK() OVER (...) is supported in SQLite 3.25+.
 
 
72
  - Return ONLY the raw SQL query — no explanation, no markdown, no backticks.
73
  - If a previous attempt scored less than 1.0, study the feedback and fix the query.
74
  """
75
 
76
- def build_user_prompt(
77
- task_description: str,
78
- schema: str,
79
- hint: str,
80
- attempt: int,
81
- previous_attempts: list,
82
- ) -> str:
83
  prompt = f"""Task:
84
  {task_description}
85
 
@@ -91,16 +71,21 @@ Hint: {hint}
91
  Attempt number: {attempt}
92
  """
93
  if previous_attempts:
94
- prompt += "\nPrevious attempts:\n"
95
- for i, prev in enumerate(previous_attempts):
96
- prompt += f"--- Attempt {i+1} ---\nSQL: {prev['sql']}\nReward: {prev['reward']}\n\n"
97
-
 
 
 
 
 
 
 
98
  prompt += "\nWrite the corrected SQL query now:"
99
  return prompt
100
 
101
- def ask_llm(task_description: str, schema: str, hint: str,
102
- attempt: int, previous_attempts: list) -> str:
103
- """Call the LLM and return a SQL string."""
104
  messages = [
105
  {"role": "system", "content": build_system_prompt()},
106
  {"role": "user", "content": build_user_prompt(
@@ -114,22 +99,19 @@ def ask_llm(task_description: str, schema: str, hint: str,
114
  max_tokens=512,
115
  )
116
  sql = response.choices[0].message.content.strip()
117
-
118
- # Strip markdown fences if model wraps in ```sql ... ```
119
  if sql.startswith("```"):
120
- sql = sql.split("\n", 1)[-1].rsplit("\n", 1)[0].replace("```", "").strip()
121
-
 
 
 
122
  return sql
123
 
124
- # ── Main Agent Loop ───────────────────────────────────────────────────────────
125
-
126
  def solve_task(task_id: int) -> dict:
127
- """Run the agent on a single task. Returns final result dict."""
128
  print(f"\n{'='*60}")
129
  print(f"TASK {task_id}")
130
  print('='*60)
131
 
132
- # Reset environment
133
  reset_resp = env_reset(task_id)
134
  obs = reset_resp["observation"]
135
  task_desc = obs["task_description"]
@@ -145,33 +127,47 @@ def solve_task(task_id: int) -> dict:
145
  final_sql = ""
146
 
147
  for attempt in range(1, MAX_ATTEMPTS + 1):
148
- print(f"Attempt {attempt}/{MAX_ATTEMPTS}...")
149
  sql = ask_llm(task_desc, schema, hint, attempt, previous_attempts)
150
- print(f"Generated SQL: {sql}")
151
-
152
  step_resp = env_step(sql)
153
- reward = step_resp["reward"]
154
- print(f"Reward: {reward}")
155
-
 
 
 
 
 
 
156
  best_reward = max(best_reward, reward)
157
- final_sql = sql
158
-
159
- if reward >= 1.0:
160
- print("Task solved successfully!")
 
 
 
 
 
 
 
161
  break
162
-
163
- previous_attempts.append({"sql": sql, "reward": reward})
 
 
164
 
165
  return {
166
- "task_id": task_id,
167
- "difficulty": difficulty,
168
- "best_reward": best_reward,
169
- "attempts": len(previous_attempts) + (1 if best_reward >= 1.0 else 0),
170
- "final_sql": final_sql,
171
- "solved": best_reward >= 1.0,
172
  }
173
 
174
-
175
  def main():
176
  print("SQL Analyst OpenEnv — Baseline Inference Agent")
177
  print(f"Model : {MODEL_NAME}")
@@ -182,26 +178,31 @@ def main():
182
 
183
  results = []
184
  for task_id in TASK_IDS:
185
- res = solve_task(task_id)
186
- results.append(res)
187
 
188
- # ── Final Summary ─────────────────────────────────────────────────────────
189
  print(f"\n{'='*60}")
190
  print("FINAL RESULTS")
191
  print('='*60)
192
 
193
  total_score = 0.0
194
- solved_count = 0
195
  for r in results:
 
 
 
196
  total_score += r["best_reward"]
197
- if r["solved"]:
198
- solved_count += 1
199
- print(f"Task {r['task_id']} ({r['difficulty']}): Reward = {r['best_reward']:.2f} | Solved = {r['solved']}")
200
-
201
- avg_score = total_score / len(results) if results else 0
202
- print(f"\nTasks solved : {solved_count} / {len(results)}")
203
- print(f"Average reward : {avg_score:.3f} / 1.000")
204
 
 
 
 
 
 
 
 
 
 
 
 
205
 
206
  if __name__ == "__main__":
207
  main()
 
1
  """
2
  inference.py — Baseline AI agent for SQL Analyst OpenEnv
 
 
 
3
  """
4
  import os
5
  import sys
6
+ import json
7
  import time
8
  import requests
9
  from openai import OpenAI
 
 
 
10
 
11
+ ENV_BASE_URL = "https://p-karthik-mohan-sql-analyst-env.hf.space"
12
+ MAX_ATTEMPTS = 5
13
+ TASK_IDS = [1, 2, 3, 4, 5]
14
 
15
+ API_BASE_URL = os.environ.get("API_BASE_URL", "https://api.groq.com/openai/v1")
16
+ MODEL_NAME = os.environ.get("MODEL_NAME", "llama-3.1-8b-instant")
17
+ HF_TOKEN = os.environ.get("HF_TOKEN", "")
 
 
18
 
19
  client = OpenAI(
20
  base_url=API_BASE_URL,
21
  api_key=HF_TOKEN if HF_TOKEN else "no-key-needed",
22
  )
23
 
 
 
24
  def env_reset(task_id: int) -> dict:
25
  r = requests.post(f"{ENV_BASE_URL}/reset", json={"task_id": task_id})
26
  r.raise_for_status()
 
31
  r.raise_for_status()
32
  return r.json()
33
 
 
 
 
 
 
34
  def wait_for_server(retries: int = 10, delay: float = 2.0):
 
35
  print("Waiting for environment server...")
36
  for i in range(retries):
37
  try:
38
+ r = requests.get(f"{ENV_BASE_URL}/health", timeout=3)
39
  if r.status_code == 200:
40
+ print("Server is ready.\n")
41
  return
42
+ except requests.exceptions.ConnectionError:
43
  pass
44
+ print(f" Not ready yet... ({i+1}/{retries})")
45
  time.sleep(delay)
46
  print("ERROR: Server did not start in time.")
47
  sys.exit(1)
48
 
 
 
49
  def build_system_prompt() -> str:
50
  return """You are an expert SQL analyst. Your job is to write correct SQLite queries.
51
 
52
  Rules:
53
  - Only write SELECT or WITH (CTE) statements. Never INSERT, UPDATE, DELETE, or DROP.
54
  - Always match the exact column names specified in the task.
55
+ - Always filter WHERE status = 'completed' unless told otherwise.
56
+ - Use STRFTIME('%Y', order_date) for year filtering in SQLite.
57
+ - RANK() OVER (...) is supported in SQLite 3.25+.
58
  - Return ONLY the raw SQL query — no explanation, no markdown, no backticks.
59
  - If a previous attempt scored less than 1.0, study the feedback and fix the query.
60
  """
61
 
62
+ def build_user_prompt(task_description, schema, hint, attempt, previous_attempts):
 
 
 
 
 
 
63
  prompt = f"""Task:
64
  {task_description}
65
 
 
71
  Attempt number: {attempt}
72
  """
73
  if previous_attempts:
74
+ prompt += "\nYour previous attempts and their scores:\n"
75
+ for prev in previous_attempts[-3:]:
76
+ prompt += f"""
77
+ Attempt {prev['attempt']}:
78
+ SQL: {prev['sql']}
79
+ Reward: {prev['reward']} / 1.0
80
+ Columns expected : {prev['details'].get('expected_columns', [])}
81
+ Columns you gave : {prev['details'].get('agent_columns', [])}
82
+ Rows expected : {prev['details'].get('expected_row_count', '?')}
83
+ Rows you gave : {prev['details'].get('agent_row_count', '?')}
84
+ """
85
  prompt += "\nWrite the corrected SQL query now:"
86
  return prompt
87
 
88
+ def ask_llm(task_description, schema, hint, attempt, previous_attempts):
 
 
89
  messages = [
90
  {"role": "system", "content": build_system_prompt()},
91
  {"role": "user", "content": build_user_prompt(
 
99
  max_tokens=512,
100
  )
101
  sql = response.choices[0].message.content.strip()
 
 
102
  if sql.startswith("```"):
103
+ lines = sql.split("\n")
104
+ sql = "\n".join(
105
+ line for line in lines
106
+ if not line.strip().startswith("```")
107
+ ).strip()
108
  return sql
109
 
 
 
110
  def solve_task(task_id: int) -> dict:
 
111
  print(f"\n{'='*60}")
112
  print(f"TASK {task_id}")
113
  print('='*60)
114
 
 
115
  reset_resp = env_reset(task_id)
116
  obs = reset_resp["observation"]
117
  task_desc = obs["task_description"]
 
127
  final_sql = ""
128
 
129
  for attempt in range(1, MAX_ATTEMPTS + 1):
130
+ print(f" Attempt {attempt}/{MAX_ATTEMPTS} — asking LLM...")
131
  sql = ask_llm(task_desc, schema, hint, attempt, previous_attempts)
132
+ print(f" SQL: {sql[:120]}{'...' if len(sql) > 120 else ''}")
133
+
134
  step_resp = env_step(sql)
135
+ reward = step_resp["reward"]
136
+ done = step_resp["done"]
137
+ details = step_resp["observation"].get("reward_breakdown", {})
138
+
139
+ print(f" Reward: {reward:.3f} "
140
+ f"(cols={details.get('column_score',0):.2f} "
141
+ f"rows={details.get('row_score',0):.2f} "
142
+ f"vals={details.get('value_score',0):.2f})")
143
+
144
  best_reward = max(best_reward, reward)
145
+ final_sql = sql
146
+
147
+ previous_attempts.append({
148
+ "attempt": attempt,
149
+ "sql": sql,
150
+ "reward": reward,
151
+ "details": details,
152
+ })
153
+
154
+ if done:
155
+ print(f" PERFECT SCORE on attempt {attempt}!")
156
  break
157
+ elif reward >= 0.8:
158
+ print(f" Score is close ({reward:.3f}). Trying to improve...")
159
+ else:
160
+ print(f" Score is low ({reward:.3f}). Refining query...")
161
 
162
  return {
163
+ "task_id": task_id,
164
+ "difficulty": difficulty,
165
+ "best_reward": best_reward,
166
+ "attempts": len(previous_attempts),
167
+ "final_sql": final_sql,
168
+ "solved": best_reward >= 1.0,
169
  }
170
 
 
171
  def main():
172
  print("SQL Analyst OpenEnv — Baseline Inference Agent")
173
  print(f"Model : {MODEL_NAME}")
 
178
 
179
  results = []
180
  for task_id in TASK_IDS:
181
+ result = solve_task(task_id)
182
+ results.append(result)
183
 
 
184
  print(f"\n{'='*60}")
185
  print("FINAL RESULTS")
186
  print('='*60)
187
 
188
  total_score = 0.0
 
189
  for r in results:
190
+ status = "SOLVED" if r["solved"] else f"best={r['best_reward']:.3f}"
191
+ print(f" Task {r['task_id']} ({r['difficulty']:6s}) {status} "
192
+ f"in {r['attempts']} attempt(s)")
193
  total_score += r["best_reward"]
 
 
 
 
 
 
 
194
 
195
+ avg_score = total_score / len(results)
196
+ print(f"\n Average reward : {avg_score:.3f} / 1.000")
197
+ print(f" Tasks solved : {sum(1 for r in results if r['solved'])} / {len(results)}")
198
+
199
+ with open("results.json", "w") as f:
200
+ json.dump({
201
+ "results": results,
202
+ "avg_score": round(avg_score, 3),
203
+ "tasks_solved": sum(1 for r in results if r["solved"]),
204
+ }, f, indent=2)
205
+ print(f"\n Results saved to results.json")
206
 
207
  if __name__ == "__main__":
208
  main()
main.py CHANGED
@@ -4,117 +4,152 @@ import json
4
  import re
5
  from datetime import datetime
6
  from typing import Any, Optional
7
- import uuid
8
 
9
  from fastapi import FastAPI, HTTPException
10
- from pydantic import BaseModel, Field
11
- import gradio as gr
12
- from ui import build_ui
13
 
14
- # ── Config ────────────────────────────────────────────────────────────────────
15
  DB_PATH = os.path.join("data", "ecommerce.db")
16
- # Under /api for direct agent access, root for UI
17
- api_app = FastAPI(
18
- title="SQL Analyst OpenEnv API",
19
- version="1.1.0",
20
- docs_url="/docs",
21
- redoc_url="/redoc",
22
- description="A real-world benchmark environment for AI agents writing SQL. Use `/reset` to select a task, `/step` to submit a query and receive partial-credit feedback, and `/state` to track history."
23
- )
24
-
25
-
26
- # ── Pydantic Models ───────────────────────────────────────────────────────────
27
 
28
  class StepRequest(BaseModel):
29
- session_id: str = Field("default", description="Unique session ID to prevent collisions.")
30
- action: str = Field(..., description="A valid SQLite SELECT or WITH statement.", examples=["SELECT COUNT(*) AS total_orders FROM orders;"])
31
 
32
  class StepResponse(BaseModel):
33
- observation: dict = Field(description="Contains result_preview (first 5 rows), reward_breakdown, and any sql execution errors.")
34
- reward: float = Field(description="Partial credit reward from 0.0 (wrong) to 1.0 (perfect).")
35
- done: bool = Field(description="True if reward == 1.0 (Task solved).")
36
- info: dict = Field(description="System message and attempt count.")
37
 
38
  class ResetRequest(BaseModel):
39
- task_id: int = Field(..., description="ID of the task to load (1 to 5).", examples=[1])
40
- session_id: str = Field("default", description="Unique session ID.")
41
 
42
  class ResetResponse(BaseModel):
43
- observation: dict = Field(description="Contains task_description, schema, and hint to be passed to the agent.")
44
- info: dict = Field(description="System message.")
45
-
46
- class StateRequest(BaseModel):
47
- session_id: str = Field("default", description="Unique session ID.")
48
 
49
  class StateResponse(BaseModel):
50
- session_id: str
51
- task_id: Optional[int] = Field(description="Currently active Task ID.")
52
- task_description: Optional[str]
53
- schema_info: str = Field(description="Raw string representation of DB Schema.")
54
- attempts: int = Field(description="Number of queries submitted so far on active task.")
55
- best_reward: float = Field(description="Highest partial credit achieved.")
56
- history: list = Field(description="Log of all queries executed.")
57
-
58
- # ── Task Definitions ──────────────────────────────────────────────────────────
59
 
60
  TASKS = {
61
  1: {
62
- "description": "Find the total number of completed orders placed in the year 2024. Return a single number with column name: total_orders",
 
 
 
63
  "difficulty": "easy",
64
  "hint": "Use COUNT with WHERE filters on status and order_date",
65
- "answer_query": "SELECT COUNT(*) AS total_orders FROM orders WHERE status = 'completed' AND order_date LIKE '2024%'",
 
 
 
 
 
66
  },
67
  2: {
68
- "description": "Find the top 5 customers by total revenue (sum of total_amount for completed orders only). Return columns: first_name, last_name, total_revenue. Order by total_revenue descending.",
 
 
 
 
69
  "difficulty": "medium",
70
  "hint": "JOIN orders with customers, GROUP BY customer, filter completed, ORDER and LIMIT",
71
- "answer_query": "SELECT c.first_name, c.last_name, ROUND(SUM(o.total_amount), 2) AS total_revenue FROM orders o JOIN customers c ON o.customer_id = c.customer_id WHERE o.status = 'completed' GROUP BY o.customer_id ORDER BY total_revenue DESC LIMIT 5",
 
 
 
 
 
 
 
 
 
72
  },
73
  3: {
74
- "description": "For each product category, calculate the total revenue (completed orders only) and rank categories by revenue using a window function. Return columns: category, total_revenue, revenue_rank. Order by revenue_rank ascending.",
 
 
 
 
 
75
  "difficulty": "hard",
76
  "hint": "Use SUM with GROUP BY inside a CTE, then apply RANK() OVER (ORDER BY ...) on the result",
77
- "answer_query": "WITH category_revenue AS ( SELECT p.category, SUM(o.total_amount) AS total_revenue FROM orders o JOIN products p ON o.product_id = p.product_id WHERE o.status = 'completed' GROUP BY p.category ) SELECT category, total_revenue, RANK() OVER (ORDER BY total_revenue DESC) AS revenue_rank FROM category_revenue ORDER BY revenue_rank ASC",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
78
  },
79
  4: {
80
- "description": "Find the average price of products in each category, but only for categories that have more than 2 products. Return columns: category, avg_price.",
 
 
 
 
81
  "difficulty": "medium",
82
- "hint": "Use GROUP BY with HAVING COUNT(...) > 2.",
83
- "answer_query": "SELECT category, ROUND(AVG(price), 2) AS avg_price FROM products GROUP BY category HAVING COUNT(product_id) > 2",
 
 
 
 
 
84
  },
85
  5: {
86
- "description": "Identify customers who have ordered products from both the 'Electronics' and 'Clothing' categories. Return columns: customer_id, first_name.",
 
 
 
87
  "difficulty": "hard",
88
- "hint": "Use INTERSECT on two queries, or GROUP BY customer HAVING COUNT(DISTINCT category) = 2.",
89
- "answer_query": "SELECT DISTINCT c.customer_id, c.first_name FROM customers c JOIN orders o ON c.customer_id = o.customer_id JOIN products p ON o.product_id = p.product_id WHERE p.category = 'Electronics' INTERSECT SELECT DISTINCT c.customer_id, c.first_name FROM customers c JOIN orders o ON c.customer_id = o.customer_id JOIN products p ON o.product_id = p.product_id WHERE p.category = 'Clothing'",
 
 
 
 
 
 
 
 
 
 
 
 
90
  },
91
  }
92
 
93
- # ── Sessions ──────────────────────────────────────────────────────────────────
94
-
95
- sessions = {}
96
-
97
- def get_session(sid: str):
98
- if sid not in sessions:
99
- sessions[sid] = {
100
- "task_id": None, "task": None,
101
- "expected_rows": None, "expected_columns": None,
102
- "attempts": 0, "best_reward": 0.0, "history": []
103
- }
104
- return sessions[sid]
105
-
106
- # ── Database Helpers ──────────────────────────────────────────────────────────
107
-
108
- def progress_handler():
109
- raise sqlite3.OperationalError("Query execution aborted: Timed out or exceeded instruction limits. Hint: Too complex CROSS JOIN?")
110
 
111
  def get_connection():
112
  if not os.path.exists(DB_PATH):
113
- raise HTTPException(status_code=500, detail=f"Database not found at {DB_PATH}.")
 
 
 
114
  conn = sqlite3.connect(DB_PATH)
115
  conn.row_factory = sqlite3.Row
116
- # Security feature: Prevent DOS
117
- conn.set_progress_handler(progress_handler, 500000)
118
  return conn
119
 
120
  def get_schema_info() -> str:
@@ -142,26 +177,24 @@ def run_query(sql: str) -> tuple[list[dict], list[str]]:
142
  conn.close()
143
  return rows, columns
144
 
145
- def compute_expected(sid: str):
146
- session = get_session(sid)
147
  task = session["task"]
148
  rows, columns = run_query(task["answer_query"])
149
  session["expected_rows"] = rows
150
  session["expected_columns"] = columns
151
 
152
- # ── Reward Function ───────────────────────────────────────────────────────────
153
-
154
- def compute_reward(sid: str, agent_rows: list[dict], agent_cols: list[str]) -> tuple[float, dict]:
155
- session = get_session(sid)
156
  expected_rows = session["expected_rows"]
157
  expected_cols = session["expected_columns"]
158
  details = {}
159
 
160
- agent_cols_lower = [c.lower() for c in agent_cols]
161
  expected_cols_lower = [c.lower() for c in expected_cols]
162
  col_matches = sum(1 for c in expected_cols_lower if c in agent_cols_lower)
163
  col_score = (col_matches / len(expected_cols_lower)) * 0.30 if expected_cols_lower else 0.0
164
  details["column_score"] = round(col_score, 3)
 
 
165
 
166
  expected_count = len(expected_rows)
167
  agent_count = len(agent_rows)
@@ -171,14 +204,19 @@ def compute_reward(sid: str, agent_rows: list[dict], agent_cols: list[str]) -> t
171
  row_ratio = min(agent_count, expected_count) / max(agent_count, expected_count)
172
  row_score = row_ratio * 0.30
173
  details["row_score"] = round(row_score, 3)
 
 
174
 
175
  if not expected_rows or not agent_rows:
176
  value_score = 0.0
177
  else:
178
  def normalize(v):
179
- if v is None: return ""
180
- try: return str(round(float(v), 1))
181
- except: return str(v).strip().lower()
 
 
 
182
 
183
  matched_cells = 0
184
  total_cells = len(expected_rows) * len(expected_cols_lower)
@@ -202,82 +240,121 @@ def compute_reward(sid: str, agent_rows: list[dict], agent_cols: list[str]) -> t
202
  details["total_reward"] = total
203
  return total, details
204
 
205
- # ── Endpoints ─────────────────────────────────────────────────────────────────
206
-
207
- @api_app.post("/reset", response_model=ResetResponse, tags=["Agent Environment"], summary="Load Task", description="Initializes the environment with a random task, or resets the current task, returning the description, hints, and database schema.")
208
  def reset(req: ResetRequest):
209
- if req.task_id not in TASKS: raise HTTPException(status_code=400, detail="Invalid task_id")
210
- session = get_session(req.session_id)
211
- session["task_id"] = req.task_id
212
- session["task"] = TASKS[req.task_id]
213
- session["attempts"] = 0
 
214
  session["best_reward"] = 0.0
215
- session["history"] = []
216
- compute_expected(req.session_id)
 
 
 
217
  observation = {
218
  "task_id": req.task_id,
219
  "difficulty": session["task"]["difficulty"],
220
  "task_description": session["task"]["description"],
221
- "schema": get_schema_info(),
222
  "hint": session["task"]["hint"],
223
  }
224
- return ResetResponse(observation=observation, info={"message": f"Task {req.task_id} loaded."})
 
 
 
225
 
226
- @api_app.post("/step", response_model=StepResponse, tags=["Agent Environment"], summary="Execute SQL", description="Executes the valid SQLite `action` against the ecommerce database, evaluating the correctness using a partial 0.0-1.0 Reward function.")
227
  def step(req: StepRequest):
228
- session = get_session(req.session_id)
229
- if session["task_id"] is None: raise HTTPException(status_code=400, detail="Call /reset first.")
 
230
  session["attempts"] += 1
231
  sql = req.action.strip()
232
 
233
  if not re.match(r"^\s*(SELECT|WITH)\b", sql, re.IGNORECASE):
234
  return StepResponse(
235
- observation={"error": "Only SELECT or WITH allowed."}, reward=0.0, done=False,
236
- info={"attempt": session["attempts"], "message": "Rejected"}
 
 
237
  )
238
 
239
  try:
240
  agent_rows, agent_cols = run_query(sql)
241
  except Exception as e:
242
- session["history"].append({"attempt": session["attempts"], "sql": sql, "reward": 0.0, "error": str(e)})
 
 
 
 
 
 
243
  return StepResponse(
244
- observation={"error": str(e), "sql_submitted": sql}, reward=0.0, done=False,
245
- info={"attempt": session["attempts"], "message": "SQL Error"}
 
 
246
  )
247
 
248
- reward, details = compute_reward(req.session_id, agent_rows, agent_cols)
249
  session["best_reward"] = max(session["best_reward"], reward)
250
  done = reward >= 1.0
251
- session["history"].append({"attempt": session["attempts"], "sql": sql, "reward": reward, "details": details})
 
 
 
 
 
 
 
 
252
 
253
  observation = {
254
- "task_id": session["task_id"], "task_description": session["task"]["description"],
255
- "sql_submitted": sql, "result_preview": agent_rows[:5], "result_row_count": len(agent_rows),
 
 
 
256
  "reward_breakdown": details,
257
  }
 
258
  return StepResponse(
259
- observation=observation, reward=reward, done=done,
260
- info={"attempt": session["attempts"], "best_reward": session["best_reward"]}
 
 
 
 
 
 
261
  )
262
 
263
- @api_app.get("/state", response_model=StateResponse, tags=["Diagnostics"], summary="Get Current State", description="Returns attempts, rewards, parsed tasks, and historical executed sql queries array.")
264
- def state(req: StateRequest):
265
- session = get_session(req.session_id)
 
 
266
  return StateResponse(
267
- session_id=req.session_id,
268
  task_id=session["task_id"],
269
- task_description=session["task"]["description"] if session["task"] else None,
270
  schema_info=get_schema_info(),
271
  attempts=session["attempts"],
272
  best_reward=session["best_reward"],
273
  history=session["history"],
274
  )
275
 
276
- @api_app.get("/")
277
  def root():
278
- return {"message": "API running at /api. Try the UI at root (handled by wrapper)!"}
279
-
280
- # ── Server Setup ──────────────────────────────────────────────────��───────────
 
 
 
281
 
282
- demo = build_ui()
283
- app = gr.mount_gradio_app(api_app, demo, path="/")
 
 
4
  import re
5
  from datetime import datetime
6
  from typing import Any, Optional
 
7
 
8
  from fastapi import FastAPI, HTTPException
9
+ from pydantic import BaseModel
 
 
10
 
 
11
  DB_PATH = os.path.join("data", "ecommerce.db")
12
+ app = FastAPI(title="SQL Analyst OpenEnv", version="1.0.0")
 
 
 
 
 
 
 
 
 
 
13
 
14
  class StepRequest(BaseModel):
15
+ action: str
 
16
 
17
  class StepResponse(BaseModel):
18
+ observation: dict
19
+ reward: float
20
+ done: bool
21
+ info: dict
22
 
23
  class ResetRequest(BaseModel):
24
+ task_id: int
 
25
 
26
  class ResetResponse(BaseModel):
27
+ observation: dict
28
+ info: dict
 
 
 
29
 
30
  class StateResponse(BaseModel):
31
+ task_id: int
32
+ task_description: str
33
+ schema_info: str
34
+ attempts: int
35
+ best_reward: float
36
+ history: list
 
 
 
37
 
38
  TASKS = {
39
  1: {
40
+ "description": (
41
+ "Find the total number of completed orders placed in the year 2024. "
42
+ "Return a single number with column name: total_orders"
43
+ ),
44
  "difficulty": "easy",
45
  "hint": "Use COUNT with WHERE filters on status and order_date",
46
+ "answer_query": """
47
+ SELECT COUNT(*) AS total_orders
48
+ FROM orders
49
+ WHERE status = 'completed'
50
+ AND order_date LIKE '2024%'
51
+ """,
52
  },
53
  2: {
54
+ "description": (
55
+ "Find the top 5 customers by total revenue (sum of total_amount for completed orders only). "
56
+ "Return columns: first_name, last_name, total_revenue. "
57
+ "Order by total_revenue descending."
58
+ ),
59
  "difficulty": "medium",
60
  "hint": "JOIN orders with customers, GROUP BY customer, filter completed, ORDER and LIMIT",
61
+ "answer_query": """
62
+ SELECT c.first_name, c.last_name,
63
+ ROUND(SUM(o.total_amount), 2) AS total_revenue
64
+ FROM orders o
65
+ JOIN customers c ON o.customer_id = c.customer_id
66
+ WHERE o.status = 'completed'
67
+ GROUP BY o.customer_id
68
+ ORDER BY total_revenue DESC
69
+ LIMIT 5
70
+ """,
71
  },
72
  3: {
73
+ "description": (
74
+ "For each product category, calculate the total revenue (completed orders only) "
75
+ "and rank categories by revenue using a window function. "
76
+ "Return columns: category, total_revenue, revenue_rank. "
77
+ "Order by revenue_rank ascending."
78
+ ),
79
  "difficulty": "hard",
80
  "hint": "Use SUM with GROUP BY inside a CTE, then apply RANK() OVER (ORDER BY ...) on the result",
81
+ "answer_query": """
82
+ WITH category_revenue AS (
83
+ SELECT p.category,
84
+ SUM(o.total_amount) AS total_revenue
85
+ FROM orders o
86
+ JOIN products p ON o.product_id = p.product_id
87
+ WHERE o.status = 'completed'
88
+ GROUP BY p.category
89
+ )
90
+ SELECT category,
91
+ total_revenue,
92
+ RANK() OVER (ORDER BY total_revenue DESC) AS revenue_rank
93
+ FROM category_revenue
94
+ ORDER BY revenue_rank ASC
95
+ """,
96
  },
97
  4: {
98
+ "description": (
99
+ "Find the average price of products in each category, "
100
+ "but only for categories that have more than 2 products. "
101
+ "Return columns: category, avg_price."
102
+ ),
103
  "difficulty": "medium",
104
+ "hint": "Use GROUP BY with HAVING COUNT(...) > 2",
105
+ "answer_query": """
106
+ SELECT category, ROUND(AVG(price), 2) AS avg_price
107
+ FROM products
108
+ GROUP BY category
109
+ HAVING COUNT(product_id) > 2
110
+ """,
111
  },
112
  5: {
113
+ "description": (
114
+ "Identify customers who have ordered products from both the Electronics "
115
+ "and Clothing categories. Return columns: customer_id, first_name."
116
+ ),
117
  "difficulty": "hard",
118
+ "hint": "Use INTERSECT on two queries filtering by category",
119
+ "answer_query": """
120
+ SELECT DISTINCT c.customer_id, c.first_name
121
+ FROM customers c
122
+ JOIN orders o ON c.customer_id = o.customer_id
123
+ JOIN products p ON o.product_id = p.product_id
124
+ WHERE p.category = 'Electronics'
125
+ INTERSECT
126
+ SELECT DISTINCT c.customer_id, c.first_name
127
+ FROM customers c
128
+ JOIN orders o ON c.customer_id = o.customer_id
129
+ JOIN products p ON o.product_id = p.product_id
130
+ WHERE p.category = 'Clothing'
131
+ """,
132
  },
133
  }
134
 
135
+ session = {
136
+ "task_id": None,
137
+ "task": None,
138
+ "expected_rows": None,
139
+ "expected_columns": None,
140
+ "attempts": 0,
141
+ "best_reward": 0.0,
142
+ "history": [],
143
+ }
 
 
 
 
 
 
 
 
144
 
145
  def get_connection():
146
  if not os.path.exists(DB_PATH):
147
+ raise HTTPException(
148
+ status_code=500,
149
+ detail=f"Database not found at {DB_PATH}. Run seed.py first."
150
+ )
151
  conn = sqlite3.connect(DB_PATH)
152
  conn.row_factory = sqlite3.Row
 
 
153
  return conn
154
 
155
  def get_schema_info() -> str:
 
177
  conn.close()
178
  return rows, columns
179
 
180
+ def compute_expected():
 
181
  task = session["task"]
182
  rows, columns = run_query(task["answer_query"])
183
  session["expected_rows"] = rows
184
  session["expected_columns"] = columns
185
 
186
+ def compute_reward(agent_rows: list[dict], agent_cols: list[str]) -> tuple[float, dict]:
 
 
 
187
  expected_rows = session["expected_rows"]
188
  expected_cols = session["expected_columns"]
189
  details = {}
190
 
191
+ agent_cols_lower = [c.lower() for c in agent_cols]
192
  expected_cols_lower = [c.lower() for c in expected_cols]
193
  col_matches = sum(1 for c in expected_cols_lower if c in agent_cols_lower)
194
  col_score = (col_matches / len(expected_cols_lower)) * 0.30 if expected_cols_lower else 0.0
195
  details["column_score"] = round(col_score, 3)
196
+ details["expected_columns"] = expected_cols
197
+ details["agent_columns"] = agent_cols
198
 
199
  expected_count = len(expected_rows)
200
  agent_count = len(agent_rows)
 
204
  row_ratio = min(agent_count, expected_count) / max(agent_count, expected_count)
205
  row_score = row_ratio * 0.30
206
  details["row_score"] = round(row_score, 3)
207
+ details["expected_row_count"] = expected_count
208
+ details["agent_row_count"] = agent_count
209
 
210
  if not expected_rows or not agent_rows:
211
  value_score = 0.0
212
  else:
213
  def normalize(v):
214
+ if v is None:
215
+ return ""
216
+ try:
217
+ return str(round(float(v), 1))
218
+ except (ValueError, TypeError):
219
+ return str(v).strip().lower()
220
 
221
  matched_cells = 0
222
  total_cells = len(expected_rows) * len(expected_cols_lower)
 
240
  details["total_reward"] = total
241
  return total, details
242
 
243
+ @app.post("/reset", response_model=ResetResponse)
 
 
244
  def reset(req: ResetRequest):
245
+ if req.task_id not in TASKS:
246
+ raise HTTPException(status_code=400, detail="task_id must be 1, 2, 3, 4, or 5")
247
+
248
+ session["task_id"] = req.task_id
249
+ session["task"] = TASKS[req.task_id]
250
+ session["attempts"] = 0
251
  session["best_reward"] = 0.0
252
+ session["history"] = []
253
+
254
+ compute_expected()
255
+
256
+ schema = get_schema_info()
257
  observation = {
258
  "task_id": req.task_id,
259
  "difficulty": session["task"]["difficulty"],
260
  "task_description": session["task"]["description"],
261
+ "schema": schema,
262
  "hint": session["task"]["hint"],
263
  }
264
+ return ResetResponse(
265
+ observation=observation,
266
+ info={"message": f"Task {req.task_id} loaded. Use POST /step with your SQL query."}
267
+ )
268
 
269
+ @app.post("/step", response_model=StepResponse)
270
  def step(req: StepRequest):
271
+ if session["task_id"] is None:
272
+ raise HTTPException(status_code=400, detail="Call /reset first to load a task.")
273
+
274
  session["attempts"] += 1
275
  sql = req.action.strip()
276
 
277
  if not re.match(r"^\s*(SELECT|WITH)\b", sql, re.IGNORECASE):
278
  return StepResponse(
279
+ observation={"error": "Only SELECT or WITH (CTE) statements are allowed."},
280
+ reward=0.0,
281
+ done=False,
282
+ info={"attempt": session["attempts"], "message": "Rejected: not a SELECT/WITH query."}
283
  )
284
 
285
  try:
286
  agent_rows, agent_cols = run_query(sql)
287
  except Exception as e:
288
+ entry = {
289
+ "attempt": session["attempts"],
290
+ "sql": sql,
291
+ "reward": 0.0,
292
+ "error": str(e),
293
+ }
294
+ session["history"].append(entry)
295
  return StepResponse(
296
+ observation={"error": str(e), "sql_submitted": sql},
297
+ reward=0.0,
298
+ done=False,
299
+ info={"attempt": session["attempts"], "message": "SQL execution error."}
300
  )
301
 
302
+ reward, details = compute_reward(agent_rows, agent_cols)
303
  session["best_reward"] = max(session["best_reward"], reward)
304
  done = reward >= 1.0
305
+
306
+ entry = {
307
+ "attempt": session["attempts"],
308
+ "sql": sql,
309
+ "reward": reward,
310
+ "details": details,
311
+ "timestamp": datetime.now().isoformat(),
312
+ }
313
+ session["history"].append(entry)
314
 
315
  observation = {
316
+ "task_id": session["task_id"],
317
+ "task_description": session["task"]["description"],
318
+ "sql_submitted": sql,
319
+ "result_preview": agent_rows[:5],
320
+ "result_row_count": len(agent_rows),
321
  "reward_breakdown": details,
322
  }
323
+
324
  return StepResponse(
325
+ observation=observation,
326
+ reward=reward,
327
+ done=done,
328
+ info={
329
+ "attempt": session["attempts"],
330
+ "best_reward": session["best_reward"],
331
+ "message": "Perfect score! Task complete." if done else "Keep refining your query.",
332
+ }
333
  )
334
 
335
+ @app.get("/state", response_model=StateResponse)
336
+ def state():
337
+ if session["task_id"] is None:
338
+ raise HTTPException(status_code=400, detail="No active task. Call /reset first.")
339
+
340
  return StateResponse(
 
341
  task_id=session["task_id"],
342
+ task_description=session["task"]["description"],
343
  schema_info=get_schema_info(),
344
  attempts=session["attempts"],
345
  best_reward=session["best_reward"],
346
  history=session["history"],
347
  )
348
 
349
+ @app.get("/")
350
  def root():
351
+ return {
352
+ "name": "SQL Analyst OpenEnv",
353
+ "version": "1.0.0",
354
+ "tasks": {k: {"difficulty": v["difficulty"], "description": v["description"]} for k, v in TASKS.items()},
355
+ "endpoints": ["/reset", "/step", "/state"],
356
+ }
357
 
358
+ @app.get("/health")
359
+ def health():
360
+ return {"status": "ok", "db_exists": os.path.exists(DB_PATH)}