Spaces:
Sleeping
Sleeping
Commit ·
949c1e6
1
Parent(s): ee70dbf
Restore clean files , removed gradio, added task 4 and 5 :)
Browse files- inference.py +81 -80
- main.py +194 -117
inference.py
CHANGED
|
@@ -1,35 +1,26 @@
|
|
| 1 |
"""
|
| 2 |
inference.py — Baseline AI agent for SQL Analyst OpenEnv
|
| 3 |
-
---------------------------------------------------------
|
| 4 |
-
Uses the OpenAI client (pointed at any compatible LLM via API_BASE_URL)
|
| 5 |
-
to solve all 3 tasks by interacting with the running FastAPI environment.
|
| 6 |
"""
|
| 7 |
import os
|
| 8 |
import sys
|
|
|
|
| 9 |
import time
|
| 10 |
import requests
|
| 11 |
from openai import OpenAI
|
| 12 |
-
from dotenv import load_dotenv
|
| 13 |
-
load_dotenv()
|
| 14 |
-
# ── Configuration ─────────────────────────────────────────────────────────────
|
| 15 |
|
| 16 |
-
ENV_BASE_URL = "https://p-karthik-mohan-sql-analyst-env.hf.space"
|
| 17 |
-
MAX_ATTEMPTS = 5
|
| 18 |
-
TASK_IDS = [1, 2, 3
|
| 19 |
|
| 20 |
-
API_BASE_URL = "https://api.groq.com/openai/v1"
|
| 21 |
-
MODEL_NAME = "llama-3.1-8b-instant"
|
| 22 |
-
HF_TOKEN = os.environ.get("HF_TOKEN")
|
| 23 |
-
|
| 24 |
-
# ── OpenAI Client ─────────────────────────────────────────────────────────────
|
| 25 |
|
| 26 |
client = OpenAI(
|
| 27 |
base_url=API_BASE_URL,
|
| 28 |
api_key=HF_TOKEN if HF_TOKEN else "no-key-needed",
|
| 29 |
)
|
| 30 |
|
| 31 |
-
# ── Environment API helpers ───────────────────────────────────────────────────
|
| 32 |
-
|
| 33 |
def env_reset(task_id: int) -> dict:
|
| 34 |
r = requests.post(f"{ENV_BASE_URL}/reset", json={"task_id": task_id})
|
| 35 |
r.raise_for_status()
|
|
@@ -40,46 +31,35 @@ def env_step(sql: str) -> dict:
|
|
| 40 |
r.raise_for_status()
|
| 41 |
return r.json()
|
| 42 |
|
| 43 |
-
def env_state() -> dict:
|
| 44 |
-
r = requests.get(f"{ENV_BASE_URL}/state")
|
| 45 |
-
r.raise_for_status()
|
| 46 |
-
return r.json()
|
| 47 |
-
|
| 48 |
def wait_for_server(retries: int = 10, delay: float = 2.0):
|
| 49 |
-
"""Wait until the FastAPI server is ready."""
|
| 50 |
print("Waiting for environment server...")
|
| 51 |
for i in range(retries):
|
| 52 |
try:
|
| 53 |
-
r = requests.get(f"{ENV_BASE_URL}/
|
| 54 |
if r.status_code == 200:
|
| 55 |
-
print("Server is
|
| 56 |
return
|
| 57 |
-
except requests.ConnectionError:
|
| 58 |
pass
|
|
|
|
| 59 |
time.sleep(delay)
|
| 60 |
print("ERROR: Server did not start in time.")
|
| 61 |
sys.exit(1)
|
| 62 |
|
| 63 |
-
# ── LLM SQL Generator ─────────────────────────────────────────────────────────
|
| 64 |
-
|
| 65 |
def build_system_prompt() -> str:
|
| 66 |
return """You are an expert SQL analyst. Your job is to write correct SQLite queries.
|
| 67 |
|
| 68 |
Rules:
|
| 69 |
- Only write SELECT or WITH (CTE) statements. Never INSERT, UPDATE, DELETE, or DROP.
|
| 70 |
- Always match the exact column names specified in the task.
|
| 71 |
-
-
|
|
|
|
|
|
|
| 72 |
- Return ONLY the raw SQL query — no explanation, no markdown, no backticks.
|
| 73 |
- If a previous attempt scored less than 1.0, study the feedback and fix the query.
|
| 74 |
"""
|
| 75 |
|
| 76 |
-
def build_user_prompt(
|
| 77 |
-
task_description: str,
|
| 78 |
-
schema: str,
|
| 79 |
-
hint: str,
|
| 80 |
-
attempt: int,
|
| 81 |
-
previous_attempts: list,
|
| 82 |
-
) -> str:
|
| 83 |
prompt = f"""Task:
|
| 84 |
{task_description}
|
| 85 |
|
|
@@ -91,16 +71,21 @@ Hint: {hint}
|
|
| 91 |
Attempt number: {attempt}
|
| 92 |
"""
|
| 93 |
if previous_attempts:
|
| 94 |
-
prompt += "\
|
| 95 |
-
for
|
| 96 |
-
prompt += f"
|
| 97 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 98 |
prompt += "\nWrite the corrected SQL query now:"
|
| 99 |
return prompt
|
| 100 |
|
| 101 |
-
def ask_llm(task_description
|
| 102 |
-
attempt: int, previous_attempts: list) -> str:
|
| 103 |
-
"""Call the LLM and return a SQL string."""
|
| 104 |
messages = [
|
| 105 |
{"role": "system", "content": build_system_prompt()},
|
| 106 |
{"role": "user", "content": build_user_prompt(
|
|
@@ -114,22 +99,19 @@ def ask_llm(task_description: str, schema: str, hint: str,
|
|
| 114 |
max_tokens=512,
|
| 115 |
)
|
| 116 |
sql = response.choices[0].message.content.strip()
|
| 117 |
-
|
| 118 |
-
# Strip markdown fences if model wraps in ```sql ... ```
|
| 119 |
if sql.startswith("```"):
|
| 120 |
-
|
| 121 |
-
|
|
|
|
|
|
|
|
|
|
| 122 |
return sql
|
| 123 |
|
| 124 |
-
# ── Main Agent Loop ───────────────────────────────────────────────────────────
|
| 125 |
-
|
| 126 |
def solve_task(task_id: int) -> dict:
|
| 127 |
-
"""Run the agent on a single task. Returns final result dict."""
|
| 128 |
print(f"\n{'='*60}")
|
| 129 |
print(f"TASK {task_id}")
|
| 130 |
print('='*60)
|
| 131 |
|
| 132 |
-
# Reset environment
|
| 133 |
reset_resp = env_reset(task_id)
|
| 134 |
obs = reset_resp["observation"]
|
| 135 |
task_desc = obs["task_description"]
|
|
@@ -145,33 +127,47 @@ def solve_task(task_id: int) -> dict:
|
|
| 145 |
final_sql = ""
|
| 146 |
|
| 147 |
for attempt in range(1, MAX_ATTEMPTS + 1):
|
| 148 |
-
print(f"Attempt {attempt}/{MAX_ATTEMPTS}...")
|
| 149 |
sql = ask_llm(task_desc, schema, hint, attempt, previous_attempts)
|
| 150 |
-
print(f"
|
| 151 |
-
|
| 152 |
step_resp = env_step(sql)
|
| 153 |
-
reward
|
| 154 |
-
|
| 155 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 156 |
best_reward = max(best_reward, reward)
|
| 157 |
-
final_sql
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 161 |
break
|
| 162 |
-
|
| 163 |
-
|
|
|
|
|
|
|
| 164 |
|
| 165 |
return {
|
| 166 |
-
"task_id":
|
| 167 |
-
"difficulty":
|
| 168 |
-
"best_reward":
|
| 169 |
-
"attempts":
|
| 170 |
-
"final_sql":
|
| 171 |
-
"solved":
|
| 172 |
}
|
| 173 |
|
| 174 |
-
|
| 175 |
def main():
|
| 176 |
print("SQL Analyst OpenEnv — Baseline Inference Agent")
|
| 177 |
print(f"Model : {MODEL_NAME}")
|
|
@@ -182,26 +178,31 @@ def main():
|
|
| 182 |
|
| 183 |
results = []
|
| 184 |
for task_id in TASK_IDS:
|
| 185 |
-
|
| 186 |
-
results.append(
|
| 187 |
|
| 188 |
-
# ── Final Summary ─────────────────────────────────────────────────────────
|
| 189 |
print(f"\n{'='*60}")
|
| 190 |
print("FINAL RESULTS")
|
| 191 |
print('='*60)
|
| 192 |
|
| 193 |
total_score = 0.0
|
| 194 |
-
solved_count = 0
|
| 195 |
for r in results:
|
|
|
|
|
|
|
|
|
|
| 196 |
total_score += r["best_reward"]
|
| 197 |
-
if r["solved"]:
|
| 198 |
-
solved_count += 1
|
| 199 |
-
print(f"Task {r['task_id']} ({r['difficulty']}): Reward = {r['best_reward']:.2f} | Solved = {r['solved']}")
|
| 200 |
-
|
| 201 |
-
avg_score = total_score / len(results) if results else 0
|
| 202 |
-
print(f"\nTasks solved : {solved_count} / {len(results)}")
|
| 203 |
-
print(f"Average reward : {avg_score:.3f} / 1.000")
|
| 204 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 205 |
|
| 206 |
if __name__ == "__main__":
|
| 207 |
main()
|
|
|
|
| 1 |
"""
|
| 2 |
inference.py — Baseline AI agent for SQL Analyst OpenEnv
|
|
|
|
|
|
|
|
|
|
| 3 |
"""
|
| 4 |
import os
|
| 5 |
import sys
|
| 6 |
+
import json
|
| 7 |
import time
|
| 8 |
import requests
|
| 9 |
from openai import OpenAI
|
|
|
|
|
|
|
|
|
|
| 10 |
|
| 11 |
+
ENV_BASE_URL = "https://p-karthik-mohan-sql-analyst-env.hf.space"
|
| 12 |
+
MAX_ATTEMPTS = 5
|
| 13 |
+
TASK_IDS = [1, 2, 3, 4, 5]
|
| 14 |
|
| 15 |
+
API_BASE_URL = os.environ.get("API_BASE_URL", "https://api.groq.com/openai/v1")
|
| 16 |
+
MODEL_NAME = os.environ.get("MODEL_NAME", "llama-3.1-8b-instant")
|
| 17 |
+
HF_TOKEN = os.environ.get("HF_TOKEN", "")
|
|
|
|
|
|
|
| 18 |
|
| 19 |
client = OpenAI(
|
| 20 |
base_url=API_BASE_URL,
|
| 21 |
api_key=HF_TOKEN if HF_TOKEN else "no-key-needed",
|
| 22 |
)
|
| 23 |
|
|
|
|
|
|
|
| 24 |
def env_reset(task_id: int) -> dict:
|
| 25 |
r = requests.post(f"{ENV_BASE_URL}/reset", json={"task_id": task_id})
|
| 26 |
r.raise_for_status()
|
|
|
|
| 31 |
r.raise_for_status()
|
| 32 |
return r.json()
|
| 33 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
def wait_for_server(retries: int = 10, delay: float = 2.0):
|
|
|
|
| 35 |
print("Waiting for environment server...")
|
| 36 |
for i in range(retries):
|
| 37 |
try:
|
| 38 |
+
r = requests.get(f"{ENV_BASE_URL}/health", timeout=3)
|
| 39 |
if r.status_code == 200:
|
| 40 |
+
print("Server is ready.\n")
|
| 41 |
return
|
| 42 |
+
except requests.exceptions.ConnectionError:
|
| 43 |
pass
|
| 44 |
+
print(f" Not ready yet... ({i+1}/{retries})")
|
| 45 |
time.sleep(delay)
|
| 46 |
print("ERROR: Server did not start in time.")
|
| 47 |
sys.exit(1)
|
| 48 |
|
|
|
|
|
|
|
| 49 |
def build_system_prompt() -> str:
|
| 50 |
return """You are an expert SQL analyst. Your job is to write correct SQLite queries.
|
| 51 |
|
| 52 |
Rules:
|
| 53 |
- Only write SELECT or WITH (CTE) statements. Never INSERT, UPDATE, DELETE, or DROP.
|
| 54 |
- Always match the exact column names specified in the task.
|
| 55 |
+
- Always filter WHERE status = 'completed' unless told otherwise.
|
| 56 |
+
- Use STRFTIME('%Y', order_date) for year filtering in SQLite.
|
| 57 |
+
- RANK() OVER (...) is supported in SQLite 3.25+.
|
| 58 |
- Return ONLY the raw SQL query — no explanation, no markdown, no backticks.
|
| 59 |
- If a previous attempt scored less than 1.0, study the feedback and fix the query.
|
| 60 |
"""
|
| 61 |
|
| 62 |
+
def build_user_prompt(task_description, schema, hint, attempt, previous_attempts):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
prompt = f"""Task:
|
| 64 |
{task_description}
|
| 65 |
|
|
|
|
| 71 |
Attempt number: {attempt}
|
| 72 |
"""
|
| 73 |
if previous_attempts:
|
| 74 |
+
prompt += "\nYour previous attempts and their scores:\n"
|
| 75 |
+
for prev in previous_attempts[-3:]:
|
| 76 |
+
prompt += f"""
|
| 77 |
+
Attempt {prev['attempt']}:
|
| 78 |
+
SQL: {prev['sql']}
|
| 79 |
+
Reward: {prev['reward']} / 1.0
|
| 80 |
+
Columns expected : {prev['details'].get('expected_columns', [])}
|
| 81 |
+
Columns you gave : {prev['details'].get('agent_columns', [])}
|
| 82 |
+
Rows expected : {prev['details'].get('expected_row_count', '?')}
|
| 83 |
+
Rows you gave : {prev['details'].get('agent_row_count', '?')}
|
| 84 |
+
"""
|
| 85 |
prompt += "\nWrite the corrected SQL query now:"
|
| 86 |
return prompt
|
| 87 |
|
| 88 |
+
def ask_llm(task_description, schema, hint, attempt, previous_attempts):
|
|
|
|
|
|
|
| 89 |
messages = [
|
| 90 |
{"role": "system", "content": build_system_prompt()},
|
| 91 |
{"role": "user", "content": build_user_prompt(
|
|
|
|
| 99 |
max_tokens=512,
|
| 100 |
)
|
| 101 |
sql = response.choices[0].message.content.strip()
|
|
|
|
|
|
|
| 102 |
if sql.startswith("```"):
|
| 103 |
+
lines = sql.split("\n")
|
| 104 |
+
sql = "\n".join(
|
| 105 |
+
line for line in lines
|
| 106 |
+
if not line.strip().startswith("```")
|
| 107 |
+
).strip()
|
| 108 |
return sql
|
| 109 |
|
|
|
|
|
|
|
| 110 |
def solve_task(task_id: int) -> dict:
|
|
|
|
| 111 |
print(f"\n{'='*60}")
|
| 112 |
print(f"TASK {task_id}")
|
| 113 |
print('='*60)
|
| 114 |
|
|
|
|
| 115 |
reset_resp = env_reset(task_id)
|
| 116 |
obs = reset_resp["observation"]
|
| 117 |
task_desc = obs["task_description"]
|
|
|
|
| 127 |
final_sql = ""
|
| 128 |
|
| 129 |
for attempt in range(1, MAX_ATTEMPTS + 1):
|
| 130 |
+
print(f" Attempt {attempt}/{MAX_ATTEMPTS} — asking LLM...")
|
| 131 |
sql = ask_llm(task_desc, schema, hint, attempt, previous_attempts)
|
| 132 |
+
print(f" SQL: {sql[:120]}{'...' if len(sql) > 120 else ''}")
|
| 133 |
+
|
| 134 |
step_resp = env_step(sql)
|
| 135 |
+
reward = step_resp["reward"]
|
| 136 |
+
done = step_resp["done"]
|
| 137 |
+
details = step_resp["observation"].get("reward_breakdown", {})
|
| 138 |
+
|
| 139 |
+
print(f" Reward: {reward:.3f} "
|
| 140 |
+
f"(cols={details.get('column_score',0):.2f} "
|
| 141 |
+
f"rows={details.get('row_score',0):.2f} "
|
| 142 |
+
f"vals={details.get('value_score',0):.2f})")
|
| 143 |
+
|
| 144 |
best_reward = max(best_reward, reward)
|
| 145 |
+
final_sql = sql
|
| 146 |
+
|
| 147 |
+
previous_attempts.append({
|
| 148 |
+
"attempt": attempt,
|
| 149 |
+
"sql": sql,
|
| 150 |
+
"reward": reward,
|
| 151 |
+
"details": details,
|
| 152 |
+
})
|
| 153 |
+
|
| 154 |
+
if done:
|
| 155 |
+
print(f" PERFECT SCORE on attempt {attempt}!")
|
| 156 |
break
|
| 157 |
+
elif reward >= 0.8:
|
| 158 |
+
print(f" Score is close ({reward:.3f}). Trying to improve...")
|
| 159 |
+
else:
|
| 160 |
+
print(f" Score is low ({reward:.3f}). Refining query...")
|
| 161 |
|
| 162 |
return {
|
| 163 |
+
"task_id": task_id,
|
| 164 |
+
"difficulty": difficulty,
|
| 165 |
+
"best_reward": best_reward,
|
| 166 |
+
"attempts": len(previous_attempts),
|
| 167 |
+
"final_sql": final_sql,
|
| 168 |
+
"solved": best_reward >= 1.0,
|
| 169 |
}
|
| 170 |
|
|
|
|
| 171 |
def main():
|
| 172 |
print("SQL Analyst OpenEnv — Baseline Inference Agent")
|
| 173 |
print(f"Model : {MODEL_NAME}")
|
|
|
|
| 178 |
|
| 179 |
results = []
|
| 180 |
for task_id in TASK_IDS:
|
| 181 |
+
result = solve_task(task_id)
|
| 182 |
+
results.append(result)
|
| 183 |
|
|
|
|
| 184 |
print(f"\n{'='*60}")
|
| 185 |
print("FINAL RESULTS")
|
| 186 |
print('='*60)
|
| 187 |
|
| 188 |
total_score = 0.0
|
|
|
|
| 189 |
for r in results:
|
| 190 |
+
status = "SOLVED" if r["solved"] else f"best={r['best_reward']:.3f}"
|
| 191 |
+
print(f" Task {r['task_id']} ({r['difficulty']:6s}) {status} "
|
| 192 |
+
f"in {r['attempts']} attempt(s)")
|
| 193 |
total_score += r["best_reward"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 194 |
|
| 195 |
+
avg_score = total_score / len(results)
|
| 196 |
+
print(f"\n Average reward : {avg_score:.3f} / 1.000")
|
| 197 |
+
print(f" Tasks solved : {sum(1 for r in results if r['solved'])} / {len(results)}")
|
| 198 |
+
|
| 199 |
+
with open("results.json", "w") as f:
|
| 200 |
+
json.dump({
|
| 201 |
+
"results": results,
|
| 202 |
+
"avg_score": round(avg_score, 3),
|
| 203 |
+
"tasks_solved": sum(1 for r in results if r["solved"]),
|
| 204 |
+
}, f, indent=2)
|
| 205 |
+
print(f"\n Results saved to results.json")
|
| 206 |
|
| 207 |
if __name__ == "__main__":
|
| 208 |
main()
|
main.py
CHANGED
|
@@ -4,117 +4,152 @@ import json
|
|
| 4 |
import re
|
| 5 |
from datetime import datetime
|
| 6 |
from typing import Any, Optional
|
| 7 |
-
import uuid
|
| 8 |
|
| 9 |
from fastapi import FastAPI, HTTPException
|
| 10 |
-
from pydantic import BaseModel
|
| 11 |
-
import gradio as gr
|
| 12 |
-
from ui import build_ui
|
| 13 |
|
| 14 |
-
# ── Config ────────────────────────────────────────────────────────────────────
|
| 15 |
DB_PATH = os.path.join("data", "ecommerce.db")
|
| 16 |
-
|
| 17 |
-
api_app = FastAPI(
|
| 18 |
-
title="SQL Analyst OpenEnv API",
|
| 19 |
-
version="1.1.0",
|
| 20 |
-
docs_url="/docs",
|
| 21 |
-
redoc_url="/redoc",
|
| 22 |
-
description="A real-world benchmark environment for AI agents writing SQL. Use `/reset` to select a task, `/step` to submit a query and receive partial-credit feedback, and `/state` to track history."
|
| 23 |
-
)
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
# ── Pydantic Models ───────────────────────────────────────────────────────────
|
| 27 |
|
| 28 |
class StepRequest(BaseModel):
|
| 29 |
-
|
| 30 |
-
action: str = Field(..., description="A valid SQLite SELECT or WITH statement.", examples=["SELECT COUNT(*) AS total_orders FROM orders;"])
|
| 31 |
|
| 32 |
class StepResponse(BaseModel):
|
| 33 |
-
observation: dict
|
| 34 |
-
reward: float
|
| 35 |
-
done: bool
|
| 36 |
-
info: dict
|
| 37 |
|
| 38 |
class ResetRequest(BaseModel):
|
| 39 |
-
task_id: int
|
| 40 |
-
session_id: str = Field("default", description="Unique session ID.")
|
| 41 |
|
| 42 |
class ResetResponse(BaseModel):
|
| 43 |
-
observation: dict
|
| 44 |
-
info: dict
|
| 45 |
-
|
| 46 |
-
class StateRequest(BaseModel):
|
| 47 |
-
session_id: str = Field("default", description="Unique session ID.")
|
| 48 |
|
| 49 |
class StateResponse(BaseModel):
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
history: list = Field(description="Log of all queries executed.")
|
| 57 |
-
|
| 58 |
-
# ── Task Definitions ──────────────────────────────────────────────────────────
|
| 59 |
|
| 60 |
TASKS = {
|
| 61 |
1: {
|
| 62 |
-
"description":
|
|
|
|
|
|
|
|
|
|
| 63 |
"difficulty": "easy",
|
| 64 |
"hint": "Use COUNT with WHERE filters on status and order_date",
|
| 65 |
-
"answer_query": "
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
},
|
| 67 |
2: {
|
| 68 |
-
"description":
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
"difficulty": "medium",
|
| 70 |
"hint": "JOIN orders with customers, GROUP BY customer, filter completed, ORDER and LIMIT",
|
| 71 |
-
"answer_query": "
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
},
|
| 73 |
3: {
|
| 74 |
-
"description":
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 75 |
"difficulty": "hard",
|
| 76 |
"hint": "Use SUM with GROUP BY inside a CTE, then apply RANK() OVER (ORDER BY ...) on the result",
|
| 77 |
-
"answer_query": "
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 78 |
},
|
| 79 |
4: {
|
| 80 |
-
"description":
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
"difficulty": "medium",
|
| 82 |
-
"hint": "Use GROUP BY with HAVING COUNT(...) > 2
|
| 83 |
-
"answer_query": "
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 84 |
},
|
| 85 |
5: {
|
| 86 |
-
"description":
|
|
|
|
|
|
|
|
|
|
| 87 |
"difficulty": "hard",
|
| 88 |
-
"hint": "Use INTERSECT on two queries
|
| 89 |
-
"answer_query": "
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
},
|
| 91 |
}
|
| 92 |
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
"attempts": 0, "best_reward": 0.0, "history": []
|
| 103 |
-
}
|
| 104 |
-
return sessions[sid]
|
| 105 |
-
|
| 106 |
-
# ── Database Helpers ──────────────────────────────────────────────────────────
|
| 107 |
-
|
| 108 |
-
def progress_handler():
|
| 109 |
-
raise sqlite3.OperationalError("Query execution aborted: Timed out or exceeded instruction limits. Hint: Too complex CROSS JOIN?")
|
| 110 |
|
| 111 |
def get_connection():
|
| 112 |
if not os.path.exists(DB_PATH):
|
| 113 |
-
raise HTTPException(
|
|
|
|
|
|
|
|
|
|
| 114 |
conn = sqlite3.connect(DB_PATH)
|
| 115 |
conn.row_factory = sqlite3.Row
|
| 116 |
-
# Security feature: Prevent DOS
|
| 117 |
-
conn.set_progress_handler(progress_handler, 500000)
|
| 118 |
return conn
|
| 119 |
|
| 120 |
def get_schema_info() -> str:
|
|
@@ -142,26 +177,24 @@ def run_query(sql: str) -> tuple[list[dict], list[str]]:
|
|
| 142 |
conn.close()
|
| 143 |
return rows, columns
|
| 144 |
|
| 145 |
-
def compute_expected(
|
| 146 |
-
session = get_session(sid)
|
| 147 |
task = session["task"]
|
| 148 |
rows, columns = run_query(task["answer_query"])
|
| 149 |
session["expected_rows"] = rows
|
| 150 |
session["expected_columns"] = columns
|
| 151 |
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
def compute_reward(sid: str, agent_rows: list[dict], agent_cols: list[str]) -> tuple[float, dict]:
|
| 155 |
-
session = get_session(sid)
|
| 156 |
expected_rows = session["expected_rows"]
|
| 157 |
expected_cols = session["expected_columns"]
|
| 158 |
details = {}
|
| 159 |
|
| 160 |
-
agent_cols_lower
|
| 161 |
expected_cols_lower = [c.lower() for c in expected_cols]
|
| 162 |
col_matches = sum(1 for c in expected_cols_lower if c in agent_cols_lower)
|
| 163 |
col_score = (col_matches / len(expected_cols_lower)) * 0.30 if expected_cols_lower else 0.0
|
| 164 |
details["column_score"] = round(col_score, 3)
|
|
|
|
|
|
|
| 165 |
|
| 166 |
expected_count = len(expected_rows)
|
| 167 |
agent_count = len(agent_rows)
|
|
@@ -171,14 +204,19 @@ def compute_reward(sid: str, agent_rows: list[dict], agent_cols: list[str]) -> t
|
|
| 171 |
row_ratio = min(agent_count, expected_count) / max(agent_count, expected_count)
|
| 172 |
row_score = row_ratio * 0.30
|
| 173 |
details["row_score"] = round(row_score, 3)
|
|
|
|
|
|
|
| 174 |
|
| 175 |
if not expected_rows or not agent_rows:
|
| 176 |
value_score = 0.0
|
| 177 |
else:
|
| 178 |
def normalize(v):
|
| 179 |
-
if v is None:
|
| 180 |
-
|
| 181 |
-
|
|
|
|
|
|
|
|
|
|
| 182 |
|
| 183 |
matched_cells = 0
|
| 184 |
total_cells = len(expected_rows) * len(expected_cols_lower)
|
|
@@ -202,82 +240,121 @@ def compute_reward(sid: str, agent_rows: list[dict], agent_cols: list[str]) -> t
|
|
| 202 |
details["total_reward"] = total
|
| 203 |
return total, details
|
| 204 |
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
@api_app.post("/reset", response_model=ResetResponse, tags=["Agent Environment"], summary="Load Task", description="Initializes the environment with a random task, or resets the current task, returning the description, hints, and database schema.")
|
| 208 |
def reset(req: ResetRequest):
|
| 209 |
-
if req.task_id not in TASKS:
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
session["
|
| 213 |
-
session["
|
|
|
|
| 214 |
session["best_reward"] = 0.0
|
| 215 |
-
session["history"]
|
| 216 |
-
|
|
|
|
|
|
|
|
|
|
| 217 |
observation = {
|
| 218 |
"task_id": req.task_id,
|
| 219 |
"difficulty": session["task"]["difficulty"],
|
| 220 |
"task_description": session["task"]["description"],
|
| 221 |
-
"schema":
|
| 222 |
"hint": session["task"]["hint"],
|
| 223 |
}
|
| 224 |
-
return ResetResponse(
|
|
|
|
|
|
|
|
|
|
| 225 |
|
| 226 |
-
@
|
| 227 |
def step(req: StepRequest):
|
| 228 |
-
session
|
| 229 |
-
|
|
|
|
| 230 |
session["attempts"] += 1
|
| 231 |
sql = req.action.strip()
|
| 232 |
|
| 233 |
if not re.match(r"^\s*(SELECT|WITH)\b", sql, re.IGNORECASE):
|
| 234 |
return StepResponse(
|
| 235 |
-
observation={"error": "Only SELECT or WITH allowed."},
|
| 236 |
-
|
|
|
|
|
|
|
| 237 |
)
|
| 238 |
|
| 239 |
try:
|
| 240 |
agent_rows, agent_cols = run_query(sql)
|
| 241 |
except Exception as e:
|
| 242 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 243 |
return StepResponse(
|
| 244 |
-
observation={"error": str(e), "sql_submitted": sql},
|
| 245 |
-
|
|
|
|
|
|
|
| 246 |
)
|
| 247 |
|
| 248 |
-
reward, details = compute_reward(
|
| 249 |
session["best_reward"] = max(session["best_reward"], reward)
|
| 250 |
done = reward >= 1.0
|
| 251 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 252 |
|
| 253 |
observation = {
|
| 254 |
-
"task_id":
|
| 255 |
-
"
|
|
|
|
|
|
|
|
|
|
| 256 |
"reward_breakdown": details,
|
| 257 |
}
|
|
|
|
| 258 |
return StepResponse(
|
| 259 |
-
observation=observation,
|
| 260 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 261 |
)
|
| 262 |
|
| 263 |
-
@
|
| 264 |
-
def state(
|
| 265 |
-
session
|
|
|
|
|
|
|
| 266 |
return StateResponse(
|
| 267 |
-
session_id=req.session_id,
|
| 268 |
task_id=session["task_id"],
|
| 269 |
-
task_description=session["task"]["description"]
|
| 270 |
schema_info=get_schema_info(),
|
| 271 |
attempts=session["attempts"],
|
| 272 |
best_reward=session["best_reward"],
|
| 273 |
history=session["history"],
|
| 274 |
)
|
| 275 |
|
| 276 |
-
@
|
| 277 |
def root():
|
| 278 |
-
return {
|
| 279 |
-
|
| 280 |
-
|
|
|
|
|
|
|
|
|
|
| 281 |
|
| 282 |
-
|
| 283 |
-
|
|
|
|
|
|
| 4 |
import re
|
| 5 |
from datetime import datetime
|
| 6 |
from typing import Any, Optional
|
|
|
|
| 7 |
|
| 8 |
from fastapi import FastAPI, HTTPException
|
| 9 |
+
from pydantic import BaseModel
|
|
|
|
|
|
|
| 10 |
|
|
|
|
| 11 |
DB_PATH = os.path.join("data", "ecommerce.db")
|
| 12 |
+
app = FastAPI(title="SQL Analyst OpenEnv", version="1.0.0")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
|
| 14 |
class StepRequest(BaseModel):
|
| 15 |
+
action: str
|
|
|
|
| 16 |
|
| 17 |
class StepResponse(BaseModel):
|
| 18 |
+
observation: dict
|
| 19 |
+
reward: float
|
| 20 |
+
done: bool
|
| 21 |
+
info: dict
|
| 22 |
|
| 23 |
class ResetRequest(BaseModel):
|
| 24 |
+
task_id: int
|
|
|
|
| 25 |
|
| 26 |
class ResetResponse(BaseModel):
|
| 27 |
+
observation: dict
|
| 28 |
+
info: dict
|
|
|
|
|
|
|
|
|
|
| 29 |
|
| 30 |
class StateResponse(BaseModel):
|
| 31 |
+
task_id: int
|
| 32 |
+
task_description: str
|
| 33 |
+
schema_info: str
|
| 34 |
+
attempts: int
|
| 35 |
+
best_reward: float
|
| 36 |
+
history: list
|
|
|
|
|
|
|
|
|
|
| 37 |
|
| 38 |
TASKS = {
|
| 39 |
1: {
|
| 40 |
+
"description": (
|
| 41 |
+
"Find the total number of completed orders placed in the year 2024. "
|
| 42 |
+
"Return a single number with column name: total_orders"
|
| 43 |
+
),
|
| 44 |
"difficulty": "easy",
|
| 45 |
"hint": "Use COUNT with WHERE filters on status and order_date",
|
| 46 |
+
"answer_query": """
|
| 47 |
+
SELECT COUNT(*) AS total_orders
|
| 48 |
+
FROM orders
|
| 49 |
+
WHERE status = 'completed'
|
| 50 |
+
AND order_date LIKE '2024%'
|
| 51 |
+
""",
|
| 52 |
},
|
| 53 |
2: {
|
| 54 |
+
"description": (
|
| 55 |
+
"Find the top 5 customers by total revenue (sum of total_amount for completed orders only). "
|
| 56 |
+
"Return columns: first_name, last_name, total_revenue. "
|
| 57 |
+
"Order by total_revenue descending."
|
| 58 |
+
),
|
| 59 |
"difficulty": "medium",
|
| 60 |
"hint": "JOIN orders with customers, GROUP BY customer, filter completed, ORDER and LIMIT",
|
| 61 |
+
"answer_query": """
|
| 62 |
+
SELECT c.first_name, c.last_name,
|
| 63 |
+
ROUND(SUM(o.total_amount), 2) AS total_revenue
|
| 64 |
+
FROM orders o
|
| 65 |
+
JOIN customers c ON o.customer_id = c.customer_id
|
| 66 |
+
WHERE o.status = 'completed'
|
| 67 |
+
GROUP BY o.customer_id
|
| 68 |
+
ORDER BY total_revenue DESC
|
| 69 |
+
LIMIT 5
|
| 70 |
+
""",
|
| 71 |
},
|
| 72 |
3: {
|
| 73 |
+
"description": (
|
| 74 |
+
"For each product category, calculate the total revenue (completed orders only) "
|
| 75 |
+
"and rank categories by revenue using a window function. "
|
| 76 |
+
"Return columns: category, total_revenue, revenue_rank. "
|
| 77 |
+
"Order by revenue_rank ascending."
|
| 78 |
+
),
|
| 79 |
"difficulty": "hard",
|
| 80 |
"hint": "Use SUM with GROUP BY inside a CTE, then apply RANK() OVER (ORDER BY ...) on the result",
|
| 81 |
+
"answer_query": """
|
| 82 |
+
WITH category_revenue AS (
|
| 83 |
+
SELECT p.category,
|
| 84 |
+
SUM(o.total_amount) AS total_revenue
|
| 85 |
+
FROM orders o
|
| 86 |
+
JOIN products p ON o.product_id = p.product_id
|
| 87 |
+
WHERE o.status = 'completed'
|
| 88 |
+
GROUP BY p.category
|
| 89 |
+
)
|
| 90 |
+
SELECT category,
|
| 91 |
+
total_revenue,
|
| 92 |
+
RANK() OVER (ORDER BY total_revenue DESC) AS revenue_rank
|
| 93 |
+
FROM category_revenue
|
| 94 |
+
ORDER BY revenue_rank ASC
|
| 95 |
+
""",
|
| 96 |
},
|
| 97 |
4: {
|
| 98 |
+
"description": (
|
| 99 |
+
"Find the average price of products in each category, "
|
| 100 |
+
"but only for categories that have more than 2 products. "
|
| 101 |
+
"Return columns: category, avg_price."
|
| 102 |
+
),
|
| 103 |
"difficulty": "medium",
|
| 104 |
+
"hint": "Use GROUP BY with HAVING COUNT(...) > 2",
|
| 105 |
+
"answer_query": """
|
| 106 |
+
SELECT category, ROUND(AVG(price), 2) AS avg_price
|
| 107 |
+
FROM products
|
| 108 |
+
GROUP BY category
|
| 109 |
+
HAVING COUNT(product_id) > 2
|
| 110 |
+
""",
|
| 111 |
},
|
| 112 |
5: {
|
| 113 |
+
"description": (
|
| 114 |
+
"Identify customers who have ordered products from both the Electronics "
|
| 115 |
+
"and Clothing categories. Return columns: customer_id, first_name."
|
| 116 |
+
),
|
| 117 |
"difficulty": "hard",
|
| 118 |
+
"hint": "Use INTERSECT on two queries filtering by category",
|
| 119 |
+
"answer_query": """
|
| 120 |
+
SELECT DISTINCT c.customer_id, c.first_name
|
| 121 |
+
FROM customers c
|
| 122 |
+
JOIN orders o ON c.customer_id = o.customer_id
|
| 123 |
+
JOIN products p ON o.product_id = p.product_id
|
| 124 |
+
WHERE p.category = 'Electronics'
|
| 125 |
+
INTERSECT
|
| 126 |
+
SELECT DISTINCT c.customer_id, c.first_name
|
| 127 |
+
FROM customers c
|
| 128 |
+
JOIN orders o ON c.customer_id = o.customer_id
|
| 129 |
+
JOIN products p ON o.product_id = p.product_id
|
| 130 |
+
WHERE p.category = 'Clothing'
|
| 131 |
+
""",
|
| 132 |
},
|
| 133 |
}
|
| 134 |
|
| 135 |
+
session = {
|
| 136 |
+
"task_id": None,
|
| 137 |
+
"task": None,
|
| 138 |
+
"expected_rows": None,
|
| 139 |
+
"expected_columns": None,
|
| 140 |
+
"attempts": 0,
|
| 141 |
+
"best_reward": 0.0,
|
| 142 |
+
"history": [],
|
| 143 |
+
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
|
| 145 |
def get_connection():
|
| 146 |
if not os.path.exists(DB_PATH):
|
| 147 |
+
raise HTTPException(
|
| 148 |
+
status_code=500,
|
| 149 |
+
detail=f"Database not found at {DB_PATH}. Run seed.py first."
|
| 150 |
+
)
|
| 151 |
conn = sqlite3.connect(DB_PATH)
|
| 152 |
conn.row_factory = sqlite3.Row
|
|
|
|
|
|
|
| 153 |
return conn
|
| 154 |
|
| 155 |
def get_schema_info() -> str:
|
|
|
|
| 177 |
conn.close()
|
| 178 |
return rows, columns
|
| 179 |
|
| 180 |
+
def compute_expected():
|
|
|
|
| 181 |
task = session["task"]
|
| 182 |
rows, columns = run_query(task["answer_query"])
|
| 183 |
session["expected_rows"] = rows
|
| 184 |
session["expected_columns"] = columns
|
| 185 |
|
| 186 |
+
def compute_reward(agent_rows: list[dict], agent_cols: list[str]) -> tuple[float, dict]:
|
|
|
|
|
|
|
|
|
|
| 187 |
expected_rows = session["expected_rows"]
|
| 188 |
expected_cols = session["expected_columns"]
|
| 189 |
details = {}
|
| 190 |
|
| 191 |
+
agent_cols_lower = [c.lower() for c in agent_cols]
|
| 192 |
expected_cols_lower = [c.lower() for c in expected_cols]
|
| 193 |
col_matches = sum(1 for c in expected_cols_lower if c in agent_cols_lower)
|
| 194 |
col_score = (col_matches / len(expected_cols_lower)) * 0.30 if expected_cols_lower else 0.0
|
| 195 |
details["column_score"] = round(col_score, 3)
|
| 196 |
+
details["expected_columns"] = expected_cols
|
| 197 |
+
details["agent_columns"] = agent_cols
|
| 198 |
|
| 199 |
expected_count = len(expected_rows)
|
| 200 |
agent_count = len(agent_rows)
|
|
|
|
| 204 |
row_ratio = min(agent_count, expected_count) / max(agent_count, expected_count)
|
| 205 |
row_score = row_ratio * 0.30
|
| 206 |
details["row_score"] = round(row_score, 3)
|
| 207 |
+
details["expected_row_count"] = expected_count
|
| 208 |
+
details["agent_row_count"] = agent_count
|
| 209 |
|
| 210 |
if not expected_rows or not agent_rows:
|
| 211 |
value_score = 0.0
|
| 212 |
else:
|
| 213 |
def normalize(v):
|
| 214 |
+
if v is None:
|
| 215 |
+
return ""
|
| 216 |
+
try:
|
| 217 |
+
return str(round(float(v), 1))
|
| 218 |
+
except (ValueError, TypeError):
|
| 219 |
+
return str(v).strip().lower()
|
| 220 |
|
| 221 |
matched_cells = 0
|
| 222 |
total_cells = len(expected_rows) * len(expected_cols_lower)
|
|
|
|
| 240 |
details["total_reward"] = total
|
| 241 |
return total, details
|
| 242 |
|
| 243 |
+
@app.post("/reset", response_model=ResetResponse)
|
|
|
|
|
|
|
| 244 |
def reset(req: ResetRequest):
|
| 245 |
+
if req.task_id not in TASKS:
|
| 246 |
+
raise HTTPException(status_code=400, detail="task_id must be 1, 2, 3, 4, or 5")
|
| 247 |
+
|
| 248 |
+
session["task_id"] = req.task_id
|
| 249 |
+
session["task"] = TASKS[req.task_id]
|
| 250 |
+
session["attempts"] = 0
|
| 251 |
session["best_reward"] = 0.0
|
| 252 |
+
session["history"] = []
|
| 253 |
+
|
| 254 |
+
compute_expected()
|
| 255 |
+
|
| 256 |
+
schema = get_schema_info()
|
| 257 |
observation = {
|
| 258 |
"task_id": req.task_id,
|
| 259 |
"difficulty": session["task"]["difficulty"],
|
| 260 |
"task_description": session["task"]["description"],
|
| 261 |
+
"schema": schema,
|
| 262 |
"hint": session["task"]["hint"],
|
| 263 |
}
|
| 264 |
+
return ResetResponse(
|
| 265 |
+
observation=observation,
|
| 266 |
+
info={"message": f"Task {req.task_id} loaded. Use POST /step with your SQL query."}
|
| 267 |
+
)
|
| 268 |
|
| 269 |
+
@app.post("/step", response_model=StepResponse)
|
| 270 |
def step(req: StepRequest):
|
| 271 |
+
if session["task_id"] is None:
|
| 272 |
+
raise HTTPException(status_code=400, detail="Call /reset first to load a task.")
|
| 273 |
+
|
| 274 |
session["attempts"] += 1
|
| 275 |
sql = req.action.strip()
|
| 276 |
|
| 277 |
if not re.match(r"^\s*(SELECT|WITH)\b", sql, re.IGNORECASE):
|
| 278 |
return StepResponse(
|
| 279 |
+
observation={"error": "Only SELECT or WITH (CTE) statements are allowed."},
|
| 280 |
+
reward=0.0,
|
| 281 |
+
done=False,
|
| 282 |
+
info={"attempt": session["attempts"], "message": "Rejected: not a SELECT/WITH query."}
|
| 283 |
)
|
| 284 |
|
| 285 |
try:
|
| 286 |
agent_rows, agent_cols = run_query(sql)
|
| 287 |
except Exception as e:
|
| 288 |
+
entry = {
|
| 289 |
+
"attempt": session["attempts"],
|
| 290 |
+
"sql": sql,
|
| 291 |
+
"reward": 0.0,
|
| 292 |
+
"error": str(e),
|
| 293 |
+
}
|
| 294 |
+
session["history"].append(entry)
|
| 295 |
return StepResponse(
|
| 296 |
+
observation={"error": str(e), "sql_submitted": sql},
|
| 297 |
+
reward=0.0,
|
| 298 |
+
done=False,
|
| 299 |
+
info={"attempt": session["attempts"], "message": "SQL execution error."}
|
| 300 |
)
|
| 301 |
|
| 302 |
+
reward, details = compute_reward(agent_rows, agent_cols)
|
| 303 |
session["best_reward"] = max(session["best_reward"], reward)
|
| 304 |
done = reward >= 1.0
|
| 305 |
+
|
| 306 |
+
entry = {
|
| 307 |
+
"attempt": session["attempts"],
|
| 308 |
+
"sql": sql,
|
| 309 |
+
"reward": reward,
|
| 310 |
+
"details": details,
|
| 311 |
+
"timestamp": datetime.now().isoformat(),
|
| 312 |
+
}
|
| 313 |
+
session["history"].append(entry)
|
| 314 |
|
| 315 |
observation = {
|
| 316 |
+
"task_id": session["task_id"],
|
| 317 |
+
"task_description": session["task"]["description"],
|
| 318 |
+
"sql_submitted": sql,
|
| 319 |
+
"result_preview": agent_rows[:5],
|
| 320 |
+
"result_row_count": len(agent_rows),
|
| 321 |
"reward_breakdown": details,
|
| 322 |
}
|
| 323 |
+
|
| 324 |
return StepResponse(
|
| 325 |
+
observation=observation,
|
| 326 |
+
reward=reward,
|
| 327 |
+
done=done,
|
| 328 |
+
info={
|
| 329 |
+
"attempt": session["attempts"],
|
| 330 |
+
"best_reward": session["best_reward"],
|
| 331 |
+
"message": "Perfect score! Task complete." if done else "Keep refining your query.",
|
| 332 |
+
}
|
| 333 |
)
|
| 334 |
|
| 335 |
+
@app.get("/state", response_model=StateResponse)
|
| 336 |
+
def state():
|
| 337 |
+
if session["task_id"] is None:
|
| 338 |
+
raise HTTPException(status_code=400, detail="No active task. Call /reset first.")
|
| 339 |
+
|
| 340 |
return StateResponse(
|
|
|
|
| 341 |
task_id=session["task_id"],
|
| 342 |
+
task_description=session["task"]["description"],
|
| 343 |
schema_info=get_schema_info(),
|
| 344 |
attempts=session["attempts"],
|
| 345 |
best_reward=session["best_reward"],
|
| 346 |
history=session["history"],
|
| 347 |
)
|
| 348 |
|
| 349 |
+
@app.get("/")
|
| 350 |
def root():
|
| 351 |
+
return {
|
| 352 |
+
"name": "SQL Analyst OpenEnv",
|
| 353 |
+
"version": "1.0.0",
|
| 354 |
+
"tasks": {k: {"difficulty": v["difficulty"], "description": v["description"]} for k, v in TASKS.items()},
|
| 355 |
+
"endpoints": ["/reset", "/step", "/state"],
|
| 356 |
+
}
|
| 357 |
|
| 358 |
+
@app.get("/health")
|
| 359 |
+
def health():
|
| 360 |
+
return {"status": "ok", "db_exists": os.path.exists(DB_PATH)}
|