Initial submission

Dockerfile

@@ -0,0 +1,28 @@
+FROM python:3.11-slim
+
+# Set working directory
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    gcc \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements first (better Docker layer caching)
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy all project files
+COPY . .
+
+# Create data directory
+RUN mkdir -p data
+
+# Seed the database at build time so it's ready instantly
+RUN python seed.py
+
+# Hugging Face Spaces uses port 7860
+EXPOSE 7860
+
+# Start the FastAPI server
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

inference.py

@@ -0,0 +1,264 @@
+"""
+inference.py — Baseline AI agent for SQL Analyst OpenEnv
+---------------------------------------------------------
+Uses the OpenAI client (pointed at any compatible LLM via API_BASE_URL)
+to solve all 3 tasks by interacting with the running FastAPI environment.
+
+Environment variables required:
+  API_BASE_URL  — LLM API base URL  (e.g. https://api.openai.com/v1)
+  MODEL_NAME    — model to use       (e.g. gpt-4o-mini)
+  HF_TOKEN      — Hugging Face token (used as the API key)
+
+Usage:
+  python inference.py
+"""
+
+import os
+import sys
+import json
+import time
+import requests
+from openai import OpenAI
+
+# ── Configuration ─────────────────────────────────────────────────────────────
+
+ENV_BASE_URL = "http://127.0.0.1:7860"   # where FastAPI server is running
+MAX_ATTEMPTS = 5                         # max SQL attempts per task
+TASK_IDS     = [1, 2, 3]                # tasks to solve
+
+API_BASE_URL = "https://api.groq.com/openai/v1"
+MODEL_NAME   = "llama-3.1-8b-instant"
+HF_TOKEN     = "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"
+
+# ── OpenAI Client ─────────────────────────────────────────────────────────────
+
+client = OpenAI(
+    base_url=API_BASE_URL,
+    api_key=HF_TOKEN if HF_TOKEN else "no-key-needed",
+)
+
+# ── Environment API helpers ───────────────────────────────────────────────────
+
+def env_reset(task_id: int) -> dict:
+    r = requests.post(f"{ENV_BASE_URL}/reset", json={"task_id": task_id})
+    r.raise_for_status()
+    return r.json()
+
+
+def env_step(sql: str) -> dict:
+    r = requests.post(f"{ENV_BASE_URL}/step", json={"action": sql})
+    r.raise_for_status()
+    return r.json()
+
+
+def env_state() -> dict:
+    r = requests.get(f"{ENV_BASE_URL}/state")
+    r.raise_for_status()
+    return r.json()
+
+
+def wait_for_server(retries: int = 10, delay: float = 2.0):
+    """Wait until the FastAPI server is ready."""
+    print("Waiting for environment server...")
+    for i in range(retries):
+        try:
+            r = requests.get(f"{ENV_BASE_URL}/health", timeout=3)
+            if r.status_code == 200:
+                print("Server is ready.\n")
+                return
+        except requests.exceptions.ConnectionError:
+            pass
+        print(f"  Not ready yet, retrying in {delay}s... ({i+1}/{retries})")
+        time.sleep(delay)
+    print("ERROR: Server did not start in time. Is uvicorn running?")
+    sys.exit(1)
+
+# ── LLM SQL Generator ─────────────────────────────────────────────────────────
+
+def build_system_prompt() -> str:
+    return """You are an expert SQL analyst. Your job is to write correct SQLite queries.
+
+Rules:
+- Only write SELECT or WITH (CTE) statements. Never INSERT, UPDATE, DELETE, or DROP.
+- Always match the exact column names specified in the task.
+- Use proper SQLite syntax. RANK() OVER (...) is supported in SQLite 3.25+.
+- Return ONLY the raw SQL query — no explanation, no markdown, no backticks.
+- If a previous attempt scored less than 1.0, study the feedback and fix the query.
+"""
+
+
+def build_user_prompt(
+    task_description: str,
+    schema: str,
+    hint: str,
+    attempt: int,
+    previous_attempts: list,
+) -> str:
+    prompt = f"""Task:
+{task_description}
+
+Database schema:
+{schema}
+
+Hint: {hint}
+
+Attempt number: {attempt}
+"""
+    if previous_attempts:
+        prompt += "\nYour previous attempts and their scores:\n"
+        for prev in previous_attempts[-3:]:   # show last 3 only
+            prompt += f"""
+  Attempt {prev['attempt']}:
+    SQL:    {prev['sql']}
+    Reward: {prev['reward']} / 1.0
+    Columns expected : {prev['details'].get('expected_columns', [])}
+    Columns you gave : {prev['details'].get('agent_columns', [])}
+    Rows expected    : {prev['details'].get('expected_row_count', '?')}
+    Rows you gave    : {prev['details'].get('agent_row_count', '?')}
+    Sub-scores       : columns={prev['details'].get('column_score', 0):.2f}  rows={prev['details'].get('row_score', 0):.2f}  values={prev['details'].get('value_score', 0):.2f}
+"""
+    prompt += "\nWrite the corrected SQL query now:"
+    return prompt
+
+
+def ask_llm(task_description: str, schema: str, hint: str,
+            attempt: int, previous_attempts: list) -> str:
+    """Call the LLM and return a SQL string."""
+    messages = [
+        {"role": "system", "content": build_system_prompt()},
+        {"role": "user",   "content": build_user_prompt(
+            task_description, schema, hint, attempt, previous_attempts
+        )},
+    ]
+    response = client.chat.completions.create(
+        model=MODEL_NAME,
+        messages=messages,
+        temperature=0.0,       # deterministic — we want correct SQL, not creative SQL
+        max_tokens=512,
+    )
+    sql = response.choices[0].message.content.strip()
+
+    # Strip markdown fences if model wraps in ```sql ... ```
+    if sql.startswith("```"):
+        lines = sql.split("\n")
+        sql = "\n".join(
+            line for line in lines
+            if not line.strip().startswith("```")
+        ).strip()
+
+    return sql
+
+# ── Main Agent Loop ───────────────────────────────────────────────────────────
+
+def solve_task(task_id: int) -> dict:
+    """Run the agent on a single task. Returns final result dict."""
+    print(f"\n{'='*60}")
+    print(f"TASK {task_id}")
+    print('='*60)
+
+    # Reset environment
+    reset_resp = env_reset(task_id)
+    obs        = reset_resp["observation"]
+    task_desc  = obs["task_description"]
+    schema     = obs["schema"]
+    hint       = obs["hint"]
+    difficulty = obs["difficulty"]
+
+    print(f"Difficulty : {difficulty.upper()}")
+    print(f"Task       : {task_desc}\n")
+
+    previous_attempts = []
+    best_reward = 0.0
+    final_sql   = ""
+
+    for attempt in range(1, MAX_ATTEMPTS + 1):
+        print(f"  Attempt {attempt}/{MAX_ATTEMPTS} — asking LLM...")
+
+        # Get SQL from LLM
+        sql = ask_llm(task_desc, schema, hint, attempt, previous_attempts)
+        print(f"  SQL: {sql[:120]}{'...' if len(sql) > 120 else ''}")
+
+        # Submit to environment
+        step_resp = env_step(sql)
+        reward    = step_resp["reward"]
+        done      = step_resp["done"]
+        details   = step_resp["observation"].get("reward_breakdown", {})
+
+        print(f"  Reward: {reward:.3f}  "
+              f"(cols={details.get('column_score',0):.2f}  "
+              f"rows={details.get('row_score',0):.2f}  "
+              f"vals={details.get('value_score',0):.2f})")
+
+        best_reward = max(best_reward, reward)
+        final_sql   = sql
+
+        # Store attempt for LLM feedback
+        previous_attempts.append({
+            "attempt": attempt,
+            "sql":     sql,
+            "reward":  reward,
+            "details": details,
+        })
+
+        if done:
+            print(f"  PERFECT SCORE on attempt {attempt}!")
+            break
+        elif reward >= 0.8:
+            print(f"  Score is close ({reward:.3f}). Trying to improve...")
+        else:
+            print(f"  Score is low ({reward:.3f}). Refining query...")
+
+    return {
+        "task_id":      task_id,
+        "difficulty":   difficulty,
+        "best_reward":  best_reward,
+        "attempts":     len(previous_attempts),
+        "final_sql":    final_sql,
+        "solved":       best_reward >= 1.0,
+    }
+
+
+def main():
+    print("SQL Analyst OpenEnv — Baseline Inference Agent")
+    print(f"Model      : {MODEL_NAME}")
+    print(f"API Base   : {API_BASE_URL}")
+    print(f"Env Server : {ENV_BASE_URL}")
+
+    wait_for_server()
+
+    results = []
+    for task_id in TASK_IDS:
+        result = solve_task(task_id)
+        results.append(result)
+
+    # ── Final Summary ─────────────────────────────────────────────────────────
+    print(f"\n{'='*60}")
+    print("FINAL RESULTS")
+    print('='*60)
+
+    total_score = 0.0
+    for r in results:
+        status = "SOLVED" if r["solved"] else f"best={r['best_reward']:.3f}"
+        print(f"  Task {r['task_id']} ({r['difficulty']:6s})  {status}  "
+              f"in {r['attempts']} attempt(s)")
+        total_score += r["best_reward"]
+
+    avg_score = total_score / len(results)
+    print(f"\n  Average reward : {avg_score:.3f} / 1.000")
+    print(f"  Tasks solved   : {sum(1 for r in results if r['solved'])} / {len(results)}")
+
+    # Save results to file (useful for judges / CI)
+    output_path = "results.json"
+    with open(output_path, "w") as f:
+        json.dump({
+            "results":     results,
+            "avg_score":   round(avg_score, 3),
+            "tasks_solved": sum(1 for r in results if r["solved"]),
+        }, f, indent=2)
+    print(f"\n  Results saved to {output_path}")
+
+    return avg_score
+
+
+if __name__ == "__main__":
+    main()

main.py

@@ -0,0 +1,367 @@
+import sqlite3
+import os
+import json
+import re
+from datetime import datetime
+from typing import Any, Optional
+
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+
+# ── Config ────────────────────────────────────────────────────────────────────
+DB_PATH = os.path.join("data", "ecommerce.db")
+app = FastAPI(title="SQL Analyst OpenEnv", version="1.0.0")
+
+# ── Pydantic Models ───────────────────────────────────────────────────────────
+
+class StepRequest(BaseModel):
+    action: str          # The SQL query the agent submits
+
+class StepResponse(BaseModel):
+    observation: dict
+    reward: float        # 0.0 – 1.0
+    done: bool
+    info: dict
+
+class ResetRequest(BaseModel):
+    task_id: int         # 1 = Easy, 2 = Medium, 3 = Hard
+
+class ResetResponse(BaseModel):
+    observation: dict
+    info: dict
+
+class StateResponse(BaseModel):
+    task_id: int
+    task_description: str
+    schema_info: str
+    attempts: int
+    best_reward: float
+    history: list
+
+# ── Task Definitions ──────────────────────────────────────────────────────────
+
+TASKS = {
+    1: {
+        "description": (
+            "Find the total number of completed orders placed in the year 2024. "
+            "Return a single number with column name: total_orders"
+        ),
+        "difficulty": "easy",
+        "hint": "Use COUNT with WHERE filters on status and order_date",
+        "answer_query": """
+            SELECT COUNT(*) AS total_orders
+            FROM orders
+            WHERE status = 'completed'
+              AND order_date LIKE '2024%'
+        """,
+    },
+    2: {
+        "description": (
+            "Find the top 5 customers by total revenue (sum of total_amount for completed orders only). "
+            "Return columns: first_name, last_name, total_revenue. "
+            "Order by total_revenue descending."
+        ),
+        "difficulty": "medium",
+        "hint": "JOIN orders with customers, GROUP BY customer, filter completed, ORDER and LIMIT",
+        "answer_query": """
+            SELECT c.first_name, c.last_name,
+                   ROUND(SUM(o.total_amount), 2) AS total_revenue
+            FROM orders o
+            JOIN customers c ON o.customer_id = c.customer_id
+            WHERE o.status = 'completed'
+            GROUP BY o.customer_id
+            ORDER BY total_revenue DESC
+            LIMIT 5
+        """,
+    },
+    3: {
+        "description": (
+            "For each product category, calculate the total revenue (completed orders only) "
+            "and rank categories by revenue using a window function. "
+            "Return columns: category, total_revenue, revenue_rank. "
+            "Order by revenue_rank ascending."
+        ),
+        "difficulty": "hard",
+        "hint": "Use SUM with GROUP BY inside a CTE, then apply RANK() OVER (ORDER BY ...) on the result",
+        "answer_query": """
+            WITH category_revenue AS (
+                SELECT p.category,
+                       SUM(o.total_amount) AS total_revenue
+                FROM orders o
+                JOIN products p ON o.product_id = p.product_id
+                WHERE o.status = 'completed'
+                GROUP BY p.category
+            )
+            SELECT category,
+                   total_revenue,
+                   RANK() OVER (ORDER BY total_revenue DESC) AS revenue_rank
+            FROM category_revenue
+            ORDER BY revenue_rank ASC
+        """,
+    },
+}
+
+# ── In-Memory Session State ───────────────────────────────────────────────────
+
+session = {
+    "task_id": None,
+    "task": None,
+    "expected_rows": None,
+    "expected_columns": None,
+    "attempts": 0,
+    "best_reward": 0.0,
+    "history": [],
+}
+
+# ── Database Helpers ──────────────────────────────────────────────────────────
+
+def get_connection():
+    if not os.path.exists(DB_PATH):
+        raise HTTPException(
+            status_code=500,
+            detail=f"Database not found at {DB_PATH}. Run seed.py first."
+        )
+    conn = sqlite3.connect(DB_PATH)
+    conn.row_factory = sqlite3.Row
+    return conn
+
+
+def get_schema_info() -> str:
+    conn = get_connection()
+    cur = conn.cursor()
+    schema_parts = []
+    cur.execute("SELECT name FROM sqlite_master WHERE type='table'")
+    tables = [r["name"] for r in cur.fetchall()]
+    for table in tables:
+        cur.execute(f"PRAGMA table_info({table})")
+        cols = cur.fetchall()
+        col_defs = ", ".join(f"{c['name']} {c['type']}" for c in cols)
+        cur.execute(f"SELECT COUNT(*) AS n FROM {table}")
+        count = cur.fetchone()["n"]
+        schema_parts.append(f"  {table} ({col_defs})  -- {count} rows")
+    conn.close()
+    return "Tables:\n" + "\n".join(schema_parts)
+
+
+def run_query(sql: str) -> tuple[list[dict], list[str]]:
+    """Run a SQL query and return (rows_as_dicts, column_names)."""
+    conn = get_connection()
+    cur = conn.cursor()
+    cur.execute(sql)
+    columns = [d[0] for d in cur.description] if cur.description else []
+    rows = [dict(zip(columns, row)) for row in cur.fetchall()]
+    conn.close()
+    return rows, columns
+
+
+def compute_expected():
+    """Run the answer query and cache the expected result."""
+    task = session["task"]
+    rows, columns = run_query(task["answer_query"])
+    session["expected_rows"] = rows
+    session["expected_columns"] = columns
+
+# ── Reward Function ───────────────────────────────────────────────────────────
+
+def compute_reward(agent_rows: list[dict], agent_cols: list[str]) -> tuple[float, dict]:
+    """
+    Partial scoring reward function — 0.0 to 1.0.
+
+    Breakdown:
+      - 0.30  correct column names
+      - 0.30  correct number of rows
+      - 0.40  correct values (cell-level match)
+    """
+    expected_rows = session["expected_rows"]
+    expected_cols = session["expected_columns"]
+    details = {}
+
+    # ── Column score (0.30) ──────────────────────────────────────────────────
+    agent_cols_lower   = [c.lower() for c in agent_cols]
+    expected_cols_lower = [c.lower() for c in expected_cols]
+    col_matches = sum(1 for c in expected_cols_lower if c in agent_cols_lower)
+    col_score = (col_matches / len(expected_cols_lower)) * 0.30 if expected_cols_lower else 0.0
+    details["column_score"] = round(col_score, 3)
+    details["expected_columns"] = expected_cols
+    details["agent_columns"] = agent_cols
+
+    # ── Row count score (0.30) ───────────────────────────────────────────────
+    expected_count = len(expected_rows)
+    agent_count    = len(agent_rows)
+    if expected_count == 0:
+        row_score = 0.30 if agent_count == 0 else 0.0
+    else:
+        row_ratio = min(agent_count, expected_count) / max(agent_count, expected_count)
+        row_score = row_ratio * 0.30
+    details["row_score"] = round(row_score, 3)
+    details["expected_row_count"] = expected_count
+    details["agent_row_count"]    = agent_count
+
+    # ── Value match score (0.40) ─────────────────────────────────────────────
+    if not expected_rows or not agent_rows:
+        value_score = 0.0
+    else:
+        def normalize(v):
+            if v is None:
+                return ""
+            try:
+                return str(round(float(v), 1))
+            except (ValueError, TypeError):
+                return str(v).strip().lower()
+
+
+        matched_cells = 0
+        total_cells   = len(expected_rows) * len(expected_cols_lower)
+
+        for exp_row, agt_row in zip(expected_rows, agent_rows):
+            for col in expected_cols_lower:
+                exp_val = normalize(exp_row.get(col) or exp_row.get(col.upper()))
+                # Try matching by column name first, then by position
+                agt_val = normalize(agt_row.get(col) or agt_row.get(col.upper()))
+                if not agt_val:
+                    # Fall back to positional match
+                    exp_idx = expected_cols_lower.index(col)
+                    if exp_idx < len(agent_cols):
+                        pos_col = agent_cols[exp_idx]
+                        agt_val = normalize(agt_row.get(pos_col))
+                if exp_val == agt_val:
+                    matched_cells += 1
+
+        value_score = (matched_cells / total_cells) * 0.40 if total_cells > 0 else 0.0
+    details["value_score"] = round(value_score, 3)
+
+    total = round(col_score + row_score + value_score, 3)
+    details["total_reward"] = total
+    return total, details
+
+# ── Endpoints ─────────────────────────────────────────────────────────────────
+
+@app.post("/reset", response_model=ResetResponse)
+def reset(req: ResetRequest):
+    """Start a new task. Call this before /step."""
+    if req.task_id not in TASKS:
+        raise HTTPException(status_code=400, detail="task_id must be 1, 2, or 3")
+
+    session["task_id"]    = req.task_id
+    session["task"]       = TASKS[req.task_id]
+    session["attempts"]   = 0
+    session["best_reward"] = 0.0
+    session["history"]    = []
+
+    compute_expected()
+
+    schema = get_schema_info()
+    observation = {
+        "task_id":          req.task_id,
+        "difficulty":       session["task"]["difficulty"],
+        "task_description": session["task"]["description"],
+        "schema":           schema,
+        "hint":             session["task"]["hint"],
+    }
+    return ResetResponse(
+        observation=observation,
+        info={"message": f"Task {req.task_id} loaded. Use POST /step with your SQL query."}
+    )
+
+
+@app.post("/step", response_model=StepResponse)
+def step(req: StepRequest):
+    """Submit a SQL query. Returns reward 0.0–1.0 and feedback."""
+    if session["task_id"] is None:
+        raise HTTPException(status_code=400, detail="Call /reset first to load a task.")
+
+    session["attempts"] += 1
+    sql = req.action.strip()
+
+    # ── Safety: only allow SELECT statements ─────────────────────────────────
+    if not re.match(r"^\s*(SELECT|WITH)\b", sql, re.IGNORECASE):
+        return StepResponse(
+            observation={"error": "Only SELECT or WITH (CTE) statements are allowed."},
+            reward=0.0,
+            done=False,
+            info={"attempt": session["attempts"], "message": "Rejected: not a SELECT/WITH query."}
+        )
+
+    # ── Run the agent's query ─────────────────────────────────────────────────
+    try:
+        agent_rows, agent_cols = run_query(sql)
+    except Exception as e:
+        entry = {
+            "attempt": session["attempts"],
+            "sql": sql,
+            "reward": 0.0,
+            "error": str(e),
+        }
+        session["history"].append(entry)
+        return StepResponse(
+            observation={"error": str(e), "sql_submitted": sql},
+            reward=0.0,
+            done=False,
+            info={"attempt": session["attempts"], "message": "SQL execution error."}
+        )
+
+    # ── Score it ──────────────────────────────────────────────────────────────
+    reward, details = compute_reward(agent_rows, agent_cols)
+    session["best_reward"] = max(session["best_reward"], reward)
+
+    done = reward >= 1.0
+
+    entry = {
+        "attempt":   session["attempts"],
+        "sql":       sql,
+        "reward":    reward,
+        "details":   details,
+        "timestamp": datetime.now().isoformat(),
+    }
+    session["history"].append(entry)
+
+    observation = {
+        "task_id":          session["task_id"],
+        "task_description": session["task"]["description"],
+        "sql_submitted":    sql,
+        "result_preview":   agent_rows[:5],   # show first 5 rows
+        "result_row_count": len(agent_rows),
+        "reward_breakdown": details,
+    }
+
+    return StepResponse(
+        observation=observation,
+        reward=reward,
+        done=done,
+        info={
+            "attempt":      session["attempts"],
+            "best_reward":  session["best_reward"],
+            "message":      "Perfect score! Task complete." if done else "Keep refining your query.",
+        }
+    )
+
+
+@app.get("/state", response_model=StateResponse)
+def state():
+    """Get current session state — task info, attempts, history."""
+    if session["task_id"] is None:
+        raise HTTPException(status_code=400, detail="No active task. Call /reset first.")
+
+    return StateResponse(
+        task_id=session["task_id"],
+        task_description=session["task"]["description"],
+        schema_info=get_schema_info(),
+        attempts=session["attempts"],
+        best_reward=session["best_reward"],
+        history=session["history"],
+    )
+
+
+@app.get("/")
+def root():
+    return {
+        "name":    "SQL Analyst OpenEnv",
+        "version": "1.0.0",
+        "tasks":   {k: {"difficulty": v["difficulty"], "description": v["description"]} for k, v in TASKS.items()},
+        "endpoints": ["/reset", "/step", "/state"],
+    }
+
+
+@app.get("/health")
+def health():
+    return {"status": "ok", "db_exists": os.path.exists(DB_PATH)}

openenv.yaml

@@ -0,0 +1,163 @@
+name: sql-analyst-env
+version: "1.0.0"
+description: >
+  A real-world SQL data analyst environment where an AI agent must write
+  correct SQL queries against an e-commerce database to answer business questions.
+  Tasks range from simple aggregations (Easy) to window functions and CTEs (Hard).
+
+author: "Hackathon Submission"
+license: MIT
+
+environment:
+  type: http
+  base_url: "http://0.0.0.0:7860"
+
+endpoints:
+  reset:
+    method: POST
+    path: /reset
+    description: Load a task. Returns task description and database schema.
+    request_body:
+      task_id:
+        type: integer
+        description: "1 = Easy, 2 = Medium, 3 = Hard"
+        required: true
+    response:
+      observation:
+        type: object
+        properties:
+          task_id:          { type: integer }
+          difficulty:       { type: string }
+          task_description: { type: string }
+          schema:           { type: string }
+          hint:             { type: string }
+      info:
+        type: object
+
+  step:
+    method: POST
+    path: /step
+    description: Submit a SQL query. Returns reward 0.0-1.0 and result preview.
+    request_body:
+      action:
+        type: string
+        description: A valid SQLite SELECT or WITH (CTE) statement.
+        required: true
+    response:
+      observation:
+        type: object
+        properties:
+          task_id:          { type: integer }
+          task_description: { type: string }
+          sql_submitted:    { type: string }
+          result_preview:   { type: array }
+          result_row_count: { type: integer }
+          reward_breakdown:
+            type: object
+            properties:
+              column_score:       { type: number }
+              row_score:          { type: number }
+              value_score:        { type: number }
+              total_reward:       { type: number }
+              expected_columns:   { type: array }
+              agent_columns:      { type: array }
+              expected_row_count: { type: integer }
+              agent_row_count:    { type: integer }
+      reward:
+        type: number
+        minimum: 0.0
+        maximum: 1.0
+        description: Partial score. 1.0 = perfect answer.
+      done:
+        type: boolean
+        description: True when reward reaches 1.0
+      info:
+        type: object
+
+  state:
+    method: GET
+    path: /state
+    description: Get current task, attempt count, best reward, and full history.
+    response:
+      task_id:          { type: integer }
+      task_description: { type: string }
+      schema_info:      { type: string }
+      attempts:         { type: integer }
+      best_reward:      { type: number }
+      history:          { type: array }
+
+observation_space:
+  description: >
+    On reset: task description, database schema (3 tables), difficulty, hint.
+    On step: submitted SQL, result preview (first 5 rows), row count,
+    reward breakdown (column / row / value sub-scores).
+
+action_space:
+  type: string
+  description: >
+    A single SQLite-compatible SELECT or WITH statement.
+    Only read operations are permitted. INSERT/UPDATE/DELETE are rejected.
+  examples:
+    - "SELECT COUNT(*) AS total_orders FROM orders WHERE status = 'completed'"
+    - "SELECT c.first_name, SUM(o.total_amount) AS revenue FROM orders o JOIN customers c ON o.customer_id = c.customer_id GROUP BY o.customer_id ORDER BY revenue DESC LIMIT 5"
+    - "WITH rev AS (SELECT p.category, SUM(o.total_amount) AS total FROM orders o JOIN products p ON o.product_id = p.product_id GROUP BY p.category) SELECT *, RANK() OVER (ORDER BY total DESC) AS rnk FROM rev"
+
+reward:
+  type: float
+  range: [0.0, 1.0]
+  description: >
+    Partial credit reward with three components:
+      - Column names correct : 0.30
+      - Row count correct    : 0.30
+      - Cell values correct  : 0.40
+  partial_credit: true
+
+tasks:
+  - id: 1
+    name: "Count completed orders"
+    difficulty: easy
+    description: >
+      Find the total number of completed orders placed in 2024.
+      Return a single number with column name: total_orders.
+
+  - id: 2
+    name: "Top 5 customers by revenue"
+    difficulty: medium
+    description: >
+      Find the top 5 customers by total revenue from completed orders.
+      Return: first_name, last_name, total_revenue. Order by revenue DESC.
+
+  - id: 3
+    name: "Category revenue ranking"
+    difficulty: hard
+    description: >
+      For each product category, calculate total revenue and rank using
+      a window function. Return: category, total_revenue, revenue_rank.
+      Order by revenue_rank ASC.
+
+database:
+  engine: SQLite
+  path: data/ecommerce.db
+  tables:
+    customers:
+      rows: 100
+      columns: [customer_id, first_name, last_name, email, city, signup_date]
+    products:
+      rows: 30
+      columns: [product_id, product_name, category, price, stock]
+    orders:
+      rows: 600
+      columns: [order_id, customer_id, product_id, quantity, total_amount, order_date, status]
+
+hardware:
+  min_cpu: 1
+  min_ram_gb: 1
+  notes: "Runs on 2 vCPU / 8GB RAM. No GPU required."
+
+inference:
+  script: inference.py
+  max_runtime_minutes: 20
+  llm_env_vars:
+    - API_BASE_URL
+    - MODEL_NAME
+    - HF_TOKEN

requirements.txt

@@ -0,0 +1,6 @@
+fastapi==0.115.0
+uvicorn==0.30.6
+pydantic==2.9.2
+requests==2.32.3
+openai==1.51.0
+pyyaml==6.0.2

results.json

@@ -0,0 +1,30 @@
+{
+  "results": [
+    {
+      "task_id": 1,
+      "difficulty": "easy",
+      "best_reward": 1.0,
+      "attempts": 1,
+      "final_sql": "SELECT COUNT(*) AS total_orders \nFROM orders \nWHERE status = 'completed' AND STRFTIME('%Y', order_date) = '2024'",
+      "solved": true
+    },
+    {
+      "task_id": 2,
+      "difficulty": "medium",
+      "best_reward": 1.0,
+      "attempts": 1,
+      "final_sql": "SELECT c.first_name, c.last_name, SUM(o.total_amount) AS total_revenue\nFROM customers c\nJOIN orders o ON c.customer_id = o.customer_id\nWHERE o.status = 'completed'\nGROUP BY c.customer_id, c.first_name, c.last_name\nORDER BY total_revenue DESC\nLIMIT 5;",
+      "solved": true
+    },
+    {
+      "task_id": 3,
+      "difficulty": "hard",
+      "best_reward": 1.0,
+      "attempts": 1,
+      "final_sql": "WITH category_revenue AS (\n  SELECT p.category, SUM(o.total_amount) AS total_revenue\n  FROM orders o\n  JOIN products p ON o.product_id = p.product_id\n  WHERE o.status = 'completed'\n  GROUP BY p.category\n)\nSELECT category, total_revenue, RANK() OVER (ORDER BY total_revenue DESC) AS revenue_rank\nFROM category_revenue\nORDER BY revenue_rank ASC;",
+      "solved": true
+    }
+  ],
+  "avg_score": 1.0,
+  "tasks_solved": 3
+}

seed.py

@@ -0,0 +1,181 @@
+import sqlite3
+import random
+from datetime import datetime, timedelta
+import os
+
+DB_PATH = os.path.join("data", "ecommerce.db")
+
+FIRST_NAMES = ["Alice", "Bob", "Carol", "David", "Eva", "Frank", "Grace", "Henry",
+               "Irene", "Jack", "Karen", "Liam", "Mona", "Nate", "Olivia", "Paul",
+               "Quinn", "Rachel", "Sam", "Tina", "Uma", "Victor", "Wendy", "Xander",
+               "Yara", "Zoe", "Aaron", "Bella", "Chris", "Diana"]
+
+LAST_NAMES = ["Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller",
+              "Davis", "Wilson", "Taylor", "Anderson", "Thomas", "Jackson", "White",
+              "Harris", "Martin", "Thompson", "Robinson", "Clark", "Lewis"]
+
+CITIES = ["New York", "Los Angeles", "Chicago", "Houston", "Phoenix", "Philadelphia",
+          "San Antonio", "San Diego", "Dallas", "San Jose", "Austin", "Jacksonville",
+          "Mumbai", "Delhi", "Chennai", "Bangalore", "Hyderabad", "Kolkata"]
+
+CATEGORIES = ["Electronics", "Clothing", "Books", "Home & Garden", "Sports",
+              "Beauty", "Toys", "Food & Grocery", "Automotive", "Music"]
+
+PRODUCTS = [
+    ("Wireless Headphones", "Electronics", 79.99),
+    ("Bluetooth Speaker", "Electronics", 49.99),
+    ("USB-C Hub", "Electronics", 34.99),
+    ("Mechanical Keyboard", "Electronics", 129.99),
+    ("Gaming Mouse", "Electronics", 59.99),
+    ("Webcam HD", "Electronics", 89.99),
+    ("Monitor 24inch", "Electronics", 249.99),
+    ("Phone Stand", "Electronics", 14.99),
+    ("Running Shoes", "Clothing", 89.99),
+    ("Denim Jacket", "Clothing", 64.99),
+    ("Cotton T-Shirt", "Clothing", 19.99),
+    ("Yoga Pants", "Clothing", 44.99),
+    ("Winter Coat", "Clothing", 149.99),
+    ("Baseball Cap", "Clothing", 24.99),
+    ("Python Programming", "Books", 39.99),
+    ("Data Science Handbook", "Books", 49.99),
+    ("Machine Learning Guide", "Books", 54.99),
+    ("Cook Book Deluxe", "Books", 29.99),
+    ("History of AI", "Books", 34.99),
+    ("Garden Hose 50ft", "Home & Garden", 44.99),
+    ("Plant Pots Set", "Home & Garden", 29.99),
+    ("LED Desk Lamp", "Home & Garden", 39.99),
+    ("Yoga Mat", "Sports", 34.99),
+    ("Resistance Bands", "Sports", 19.99),
+    ("Dumbbell Set 20kg", "Sports", 79.99),
+    ("Jump Rope", "Sports", 12.99),
+    ("Face Moisturizer", "Beauty", 24.99),
+    ("Shampoo Pro", "Beauty", 14.99),
+    ("Perfume Set", "Beauty", 59.99),
+    ("Building Blocks", "Toys", 34.99),
+]
+
+STATUSES = ["completed", "completed", "completed", "pending", "cancelled"]
+
+
+def create_tables(conn):
+    conn.executescript("""
+        DROP TABLE IF EXISTS orders;
+        DROP TABLE IF EXISTS products;
+        DROP TABLE IF EXISTS customers;
+
+        CREATE TABLE customers (
+            customer_id   INTEGER PRIMARY KEY,
+            first_name    TEXT NOT NULL,
+            last_name     TEXT NOT NULL,
+            email         TEXT UNIQUE NOT NULL,
+            city          TEXT NOT NULL,
+            signup_date   TEXT NOT NULL
+        );
+
+        CREATE TABLE products (
+            product_id    INTEGER PRIMARY KEY,
+            product_name  TEXT NOT NULL,
+            category      TEXT NOT NULL,
+            price         REAL NOT NULL,
+            stock         INTEGER NOT NULL
+        );
+
+        CREATE TABLE orders (
+            order_id      INTEGER PRIMARY KEY,
+            customer_id   INTEGER NOT NULL,
+            product_id    INTEGER NOT NULL,
+            quantity      INTEGER NOT NULL,
+            total_amount  REAL NOT NULL,
+            order_date    TEXT NOT NULL,
+            status        TEXT NOT NULL,
+            FOREIGN KEY (customer_id) REFERENCES customers(customer_id),
+            FOREIGN KEY (product_id)  REFERENCES products(product_id)
+        );
+    """)
+    print("Tables created.")
+
+
+def seed_customers(conn, n=100):
+    used_emails = set()
+    rows = []
+    for i in range(1, n + 1):
+        fn = random.choice(FIRST_NAMES)
+        ln = random.choice(LAST_NAMES)
+        base_email = f"{fn.lower()}.{ln.lower()}{i}@example.com"
+        while base_email in used_emails:
+            base_email = f"{fn.lower()}.{ln.lower()}{i}_{random.randint(10,99)}@example.com"
+        used_emails.add(base_email)
+        city = random.choice(CITIES)
+        days_ago = random.randint(30, 730)
+        signup = (datetime.now() - timedelta(days=days_ago)).strftime("%Y-%m-%d")
+        rows.append((i, fn, ln, base_email, city, signup))
+    conn.executemany(
+        "INSERT INTO customers VALUES (?,?,?,?,?,?)", rows
+    )
+    print(f"Inserted {n} customers.")
+
+
+def seed_products(conn):
+    rows = []
+    for i, (name, cat, price) in enumerate(PRODUCTS, start=1):
+        stock = random.randint(0, 200)
+        rows.append((i, name, cat, price, stock))
+    conn.executemany(
+        "INSERT INTO products VALUES (?,?,?,?,?)", rows
+    )
+    print(f"Inserted {len(PRODUCTS)} products.")
+
+
+def seed_orders(conn, n=600):
+    rows = []
+    base_date = datetime(2024, 1, 1)
+    for i in range(1, n + 1):
+        cust_id = random.randint(1, 100)
+        prod_id = random.randint(1, len(PRODUCTS))
+        qty = random.randint(1, 5)
+        price = PRODUCTS[prod_id - 1][2]
+        total = round(price * qty, 2)
+        days_offset = random.randint(0, 364)
+        order_date = (base_date + timedelta(days=days_offset)).strftime("%Y-%m-%d")
+        status = random.choice(STATUSES)
+        rows.append((i, cust_id, prod_id, qty, total, order_date, status))
+    conn.executemany(
+        "INSERT INTO orders VALUES (?,?,?,?,?,?,?)", rows
+    )
+    print(f"Inserted {n} orders.")
+
+
+def main():
+    os.makedirs("data", exist_ok=True)
+    conn = sqlite3.connect(DB_PATH)
+    create_tables(conn)
+    seed_customers(conn)
+    seed_products(conn)
+    seed_orders(conn)
+    conn.commit()
+
+    # Quick sanity check
+    cur = conn.cursor()
+    print("\n--- Sanity Check ---")
+    for table in ["customers", "products", "orders"]:
+        cur.execute(f"SELECT COUNT(*) FROM {table}")
+        print(f"  {table}: {cur.fetchone()[0]} rows")
+
+    # Preview a joined query
+    cur.execute("""
+        SELECT c.first_name, c.last_name, p.product_name, o.total_amount, o.order_date
+        FROM orders o
+        JOIN customers c ON o.customer_id = c.customer_id
+        JOIN products  p ON o.product_id  = p.product_id
+        LIMIT 5
+    """)
+    print("\n--- Sample joined rows ---")
+    for row in cur.fetchall():
+        print(" ", row)
+
+    conn.close()
+    print(f"\nDatabase saved to: {DB_PATH}")
+
+
+if __name__ == "__main__":
+    main()