first commit

Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Install dependencies first (layer cache)
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code
+COPY server/ ./server/
+
+# Environment defaults
+ENV PORT=7860
+ENV PYTHONPATH=/app
+ENV ENABLE_WEB_INTERFACE=false
+
+EXPOSE 7860
+
+CMD ["python", "-m", "uvicorn", "server.app:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -0,0 +1,148 @@
+# Code Security Review — OpenEnv
+
+> An RL environment for training AI agents to detect bugs and security
+> vulnerabilities in Python code.
+
+## Motivation
+
+Code review is one of the highest-leverage tasks in software engineering, yet it
+remains bottlenecked on human attention. This environment trains agents to catch
+real bug categories — from simple off-by-one errors to critical SQL injection
+vulnerabilities — using structured, deterministic reward signals.
+
+---
+
+## Action Space
+
+| Field | Type | Description |
+|---|---|---|
+| `bug_identified` | bool | Whether a bug was found |
+| `bug_location` | string | Exact location (function, expression) |
+| `bug_type` | string | `off-by-one`, `logic-error`, `security-vulnerability`, `none` |
+| `bug_description` | string | Explanation of the bug and its impact |
+| `severity` | string | `none` / `low` / `medium` / `high` / `critical` |
+| `suggested_fix` | string | Corrected code or fix description |
+
+## Observation Space
+
+| Field | Type | Description |
+|---|---|---|
+| `code_snippet` | string | The code to review |
+| `language` | string | Programming language |
+| `task_description` | string | What the code is supposed to do |
+| `task_id` | string | Unique task identifier |
+| `difficulty` | string | `easy` / `medium` / `hard` |
+| `step_number` | int | Current step within the episode |
+| `max_steps` | int | Maximum steps allowed (3) |
+| `previous_feedback` | string? | Feedback from prior step |
+
+---
+
+## Tasks
+
+### Easy — Off-by-one in array traversal
+- **Code:** `sum_elements(arr)` iterates `range(1, len(arr)+1)` causing `IndexError`
+- **Expected bug type:** `off-by-one`
+- **Expected severity:** `high`
+- **Baseline score:** ~0.72
+
+### Medium — Authentication logic flaw
+- **Code:** `authenticate_user()` uses `or` instead of `and` for admin check
+- **Expected bug type:** `logic-error`
+- **Expected severity:** `critical`
+- **Baseline score:** ~0.60
+
+### Hard — SQL injection via f-string
+- **Code:** `fetch_records()` interpolates `user_id` and `sort_column` directly into SQL
+- **Expected bug type:** `security-vulnerability`
+- **Expected severity:** `critical`
+- **Baseline score:** ~0.55
+
+---
+
+## Reward Function
+
+Rewards are deterministic and provide partial progress signal:
+
+| Component | Max Score | Description |
+|---|---|---|
+| Bug identified | 0.20 | Correctly flags presence/absence of bug |
+| Bug type | 0.20 | Correct category of bug |
+| Bug location | 0.10 | Precise location identified |
+| Description quality | 0.25 | Keyword density in explanation |
+| Fix quality | 0.15 | Correct fix keywords present |
+| Severity | 0.10 | Correct severity level |
+| **Total** | **1.00** | |
+
+---
+
+## Setup
+
+### 1. Build and run Docker
+
+```bash
+docker build -t code-review-env .
+docker run -p 7860:7860 code-review-env
+```
+
+### 2. Run inference baseline
+
+```bash
+# Set your environment variables
+export HF_TOKEN=hf_your_token_here
+export MODEL_NAME=Qwen/Qwen2.5-72B-Instruct
+export API_BASE_URL=https://router.huggingface.co/v1
+export ENV_BASE_URL=http://localhost:7860
+
+# Install dependencies
+pip install -r requirements.txt
+
+# Run
+python inference.py
+```
+
+### 3. Validate (OpenEnv CLI)
+
+```bash
+openenv validate
+```
+
+---
+
+## API Endpoints
+
+| Method | Path | Description |
+|---|---|---|
+| GET | `/health` | Health check |
+| POST | `/reset?difficulty=easy` | Reset environment |
+| POST | `/step` | Submit a review action |
+| GET | `/state` | Current episode state |
+
+---
+
+## Baseline Scores
+
+| Task | Difficulty | Reward |
+|---|---|---|
+| Off-by-one detection | Easy | ~0.72 |
+| Auth logic flaw | Medium | ~0.60 |
+| SQL injection | Hard | ~0.55 |
+| **Average** | | **~0.62** |
+
+---
+
+## Project Structure
+
+```
+code-review-env/
+├── Dockerfile
+├── openenv.yaml
+├── requirements.txt
+├── inference.py
+├── README.md
+└── server/
+    ├── __init__.py
+    ├── app.py          # FastAPI endpoints
+    ├── environment.py  # Tasks + grader logic
+    └── models.py       # Pydantic action/observation/state
+```

app.py

@@ -0,0 +1,64 @@
+import os
+import uvicorn
+from fastapi import FastAPI, HTTPException, Query
+from fastapi.middleware.cors import CORSMiddleware
+
+from .models import CodeReviewAction, CodeReviewState, StepResponse, ResetResponse
+from .environment import CodeReviewEnvironment
+
+app = FastAPI(
+    title="Code Security Review — OpenEnv",
+    description=(
+        "RL environment for training AI agents to detect bugs and security "
+        "vulnerabilities in code. Compatible with the OpenEnv spec."
+    ),
+    version="1.0.0",
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+env = CodeReviewEnvironment()
+
+
+@app.get("/health")
+def health():
+    return {"status": "ok", "env": "code-review-env", "version": "1.0.0"}
+
+
+@app.post("/reset", response_model=ResetResponse)
+def reset(difficulty: str = Query(default="easy", description="easy | medium | hard")):
+    """Reset the environment and return the first observation."""
+    obs = env.reset(difficulty=difficulty)
+    return ResetResponse(observation=obs)
+
+
+@app.post("/step", response_model=StepResponse)
+def step(action: CodeReviewAction):
+    """Submit a code review action and receive a reward signal."""
+    try:
+        obs, reward, done, info = env.step(action)
+        return StepResponse(observation=obs, reward=reward, done=done, info=info)
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc))
+
+
+@app.get("/state", response_model=CodeReviewState)
+def state():
+    """Return the current environment state."""
+    return env.state()
+
+
+if __name__ == "__main__":
+    port = int(os.environ.get("PORT", 7860))
+    enable_web = os.environ.get("ENABLE_WEB_INTERFACE", "false").lower() == "true"
+    uvicorn.run(
+        "server.app:app",
+        host="0.0.0.0",
+        port=port,
+        reload=False,
+    )

environment.py

@@ -0,0 +1,309 @@
+from typing import Dict, Any, Tuple, Optional
+from .models import CodeReviewAction, CodeReviewObservation, CodeReviewState
+
+MAX_STEPS = 3
+
+# ──────────────────────────────────────────────
+# TASK DEFINITIONS
+# ──────────────────────────────────────────────
+
+TASKS: Dict[str, dict] = {
+
+    # ── EASY ─────────────────────────────────
+    "easy": {
+        "id": "task_easy_001",
+        "difficulty": "easy",
+        "language": "python",
+        "description": (
+            "This function is supposed to sum all elements in a list. "
+            "Find any bugs and suggest a fix."
+        ),
+        "code": (
+            "def sum_elements(arr):\n"
+            '    """Return the sum of all elements."""\n'
+            "    total = 0\n"
+            "    for i in range(1, len(arr) + 1):  # iterates over indices\n"
+            "        total += arr[i]\n"
+            "    return total"
+        ),
+        "ground_truth": {
+            "bug_identified": True,
+            "bug_type_keywords": [
+                "off-by-one", "off by one", "index error", "indexerror",
+                "out of bounds", "out of range", "index out",
+            ],
+            "location_keywords": [
+                "range(1, len(arr) + 1)", "len(arr) + 1", "len(arr)+1",
+                "range", "loop", "index", "arr[i]",
+            ],
+            "description_keywords": [
+                "index", "range", "len", "off-by-one", "off by one",
+                "IndexError", "out of bounds", "+1", "exceed", "arr[i]",
+                "zero", "start",
+            ],
+            "fix_keywords": [
+                "range(len(arr))", "range(0, len(arr))",
+                "for i in range(len", "for element in arr",
+                "arr[i]" , "len(arr))",
+            ],
+            "severity_valid": ["high", "medium"],
+        },
+    },
+
+    # ── MEDIUM ────────────────────────────────
+    "medium": {
+        "id": "task_medium_001",
+        "difficulty": "medium",
+        "language": "python",
+        "description": (
+            "This authentication function controls admin access. "
+            "Find the logical security bug."
+        ),
+        "code": (
+            "def authenticate_user(username, password, request_admin=False):\n"
+            '    """Authenticate user and return access level."""\n'
+            "    user = db.find_user(username)\n"
+            "    if not user or user.password_hash != hash_password(password):\n"
+            '        return {"authenticated": False, "level": "none"}\n'
+            "\n"
+            "    # Elevate to admin if caller requests it OR user has admin role\n"
+            "    if request_admin or user.role == 'admin':   # <-- review this\n"
+            '        return {"authenticated": True, "level": "admin"}\n'
+            "\n"
+            '    return {"authenticated": True, "level": "user"}'
+        ),
+        "ground_truth": {
+            "bug_identified": True,
+            "bug_type_keywords": [
+                "logic", "logic error", "logical", "privilege escalation",
+                "authorization", "authentication bypass", "access control",
+            ],
+            "location_keywords": [
+                "request_admin or", "or user.role", "or", "condition",
+                "if request_admin", "or user.role == 'admin'",
+            ],
+            "description_keywords": [
+                "or", "and", "privilege", "escalation", "bypass", "admin",
+                "role", "caller", "request_admin", "logic", "elevation",
+                "any caller", "arbitrary",
+            ],
+            "fix_keywords": [
+                "and", "request_admin and user.role", "and user.role == 'admin'",
+                "and user.role", "both",
+            ],
+            "severity_valid": ["critical", "high"],
+        },
+    },
+
+    # ── HARD ──────────────────────────────────
+    "hard": {
+        "id": "task_hard_001",
+        "difficulty": "hard",
+        "language": "python",
+        "description": (
+            "This function fetches records from a database using user-supplied input. "
+            "Identify the security vulnerability."
+        ),
+        "code": (
+            "def fetch_records(user_id: str, sort_column: str):\n"
+            '    """Fetch user records sorted by a given column."""\n'
+            "    conn = get_db_connection()\n"
+            "    cursor = conn.cursor()\n"
+            "\n"
+            "    query = (\n"
+            '        f"SELECT id, name, email FROM users "\n'
+            '        f"WHERE user_id = {user_id} "\n'
+            '        f"ORDER BY {sort_column}"\n'
+            "    )\n"
+            "    cursor.execute(query)\n"
+            "    rows = cursor.fetchall()\n"
+            "    conn.close()\n"
+            "    return rows"
+        ),
+        "ground_truth": {
+            "bug_identified": True,
+            "bug_type_keywords": [
+                "sql injection", "injection", "sqli", "sql",
+                "security vulnerability", "security", "second-order",
+            ],
+            "location_keywords": [
+                "f\"", "f-string", "format", "user_id", "sort_column",
+                "query", "ORDER BY", "WHERE user_id",
+            ],
+            "description_keywords": [
+                "sql injection", "injection", "parameterized", "f-string",
+                "format string", "user input", "sanitize", "escape",
+                "malicious", "attack", "tautology", "union", "drop",
+                "ORDER BY", "sort_column", "arbitrary",
+            ],
+            "fix_keywords": [
+                "parameterized", "?", "%s", "cursor.execute(query, (",
+                "cursor.execute(query, [", "prepared statement",
+                "whitelist", "allowlist", "ALLOWED_COLUMNS",
+                "sanitize", "if sort_column not in",
+            ],
+            "severity_valid": ["critical"],
+        },
+    },
+}
+
+
+# ──────────────────────────────────────────────
+# GRADER
+# ──────────────────────────────────────────────
+
+def grade_action(action: CodeReviewAction, task: dict) -> Tuple[float, Dict]:
+    """
+    Score the agent's review on a 0.0–1.0 scale.
+
+    Breakdown:
+      bug_identified   0.20
+      bug_type         0.20
+      bug_location     0.10
+      bug_description  0.25  (keyword density, capped)
+      suggested_fix    0.15  (keyword density, capped)
+      severity         0.10
+      ─────────────────────
+      Total            1.00
+    """
+    gt = task["ground_truth"]
+    score = 0.0
+    breakdown: Dict[str, float] = {}
+
+    # 1. Bug identification
+    if action.bug_identified == gt["bug_identified"]:
+        score += 0.20
+        breakdown["bug_identified"] = 0.20
+    else:
+        breakdown["bug_identified"] = 0.00
+        if not action.bug_identified:
+            return 0.0, {
+                "breakdown": breakdown,
+                "total_score": 0.0,
+                "feedback": "No bug identified — one definitely exists. Look more carefully.",
+            }
+
+    # 2. Bug type
+    bug_type_lower = action.bug_type.lower()
+    type_match = any(kw in bug_type_lower for kw in gt["bug_type_keywords"])
+    if type_match:
+        score += 0.20
+        breakdown["bug_type"] = 0.20
+    else:
+        breakdown["bug_type"] = 0.00
+
+    # 3. Bug location
+    loc_lower = action.bug_location.lower()
+    loc_match = any(kw.lower() in loc_lower for kw in gt["location_keywords"])
+    if loc_match:
+        score += 0.10
+        breakdown["bug_location"] = 0.10
+    else:
+        breakdown["bug_location"] = 0.00
+
+    # 4. Description quality (keyword density, capped at 0.25)
+    desc_lower = action.bug_description.lower()
+    desc_hits = sum(1 for kw in gt["description_keywords"] if kw.lower() in desc_lower)
+    desc_score = round(min(0.25, desc_hits * 0.07), 3)
+    score += desc_score
+    breakdown["bug_description"] = desc_score
+
+    # 5. Fix quality (keyword density, capped at 0.15)
+    fix_lower = action.suggested_fix.lower()
+    fix_hits = sum(1 for kw in gt["fix_keywords"] if kw.lower() in fix_lower)
+    fix_score = round(min(0.15, fix_hits * 0.08), 3)
+    score += fix_score
+    breakdown["suggested_fix"] = fix_score
+
+    # 6. Severity
+    if action.severity.lower() in gt["severity_valid"]:
+        score += 0.10
+        breakdown["severity"] = 0.10
+    else:
+        breakdown["severity"] = 0.00
+
+    total = round(min(1.0, score), 3)
+
+    # Build human-readable feedback
+    hints = []
+    if breakdown["bug_type"] == 0:
+        hints.append("Reconsider the bug category — be more specific.")
+    if breakdown["bug_location"] == 0:
+        hints.append("Pinpoint the exact line or expression that contains the bug.")
+    if breakdown["suggested_fix"] < 0.08:
+        hints.append("Your fix does not address the root cause — revise it.")
+    if breakdown["severity"] == 0:
+        hints.append("Re-evaluate the severity level.")
+
+    feedback = " ".join(hints) if hints else "Strong analysis — refine the fix if needed."
+
+    return total, {"breakdown": breakdown, "total_score": total, "feedback": feedback}
+
+
+# ──────────────────────────────────────────────
+# ENVIRONMENT
+# ──────────────────────────────────────────────
+
+class CodeReviewEnvironment:
+    def __init__(self):
+        self._state: Optional[CodeReviewState] = None
+        self._current_task: Optional[dict] = None
+
+    def reset(self, difficulty: str = "easy") -> CodeReviewObservation:
+        if difficulty not in TASKS:
+            difficulty = "easy"
+        task = TASKS[difficulty]
+        self._current_task = task
+        self._state = CodeReviewState(
+            task_id=task["id"],
+            difficulty=difficulty,
+            step_count=0,
+            done=False,
+            total_reward=0.0,
+            task_complete=False,
+        )
+        return self._build_obs(step_number=0, previous_feedback=None)
+
+    def step(self, action: CodeReviewAction) -> Tuple[CodeReviewObservation, float, bool, Dict]:
+        if self._state is None or self._state.done:
+            raise ValueError("Call reset() before step().")
+
+        self._state.step_count += 1
+        reward, info = grade_action(action, self._current_task)
+        self._state.total_reward = round(self._state.total_reward + reward, 3)
+
+        # Done if agent nailed it or max steps reached
+        done = reward >= 0.80 or self._state.step_count >= MAX_STEPS
+        self._state.done = done
+        self._state.task_complete = reward >= 0.80
+
+        feedback = info.get("feedback") if not done else None
+        obs = self._build_obs(
+            step_number=self._state.step_count,
+            previous_feedback=feedback,
+        )
+        return obs, reward, done, info
+
+    def state(self) -> CodeReviewState:
+        if self._state is None:
+            return CodeReviewState(
+                task_id="", difficulty="easy",
+                step_count=0, done=False,
+                total_reward=0.0, task_complete=False,
+            )
+        return self._state
+
+    # ── helpers ───────────────────────────────
+
+    def _build_obs(self, step_number: int, previous_feedback: Optional[str]) -> CodeReviewObservation:
+        t = self._current_task
+        return CodeReviewObservation(
+            code_snippet=t["code"],
+            language=t["language"],
+            task_description=t["description"],
+            task_id=t["id"],
+            difficulty=t["difficulty"],
+            step_number=step_number,
+            max_steps=MAX_STEPS,
+            previous_feedback=previous_feedback,
+        )

inference.py

@@ -0,0 +1,178 @@
+"""
+Baseline inference script for Code Security Review OpenEnv.
+
+Usage:
+    python inference.py
+
+Required environment variables:
+    API_BASE_URL  — LLM API endpoint  (default: HF router)
+    MODEL_NAME    — Model identifier
+    HF_TOKEN      — Hugging Face / API key
+    ENV_BASE_URL  — Running environment URL (default: http://localhost:7860)
+"""
+
+import os
+import json
+import time
+import re
+import requests
+from openai import OpenAI
+
+# ── Config ────────────────────────────────────────────────────────────────────
+API_BASE_URL = os.environ.get("API_BASE_URL", "https://router.huggingface.co/v1")
+MODEL_NAME   = os.environ.get("MODEL_NAME",   "Qwen/Qwen2.5-72B-Instruct")
+HF_TOKEN     = os.environ.get("HF_TOKEN",     "")
+ENV_BASE_URL = os.environ.get("ENV_BASE_URL", "http://localhost:7860")
+
+client = OpenAI(base_url=API_BASE_URL, api_key=HF_TOKEN)
+
+SYSTEM_PROMPT = """You are a senior security-focused code reviewer.
+
+When given a code snippet, carefully analyse it for bugs and security issues.
+
+Respond with ONLY a valid JSON object — no markdown, no explanation outside the JSON.
+
+Schema:
+{
+  "bug_identified": true or false,
+  "bug_location": "exact location (function name, line description, variable, expression)",
+  "bug_type": "off-by-one | logic-error | security-vulnerability | null-dereference | none",
+  "bug_description": "detailed explanation of why this is a bug and the impact",
+  "severity": "none | low | medium | high | critical",
+  "suggested_fix": "the corrected code snippet or a precise description of the fix"
+}"""
+
+# ── Helpers ───────────────────────────────────────────────────────────────────
+
+def env_post(path: str, data: dict | None = None, params: dict | None = None) -> dict:
+    url = f"{ENV_BASE_URL}{path}"
+    resp = requests.post(url, json=data or {}, params=params or {}, timeout=30)
+    resp.raise_for_status()
+    return resp.json()
+
+
+def parse_json_from_llm(text: str) -> dict:
+    """Robustly extract JSON from LLM output, stripping markdown fences."""
+    text = text.strip()
+    # Strip ```json ... ``` or ``` ... ```
+    text = re.sub(r"^```(?:json)?\s*", "", text)
+    text = re.sub(r"\s*```$", "", text)
+    return json.loads(text)
+
+
+def build_prompt(obs: dict) -> str:
+    lines = [
+        f"Language: {obs['language']}",
+        f"Task: {obs['task_description']}",
+        "",
+        f"```{obs['language']}",
+        obs["code_snippet"],
+        "```",
+    ]
+    if obs.get("previous_feedback"):
+        lines += ["", f"Previous feedback: {obs['previous_feedback']}",
+                  "Revise your analysis accordingly."]
+    return "\n".join(lines)
+
+
+# ── Task runner ───────────────────────────────────────────────────────────────
+
+def run_task(difficulty: str, task_num: int) -> dict:
+    reset_resp = env_post("/reset", params={"difficulty": difficulty})
+    obs = reset_resp["observation"]
+
+    print(f"[START] task={task_num} difficulty={difficulty} task_id={obs['task_id']} max_steps={obs['max_steps']}")
+
+    cumulative_reward = 0.0
+    step_num = 0
+    done = False
+
+    while not done and step_num < obs["max_steps"]:
+        step_num += 1
+        prompt = build_prompt(obs)
+
+        # ── LLM call ──────────────────────────────────────────────────────────
+        t0 = time.time()
+        try:
+            response = client.chat.completions.create(
+                model=MODEL_NAME,
+                messages=[
+                    {"role": "system", "content": SYSTEM_PROMPT},
+                    {"role": "user",   "content": prompt},
+                ],
+                temperature=0.1,
+                max_tokens=600,
+            )
+            raw = response.choices[0].message.content
+            action_dict = parse_json_from_llm(raw)
+        except Exception as exc:
+            print(f"[ERROR] task={task_num} step={step_num} llm_error={exc}")
+            action_dict = {
+                "bug_identified": False,
+                "bug_location": "",
+                "bug_type": "none",
+                "bug_description": "",
+                "severity": "none",
+                "suggested_fix": "",
+            }
+        latency = round(time.time() - t0, 2)
+
+        # ── Step env ──────────────────────────────────────────────────────────
+        step_resp = env_post("/step", data=action_dict)
+        reward = step_resp["reward"]
+        done   = step_resp["done"]
+        obs    = step_resp["observation"]
+        info   = step_resp.get("info", {})
+
+        cumulative_reward += reward
+
+        print(
+            f"[STEP] task={task_num} step={step_num} "
+            f"reward={reward:.3f} cumulative={cumulative_reward:.3f} "
+            f"done={done} latency_s={latency}"
+        )
+
+    result = {
+        "task_num":        task_num,
+        "difficulty":      difficulty,
+        "total_reward":    round(cumulative_reward, 3),
+        "steps_taken":     step_num,
+        "success":         cumulative_reward >= 0.8,
+    }
+    print(
+        f"[END] task={task_num} difficulty={difficulty} "
+        f"total_reward={result['total_reward']} success={result['success']}"
+    )
+    return result
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def main():
+    print(f"[INFO] model={MODEL_NAME} env={ENV_BASE_URL}")
+
+    tasks = [
+        ("easy",   1),
+        ("medium", 2),
+        ("hard",   3),
+    ]
+    results = []
+
+    for difficulty, task_num in tasks:
+        try:
+            r = run_task(difficulty, task_num)
+        except Exception as exc:
+            print(f"[ERROR] task={task_num} difficulty={difficulty} error={exc}")
+            r = {"task_num": task_num, "difficulty": difficulty,
+                 "total_reward": 0.0, "success": False}
+        results.append(r)
+
+    avg = round(sum(r["total_reward"] for r in results) / len(results), 3)
+    successes = sum(1 for r in results if r.get("success"))
+    print(f"\n[SUMMARY] avg_reward={avg} tasks_passed={successes}/{len(results)}")
+    for r in results:
+        print(f"  [{r['difficulty']:6}] reward={r['total_reward']:.3f} success={r.get('success', False)}")
+
+
+if __name__ == "__main__":
+    main()

models.py

@@ -0,0 +1,45 @@
+from pydantic import BaseModel, Field
+from typing import Optional, Any, Dict
+
+
+class CodeReviewAction(BaseModel):
+    """Action taken by the agent: a structured code review."""
+    bug_identified: bool = Field(..., description="Whether a bug was found")
+    bug_location: str = Field(..., description="Location of the bug (function, line, variable)")
+    bug_type: str = Field(..., description="Type: off-by-one | logic-error | security-vulnerability | none")
+    bug_description: str = Field(..., description="Detailed explanation of why this is a bug")
+    severity: str = Field(..., description="Severity: none | low | medium | high | critical")
+    suggested_fix: str = Field(..., description="The corrected code or a description of how to fix it")
+
+
+class CodeReviewObservation(BaseModel):
+    """What the agent sees at each step."""
+    code_snippet: str = Field(..., description="The code to review")
+    language: str = Field(..., description="Programming language")
+    task_description: str = Field(..., description="What the code is supposed to do")
+    task_id: str = Field(..., description="Unique task identifier")
+    difficulty: str = Field(..., description="easy | medium | hard")
+    step_number: int = Field(..., description="Current step within this episode")
+    max_steps: int = Field(..., description="Maximum steps allowed per episode")
+    previous_feedback: Optional[str] = Field(None, description="Feedback from previous step if any")
+
+
+class CodeReviewState(BaseModel):
+    """Internal environment state."""
+    task_id: str
+    difficulty: str
+    step_count: int
+    done: bool
+    total_reward: float
+    task_complete: bool
+
+
+class StepResponse(BaseModel):
+    observation: CodeReviewObservation
+    reward: float
+    done: bool
+    info: Dict[str, Any]
+
+
+class ResetResponse(BaseModel):
+    observation: CodeReviewObservation

openenv.yaml

@@ -0,0 +1,77 @@
+name: code-review-env
+version: 1.0.0
+description: >
+  RL environment for training AI agents to detect bugs and security
+  vulnerabilities in real Python code. Covers off-by-one errors,
+  authentication logic flaws, and SQL injection — with deterministic
+  programmatic graders and partial-progress reward signals.
+
+author: Inmodel Labs
+tags:
+  - code-review
+  - security
+  - software-engineering
+  - real-world
+  - python
+
+tasks:
+  - id: task_easy_001
+    difficulty: easy
+    description: "Detect off-by-one error in array traversal loop"
+    reset_params:
+      difficulty: easy
+
+  - id: task_medium_001
+    difficulty: medium
+    description: "Detect authentication logic flaw enabling privilege escalation"
+    reset_params:
+      difficulty: medium
+
+  - id: task_hard_001
+    difficulty: hard
+    description: "Detect SQL injection via unsanitised f-string database query"
+    reset_params:
+      difficulty: hard
+
+action_space:
+  type: object
+  properties:
+    bug_identified:   { type: boolean }
+    bug_location:     { type: string }
+    bug_type:         { type: string }
+    bug_description:  { type: string }
+    severity:         { type: string, enum: [none, low, medium, high, critical] }
+    suggested_fix:    { type: string }
+  required:
+    - bug_identified
+    - bug_location
+    - bug_type
+    - bug_description
+    - severity
+    - suggested_fix
+
+observation_space:
+  type: object
+  properties:
+    code_snippet:       { type: string }
+    language:           { type: string }
+    task_description:   { type: string }
+    task_id:            { type: string }
+    difficulty:         { type: string, enum: [easy, medium, hard] }
+    step_number:        { type: integer }
+    max_steps:          { type: integer }
+    previous_feedback:  { type: string, nullable: true }
+
+reward:
+  min: 0.0
+  max: 1.0
+  description: >
+    Partial rewards for: bug identification (0.20), correct bug type (0.20),
+    precise location (0.10), description quality (0.25, keyword density),
+    fix quality (0.15, keyword density), correct severity (0.10).
+
+endpoints:
+  health: GET  /health
+  reset:  POST /reset
+  step:   POST /step
+  state:  GET  /state

requirements.txt

@@ -0,0 +1,5 @@
+fastapi==0.115.0
+uvicorn[standard]==0.30.6
+pydantic==2.7.4
+requests==2.32.3
+openai==1.40.0