Spaces:
Runtime error
Runtime error
Fix critical crash bug + audit fixes
Browse files1. CRASH FIX: grader_hard.py line 213 β critical_caught was
undefined (NameError). Now computed from _bugs_caught() before use.
This would crash if any LLM approved a critical-severity PR.
2. README: Fixed stale references:
- Language enum: 4 β 7 (rust, typescript, ruby added)
- Test count: 19 β 21
- HF URL: openenv β ragavrida
- Validation count: 15 β 17
3. inference.py: Added 3-attempt retry with exponential backoff
for LLM API calls to handle transient failures.
4. env/models.py: Updated language comment to match validator.
All verified: 21/21 tests, 17/17 validation, baseline runs.
- README.md +4 -4
- env/models.py +1 -1
- graders/grader_hard.py +2 -0
- inference.py +21 -17
README.md
CHANGED
|
@@ -136,7 +136,7 @@ Each file in `files` contains:
|
|
| 136 |
| Field | Type | Description |
|
| 137 |
|-------|------|-------------|
|
| 138 |
| `filename` | `str` | File path (e.g. `UserService.java`) |
|
| 139 |
-
| `language` | `str β {python, javascript, java, go}` | Programming language |
|
| 140 |
| `diff` | `str` | Unified diff of changes |
|
| 141 |
| `lines_changed` | `int` | Number of modified lines |
|
| 142 |
| `has_tests` | `bool` | Whether the PR includes test coverage |
|
|
@@ -463,13 +463,13 @@ The inference script emits structured stdout logs in the **mandatory** `[START]`
|
|
| 463 |
|
| 464 |
| Check | Command / Verification | Status |
|
| 465 |
|-------|----------------------|--------|
|
| 466 |
-
| HF Space deploys | `curl https://
|
| 467 |
-
| OpenEnv spec compliance | `python validate.py` β all
|
| 468 |
| Dockerfile builds | `docker build -t code-review-env .` | β
|
|
| 469 |
| Baseline reproduces | `python baseline.py` completes end-to-end without errors | β
|
|
| 470 |
| LLM inference | `python inference.py` completes with API key, saves `baseline/results.json` | β
|
|
| 471 |
| 3+ tasks with graders | `easy`, `medium`, `hard` β all graders produce scores in `[0.0, 1.0]` | β
|
|
| 472 |
-
| Tests pass | `pytest tests/ -v` β
|
| 473 |
| `API_BASE_URL` defined | Used by `inference.py` | β
|
|
| 474 |
| `MODEL_NAME` defined | Used by `inference.py` | β
|
|
| 475 |
| `HF_TOKEN` defined | Used by `inference.py` | β
|
|
|
|
|
| 136 |
| Field | Type | Description |
|
| 137 |
|-------|------|-------------|
|
| 138 |
| `filename` | `str` | File path (e.g. `UserService.java`) |
|
| 139 |
+
| `language` | `str β {python, javascript, java, go, rust, typescript, ruby}` | Programming language |
|
| 140 |
| `diff` | `str` | Unified diff of changes |
|
| 141 |
| `lines_changed` | `int` | Number of modified lines |
|
| 142 |
| `has_tests` | `bool` | Whether the PR includes test coverage |
|
|
|
|
| 463 |
|
| 464 |
| Check | Command / Verification | Status |
|
| 465 |
|-------|----------------------|--------|
|
| 466 |
+
| HF Space deploys | `curl https://ragavrida-code-review-env.hf.space/health` returns 200 | β
|
|
| 467 |
+
| OpenEnv spec compliance | `python validate.py` β all 17 checks pass | β
|
|
| 468 |
| Dockerfile builds | `docker build -t code-review-env .` | β
|
|
| 469 |
| Baseline reproduces | `python baseline.py` completes end-to-end without errors | β
|
|
| 470 |
| LLM inference | `python inference.py` completes with API key, saves `baseline/results.json` | β
|
|
| 471 |
| 3+ tasks with graders | `easy`, `medium`, `hard` β all graders produce scores in `[0.0, 1.0]` | β
|
|
| 472 |
+
| Tests pass | `pytest tests/ -v` β 21 tests across 6 categories | β
|
|
| 473 |
| `API_BASE_URL` defined | Used by `inference.py` | β
|
|
| 474 |
| `MODEL_NAME` defined | Used by `inference.py` | β
|
|
| 475 |
| `HF_TOKEN` defined | Used by `inference.py` | β
|
|
env/models.py
CHANGED
|
@@ -18,7 +18,7 @@ from typing import List, Optional, Dict, Any
|
|
| 18 |
class PRFile(BaseModel):
|
| 19 |
"""A single file within a pull request diff."""
|
| 20 |
filename: str
|
| 21 |
-
language: str # python | javascript | java | go
|
| 22 |
diff: str
|
| 23 |
lines_changed: int
|
| 24 |
has_tests: bool
|
|
|
|
| 18 |
class PRFile(BaseModel):
|
| 19 |
"""A single file within a pull request diff."""
|
| 20 |
filename: str
|
| 21 |
+
language: str # python | javascript | java | go | rust | typescript | ruby
|
| 22 |
diff: str
|
| 23 |
lines_changed: int
|
| 24 |
has_tests: bool
|
graders/grader_hard.py
CHANGED
|
@@ -210,6 +210,8 @@ class HardGrader:
|
|
| 210 |
|
| 211 |
# ββ Exploit: approve with unaddressed critical bug βββββββββββ
|
| 212 |
breakdown["critical_approve_penalty"] = 0.0
|
|
|
|
|
|
|
| 213 |
if decision == "approve" and true_severity == "critical" and critical_caught == 0:
|
| 214 |
breakdown["critical_approve_penalty"] = -0.5
|
| 215 |
self.episode_penalties += -0.5
|
|
|
|
| 210 |
|
| 211 |
# ββ Exploit: approve with unaddressed critical bug βββββββββββ
|
| 212 |
breakdown["critical_approve_penalty"] = 0.0
|
| 213 |
+
bugs = self._bugs_caught(comments, gt)
|
| 214 |
+
critical_caught = bugs.get("critical", 0)
|
| 215 |
if decision == "approve" and true_severity == "critical" and critical_caught == 0:
|
| 216 |
breakdown["critical_approve_penalty"] = -0.5
|
| 217 |
self.episode_penalties += -0.5
|
inference.py
CHANGED
|
@@ -86,23 +86,27 @@ def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> No
|
|
| 86 |
|
| 87 |
# βββ LLM Interface ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 88 |
|
| 89 |
-
def call_llm(client: OpenAI, system_prompt: str, user_prompt: str) -> str:
|
| 90 |
-
"""Call the LLM using OpenAI Client. Returns response text."""
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 106 |
|
| 107 |
|
| 108 |
def parse_json_response(response: str) -> Optional[Dict]:
|
|
|
|
| 86 |
|
| 87 |
# βββ LLM Interface ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 88 |
|
| 89 |
+
def call_llm(client: OpenAI, system_prompt: str, user_prompt: str, max_retries: int = 3) -> str:
|
| 90 |
+
"""Call the LLM using OpenAI Client with retry. Returns response text."""
|
| 91 |
+
for attempt in range(max_retries):
|
| 92 |
+
try:
|
| 93 |
+
completion = client.chat.completions.create(
|
| 94 |
+
model=MODEL_NAME,
|
| 95 |
+
messages=[
|
| 96 |
+
{"role": "system", "content": system_prompt},
|
| 97 |
+
{"role": "user", "content": user_prompt},
|
| 98 |
+
],
|
| 99 |
+
temperature=TEMPERATURE,
|
| 100 |
+
max_tokens=MAX_TOKENS,
|
| 101 |
+
stream=False,
|
| 102 |
+
)
|
| 103 |
+
return (completion.choices[0].message.content or "").strip()
|
| 104 |
+
except Exception as exc:
|
| 105 |
+
print(f"[DEBUG] Attempt {attempt+1}/{max_retries} failed: {exc}", file=sys.stderr, flush=True)
|
| 106 |
+
if attempt < max_retries - 1:
|
| 107 |
+
import time
|
| 108 |
+
time.sleep(2 ** attempt)
|
| 109 |
+
return ""
|
| 110 |
|
| 111 |
|
| 112 |
def parse_json_response(response: str) -> Optional[Dict]:
|