ragavrida commited on
Commit
1a6681a
Β·
1 Parent(s): 7e55f05

Fix critical crash bug + audit fixes

Browse files

1. CRASH FIX: grader_hard.py line 213 β€” critical_caught was
undefined (NameError). Now computed from _bugs_caught() before use.
This would crash if any LLM approved a critical-severity PR.

2. README: Fixed stale references:
- Language enum: 4 β†’ 7 (rust, typescript, ruby added)
- Test count: 19 β†’ 21
- HF URL: openenv β†’ ragavrida
- Validation count: 15 β†’ 17

3. inference.py: Added 3-attempt retry with exponential backoff
for LLM API calls to handle transient failures.

4. env/models.py: Updated language comment to match validator.

All verified: 21/21 tests, 17/17 validation, baseline runs.

Files changed (4) hide show
  1. README.md +4 -4
  2. env/models.py +1 -1
  3. graders/grader_hard.py +2 -0
  4. inference.py +21 -17
README.md CHANGED
@@ -136,7 +136,7 @@ Each file in `files` contains:
136
  | Field | Type | Description |
137
  |-------|------|-------------|
138
  | `filename` | `str` | File path (e.g. `UserService.java`) |
139
- | `language` | `str ∈ {python, javascript, java, go}` | Programming language |
140
  | `diff` | `str` | Unified diff of changes |
141
  | `lines_changed` | `int` | Number of modified lines |
142
  | `has_tests` | `bool` | Whether the PR includes test coverage |
@@ -463,13 +463,13 @@ The inference script emits structured stdout logs in the **mandatory** `[START]`
463
 
464
  | Check | Command / Verification | Status |
465
  |-------|----------------------|--------|
466
- | HF Space deploys | `curl https://openenv-code-review-env.hf.space/health` returns 200 | βœ… |
467
- | OpenEnv spec compliance | `python validate.py` β€” all 15 checks pass | βœ… |
468
  | Dockerfile builds | `docker build -t code-review-env .` | βœ… |
469
  | Baseline reproduces | `python baseline.py` completes end-to-end without errors | βœ… |
470
  | LLM inference | `python inference.py` completes with API key, saves `baseline/results.json` | βœ… |
471
  | 3+ tasks with graders | `easy`, `medium`, `hard` β€” all graders produce scores in `[0.0, 1.0]` | βœ… |
472
- | Tests pass | `pytest tests/ -v` β€” 19 tests across 5 categories | βœ… |
473
  | `API_BASE_URL` defined | Used by `inference.py` | βœ… |
474
  | `MODEL_NAME` defined | Used by `inference.py` | βœ… |
475
  | `HF_TOKEN` defined | Used by `inference.py` | βœ… |
 
136
  | Field | Type | Description |
137
  |-------|------|-------------|
138
  | `filename` | `str` | File path (e.g. `UserService.java`) |
139
+ | `language` | `str ∈ {python, javascript, java, go, rust, typescript, ruby}` | Programming language |
140
  | `diff` | `str` | Unified diff of changes |
141
  | `lines_changed` | `int` | Number of modified lines |
142
  | `has_tests` | `bool` | Whether the PR includes test coverage |
 
463
 
464
  | Check | Command / Verification | Status |
465
  |-------|----------------------|--------|
466
+ | HF Space deploys | `curl https://ragavrida-code-review-env.hf.space/health` returns 200 | βœ… |
467
+ | OpenEnv spec compliance | `python validate.py` β€” all 17 checks pass | βœ… |
468
  | Dockerfile builds | `docker build -t code-review-env .` | βœ… |
469
  | Baseline reproduces | `python baseline.py` completes end-to-end without errors | βœ… |
470
  | LLM inference | `python inference.py` completes with API key, saves `baseline/results.json` | βœ… |
471
  | 3+ tasks with graders | `easy`, `medium`, `hard` β€” all graders produce scores in `[0.0, 1.0]` | βœ… |
472
+ | Tests pass | `pytest tests/ -v` β€” 21 tests across 6 categories | βœ… |
473
  | `API_BASE_URL` defined | Used by `inference.py` | βœ… |
474
  | `MODEL_NAME` defined | Used by `inference.py` | βœ… |
475
  | `HF_TOKEN` defined | Used by `inference.py` | βœ… |
env/models.py CHANGED
@@ -18,7 +18,7 @@ from typing import List, Optional, Dict, Any
18
  class PRFile(BaseModel):
19
  """A single file within a pull request diff."""
20
  filename: str
21
- language: str # python | javascript | java | go
22
  diff: str
23
  lines_changed: int
24
  has_tests: bool
 
18
  class PRFile(BaseModel):
19
  """A single file within a pull request diff."""
20
  filename: str
21
+ language: str # python | javascript | java | go | rust | typescript | ruby
22
  diff: str
23
  lines_changed: int
24
  has_tests: bool
graders/grader_hard.py CHANGED
@@ -210,6 +210,8 @@ class HardGrader:
210
 
211
  # ── Exploit: approve with unaddressed critical bug ───────────
212
  breakdown["critical_approve_penalty"] = 0.0
 
 
213
  if decision == "approve" and true_severity == "critical" and critical_caught == 0:
214
  breakdown["critical_approve_penalty"] = -0.5
215
  self.episode_penalties += -0.5
 
210
 
211
  # ── Exploit: approve with unaddressed critical bug ───────────
212
  breakdown["critical_approve_penalty"] = 0.0
213
+ bugs = self._bugs_caught(comments, gt)
214
+ critical_caught = bugs.get("critical", 0)
215
  if decision == "approve" and true_severity == "critical" and critical_caught == 0:
216
  breakdown["critical_approve_penalty"] = -0.5
217
  self.episode_penalties += -0.5
inference.py CHANGED
@@ -86,23 +86,27 @@ def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> No
86
 
87
  # ─── LLM Interface ──────────────────────────────────────────────────────────
88
 
89
- def call_llm(client: OpenAI, system_prompt: str, user_prompt: str) -> str:
90
- """Call the LLM using OpenAI Client. Returns response text."""
91
- try:
92
- completion = client.chat.completions.create(
93
- model=MODEL_NAME,
94
- messages=[
95
- {"role": "system", "content": system_prompt},
96
- {"role": "user", "content": user_prompt},
97
- ],
98
- temperature=TEMPERATURE,
99
- max_tokens=MAX_TOKENS,
100
- stream=False,
101
- )
102
- return (completion.choices[0].message.content or "").strip()
103
- except Exception as exc:
104
- print(f"[DEBUG] Model request failed: {exc}", file=sys.stderr, flush=True)
105
- return ""
 
 
 
 
106
 
107
 
108
  def parse_json_response(response: str) -> Optional[Dict]:
 
86
 
87
  # ─── LLM Interface ──────────────────────────────────────────────────────────
88
 
89
+ def call_llm(client: OpenAI, system_prompt: str, user_prompt: str, max_retries: int = 3) -> str:
90
+ """Call the LLM using OpenAI Client with retry. Returns response text."""
91
+ for attempt in range(max_retries):
92
+ try:
93
+ completion = client.chat.completions.create(
94
+ model=MODEL_NAME,
95
+ messages=[
96
+ {"role": "system", "content": system_prompt},
97
+ {"role": "user", "content": user_prompt},
98
+ ],
99
+ temperature=TEMPERATURE,
100
+ max_tokens=MAX_TOKENS,
101
+ stream=False,
102
+ )
103
+ return (completion.choices[0].message.content or "").strip()
104
+ except Exception as exc:
105
+ print(f"[DEBUG] Attempt {attempt+1}/{max_retries} failed: {exc}", file=sys.stderr, flush=True)
106
+ if attempt < max_retries - 1:
107
+ import time
108
+ time.sleep(2 ** attempt)
109
+ return ""
110
 
111
 
112
  def parse_json_response(response: str) -> Optional[Dict]: