Upload jobs/run_real_llm_standalone_v7.py

jobs/run_real_llm_standalone_v7.py

@@ -1,10 +1,10 @@
 """
 Self-contained GPU job for real LLM code benchmark — V7.
 FIXES:
-1. Better markdown stripping (only remove outer fences, not internal backticks)
+1. Regex-based markdown extraction (handles ```python...``` blocks)
 2. Larger model: Qwen2.5-Coder-1.5B-Instruct
 3. 512 max_new_tokens
-4. Better error logging: shows the actual test file that failed
+4. Debug: show stripped code for first 3 failures
 """
 import json
 import os
@@ -136,20 +136,22 @@ class ResourceBroker:
 
 # --- HELPERS ---
 
-def strip_markdown_fences(text: str) -> str:
-    """Remove only the outer markdown code fences, not internal backticks."""
+def extract_code_block(text: str) -> str:
+    """Extract code from markdown fenced code block."""
     text = text.strip()
-    # Remove leading ```python or ``` on first line only
-    if text.startswith("```"):
-        first_newline = text.find("\n")
-        if first_newline != -1:
-            text = text[first_newline+1:]
-    # Remove trailing ``` only if on the last line
-    lines = text.splitlines()
-    if lines and lines[-1].strip() == "```":
-        lines = lines[:-1]
-    text = "\n".join(lines)
-    return text.strip()
+    # Match ```python or ``` followed by content until ```
+    match = re.search(r'```(?:\w+)?\s*\n(.*?)\n```', text, re.DOTALL)
+    if match:
+        return match.group(1).strip()
+    # Also try without newline after opening fence
+    match2 = re.search(r'```(?:\w+)?\s*\n(.*)', text, re.DOTALL)
+    if match2:
+        candidate = match2.group(1).strip()
+        # Remove trailing ``` if present
+        if candidate.endswith("```"):
+            candidate = candidate[:-3].strip()
+        return candidate
+    return text
 
 
 def contains_function_definition(code: str, entry_point: str) -> bool:
@@ -226,7 +228,7 @@ class RealLLMBenchmarkV7:
     def evaluate_one(self, problem, model, tok, max_new_tokens=512):
         raw = self.generate(model, tok, problem["prompt"], max_new_tokens=max_new_tokens)
         tokens = len(tok.encode(raw))
-        code = strip_markdown_fences(raw)
+        code = extract_code_block(raw)
 
         # Strategy 1: Generated code is complete function
         if contains_function_definition(code, problem["entry_point"]):
@@ -249,24 +251,23 @@ class RealLLMBenchmarkV7:
             else:
                 error = error if len(error) < len(error2) else error2
 
-        return passed, tokens, raw, error
+        return passed, tokens, raw, error, code
 
     def run_baseline(self, problems, model, tok, max_new_tokens=512):
         results = []; total_compute = 0
-        for problem in problems:
-            passed, tokens, raw, error = self.evaluate_one(problem, model, tok, max_new_tokens)
+        for i, problem in enumerate(problems):
+            passed, tokens, raw, error, code = self.evaluate_one(problem, model, tok, max_new_tokens)
             total_compute += tokens
             results.append({"task_id": problem["task_id"], "passed": passed, "tokens": tokens, "raw": raw[:300], "error": error[:200]})
             print(f"  {problem['task_id']}: passed={passed}, tokens={tokens}")
             if not passed:
                 print(f"    error={error[:200]!r}")
-                # Debug: show test file for first 3 failures
-                code = strip_markdown_fences(raw)
-                if contains_function_definition(code, problem["entry_point"]):
-                    test_file = code + "\n\n" + problem["test"] + "\n\ncheck()\n"
-                else:
-                    test_file = problem["prompt"] + code + "\n\n" + problem["test"] + "\n\ncheck()\n"
-                print(f"    [DEBUG test file first 300 chars]: {test_file[:300]!r}")
+                if i < 3:  # Debug first 3 failures
+                    print(f"    [STRIPPED CODE first 300 chars]:")
+                    print(code[:300])
+                    print(f"    [TEST FILE first 300 chars]:")
+                    test_file = code + "\n\n" + problem["test"] + "\n\ncheck()\n" if contains_function_definition(code, problem["entry_point"]) else problem["prompt"] + code + "\n\n" + problem["test"] + "\n\ncheck()\n"
+                    print(test_file[:300])
         return {"accuracy": sum(1 for r in results if r["passed"]) / len(results), "total_compute": total_compute, "results": results}
 
     def run_occ(self, problems, model, tok, max_new_tokens_first=256, max_new_tokens_retry=512):
@@ -286,7 +287,7 @@ class RealLLMBenchmarkV7:
                 max_tok = max_new_tokens_first if attempts == 1 else max_new_tokens_retry
                 code_raw = self.generate(model, tok, problem["prompt"], max_new_tokens=max_tok)
                 tokens = len(tok.encode(code_raw)); budget_remaining -= tokens; total_compute += tokens
-                code = strip_markdown_fences(code_raw)
+                code = extract_code_block(code_raw)
                 if contains_function_definition(code, problem["entry_point"]):
                     test_code = code
                 else: