Spaces:
Running
Running
Phase M-prep fix: capture env-server logs + extend boot timeout in GRPO wrapper
Browse filesStage 2 [4/7] failed with 'env-server failed to come up in 28s'. The
wrapper discarded uvicorn's stdout+stderr to DEVNULL, so we had no
diagnostic info on what actually went wrong inside the container.
Changes to scripts/phase_m_grpo_job.py [4/7] block:
- Capture uvicorn stdout+stderr to /workspace/env_server.log (NOT discard)
- Extend boot wait from 8s warm + 20s polling = 28s, to 15s warm + 90s
polling = 105s total
- Detect process death during boot (poll exit code) and dump log
- Add /tasks endpoint check after /reset succeeds (stricter health gate)
- On any failure, print last 3KB of env_server.log so next run is debuggable
Yashash will relaunch Stage 2 manually after push.
- scripts/phase_m_grpo_job.py +50 -12
scripts/phase_m_grpo_job.py
CHANGED
|
@@ -63,19 +63,39 @@ print(f" SFT adapter at: {sft_path}", flush=True)
|
|
| 63 |
|
| 64 |
# Step 4: Boot env-server in background
|
| 65 |
print("[4/7] Booting env-server on localhost:8000...", flush=True)
|
|
|
|
|
|
|
| 66 |
env_proc = subprocess.Popen(
|
| 67 |
[
|
| 68 |
sys.executable, "-m", "uvicorn", "server.app:app",
|
| 69 |
"--host", "127.0.0.1", "--port", "8000",
|
|
|
|
| 70 |
],
|
| 71 |
-
stdout=
|
| 72 |
-
stderr=subprocess.
|
| 73 |
)
|
|
|
|
| 74 |
|
| 75 |
-
time.
|
|
|
|
| 76 |
|
| 77 |
-
|
| 78 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 79 |
try:
|
| 80 |
req = urllib.request.Request(
|
| 81 |
"http://localhost:8000/reset",
|
|
@@ -85,14 +105,32 @@ for i in range(10):
|
|
| 85 |
)
|
| 86 |
with urllib.request.urlopen(req, timeout=5) as r:
|
| 87 |
if r.status == 200:
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 94 |
env_proc.terminate()
|
| 95 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 96 |
|
| 97 |
try:
|
| 98 |
# Step 5: Run GRPO training
|
|
|
|
| 63 |
|
| 64 |
# Step 4: Boot env-server in background
|
| 65 |
print("[4/7] Booting env-server on localhost:8000...", flush=True)
|
| 66 |
+
env_log_path = "/workspace/env_server.log"
|
| 67 |
+
env_log_fh = open(env_log_path, "w")
|
| 68 |
env_proc = subprocess.Popen(
|
| 69 |
[
|
| 70 |
sys.executable, "-m", "uvicorn", "server.app:app",
|
| 71 |
"--host", "127.0.0.1", "--port", "8000",
|
| 72 |
+
"--log-level", "info",
|
| 73 |
],
|
| 74 |
+
stdout=env_log_fh,
|
| 75 |
+
stderr=subprocess.STDOUT,
|
| 76 |
)
|
| 77 |
+
print(f" env-server PID: {env_proc.pid}, logs: {env_log_path}", flush=True)
|
| 78 |
|
| 79 |
+
# Give uvicorn time to boot. L4 cold start + import-time work can be slow.
|
| 80 |
+
time.sleep(15)
|
| 81 |
|
| 82 |
+
health_ok = False
|
| 83 |
+
last_error = None
|
| 84 |
+
|
| 85 |
+
# Try up to 90 more seconds (60 attempts at 1.5s each)
|
| 86 |
+
for i in range(60):
|
| 87 |
+
# Check if process died
|
| 88 |
+
if env_proc.poll() is not None:
|
| 89 |
+
env_log_fh.flush()
|
| 90 |
+
with open(env_log_path) as f:
|
| 91 |
+
log_content = f.read()
|
| 92 |
+
print(f" env-server PROCESS DIED with exit code {env_proc.returncode}", flush=True)
|
| 93 |
+
print(f" --- env-server log ---", flush=True)
|
| 94 |
+
print(log_content[-3000:], flush=True)
|
| 95 |
+
print(f" --- end log ---", flush=True)
|
| 96 |
+
raise RuntimeError(f"env-server died during boot. Exit: {env_proc.returncode}")
|
| 97 |
+
|
| 98 |
+
# Try /reset health check
|
| 99 |
try:
|
| 100 |
req = urllib.request.Request(
|
| 101 |
"http://localhost:8000/reset",
|
|
|
|
| 105 |
)
|
| 106 |
with urllib.request.urlopen(req, timeout=5) as r:
|
| 107 |
if r.status == 200:
|
| 108 |
+
# Also verify /tasks responds (stricter check)
|
| 109 |
+
req2 = urllib.request.Request("http://localhost:8000/tasks")
|
| 110 |
+
with urllib.request.urlopen(req2, timeout=5) as r2:
|
| 111 |
+
if r2.status == 200:
|
| 112 |
+
print(f" Env-server is healthy (after {15 + i*1.5:.0f}s)", flush=True)
|
| 113 |
+
health_ok = True
|
| 114 |
+
break
|
| 115 |
+
except Exception as e:
|
| 116 |
+
last_error = str(e)[:120]
|
| 117 |
+
|
| 118 |
+
time.sleep(1.5)
|
| 119 |
+
|
| 120 |
+
if not health_ok:
|
| 121 |
+
env_log_fh.flush()
|
| 122 |
+
with open(env_log_path) as f:
|
| 123 |
+
log_content = f.read()
|
| 124 |
+
print(f" Health check FAILED. Last error: {last_error}", flush=True)
|
| 125 |
+
print(f" --- env-server log (last 3KB) ---", flush=True)
|
| 126 |
+
print(log_content[-3000:], flush=True)
|
| 127 |
+
print(f" --- end log ---", flush=True)
|
| 128 |
env_proc.terminate()
|
| 129 |
+
try:
|
| 130 |
+
env_proc.wait(timeout=5)
|
| 131 |
+
except subprocess.TimeoutExpired:
|
| 132 |
+
env_proc.kill()
|
| 133 |
+
raise RuntimeError(f"env-server failed to come up in 105s")
|
| 134 |
|
| 135 |
try:
|
| 136 |
# Step 5: Run GRPO training
|