yashash04 commited on
Commit
85b09ec
·
1 Parent(s): a43e8f7

Phase M-prep fix: capture env-server logs + extend boot timeout in GRPO wrapper

Browse files

Stage 2 [4/7] failed with 'env-server failed to come up in 28s'. The
wrapper discarded uvicorn's stdout+stderr to DEVNULL, so we had no
diagnostic info on what actually went wrong inside the container.

Changes to scripts/phase_m_grpo_job.py [4/7] block:
- Capture uvicorn stdout+stderr to /workspace/env_server.log (NOT discard)
- Extend boot wait from 8s warm + 20s polling = 28s, to 15s warm + 90s
polling = 105s total
- Detect process death during boot (poll exit code) and dump log
- Add /tasks endpoint check after /reset succeeds (stricter health gate)
- On any failure, print last 3KB of env_server.log so next run is debuggable

Yashash will relaunch Stage 2 manually after push.

Files changed (1) hide show
  1. scripts/phase_m_grpo_job.py +50 -12
scripts/phase_m_grpo_job.py CHANGED
@@ -63,19 +63,39 @@ print(f" SFT adapter at: {sft_path}", flush=True)
63
 
64
  # Step 4: Boot env-server in background
65
  print("[4/7] Booting env-server on localhost:8000...", flush=True)
 
 
66
  env_proc = subprocess.Popen(
67
  [
68
  sys.executable, "-m", "uvicorn", "server.app:app",
69
  "--host", "127.0.0.1", "--port", "8000",
 
70
  ],
71
- stdout=subprocess.DEVNULL,
72
- stderr=subprocess.DEVNULL,
73
  )
 
74
 
75
- time.sleep(8)
 
76
 
77
- # Verify env is responding
78
- for i in range(10):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
79
  try:
80
  req = urllib.request.Request(
81
  "http://localhost:8000/reset",
@@ -85,14 +105,32 @@ for i in range(10):
85
  )
86
  with urllib.request.urlopen(req, timeout=5) as r:
87
  if r.status == 200:
88
- print(f" Env-server is healthy (after {8 + i*2}s)", flush=True)
89
- break
90
- except Exception:
91
- pass
92
- time.sleep(2)
93
- else:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
94
  env_proc.terminate()
95
- raise RuntimeError("env-server failed to come up in 28s")
 
 
 
 
96
 
97
  try:
98
  # Step 5: Run GRPO training
 
63
 
64
  # Step 4: Boot env-server in background
65
  print("[4/7] Booting env-server on localhost:8000...", flush=True)
66
+ env_log_path = "/workspace/env_server.log"
67
+ env_log_fh = open(env_log_path, "w")
68
  env_proc = subprocess.Popen(
69
  [
70
  sys.executable, "-m", "uvicorn", "server.app:app",
71
  "--host", "127.0.0.1", "--port", "8000",
72
+ "--log-level", "info",
73
  ],
74
+ stdout=env_log_fh,
75
+ stderr=subprocess.STDOUT,
76
  )
77
+ print(f" env-server PID: {env_proc.pid}, logs: {env_log_path}", flush=True)
78
 
79
+ # Give uvicorn time to boot. L4 cold start + import-time work can be slow.
80
+ time.sleep(15)
81
 
82
+ health_ok = False
83
+ last_error = None
84
+
85
+ # Try up to 90 more seconds (60 attempts at 1.5s each)
86
+ for i in range(60):
87
+ # Check if process died
88
+ if env_proc.poll() is not None:
89
+ env_log_fh.flush()
90
+ with open(env_log_path) as f:
91
+ log_content = f.read()
92
+ print(f" env-server PROCESS DIED with exit code {env_proc.returncode}", flush=True)
93
+ print(f" --- env-server log ---", flush=True)
94
+ print(log_content[-3000:], flush=True)
95
+ print(f" --- end log ---", flush=True)
96
+ raise RuntimeError(f"env-server died during boot. Exit: {env_proc.returncode}")
97
+
98
+ # Try /reset health check
99
  try:
100
  req = urllib.request.Request(
101
  "http://localhost:8000/reset",
 
105
  )
106
  with urllib.request.urlopen(req, timeout=5) as r:
107
  if r.status == 200:
108
+ # Also verify /tasks responds (stricter check)
109
+ req2 = urllib.request.Request("http://localhost:8000/tasks")
110
+ with urllib.request.urlopen(req2, timeout=5) as r2:
111
+ if r2.status == 200:
112
+ print(f" Env-server is healthy (after {15 + i*1.5:.0f}s)", flush=True)
113
+ health_ok = True
114
+ break
115
+ except Exception as e:
116
+ last_error = str(e)[:120]
117
+
118
+ time.sleep(1.5)
119
+
120
+ if not health_ok:
121
+ env_log_fh.flush()
122
+ with open(env_log_path) as f:
123
+ log_content = f.read()
124
+ print(f" Health check FAILED. Last error: {last_error}", flush=True)
125
+ print(f" --- env-server log (last 3KB) ---", flush=True)
126
+ print(log_content[-3000:], flush=True)
127
+ print(f" --- end log ---", flush=True)
128
  env_proc.terminate()
129
+ try:
130
+ env_proc.wait(timeout=5)
131
+ except subprocess.TimeoutExpired:
132
+ env_proc.kill()
133
+ raise RuntimeError(f"env-server failed to come up in 105s")
134
 
135
  try:
136
  # Step 5: Run GRPO training