dwehr commited on
Commit
fd602be
·
1 Parent(s): 552590d

fix thread leaks

Browse files
cosmos-framework/cosmos_framework/data/vfm/action/action_viz/app.py CHANGED
@@ -356,6 +356,7 @@ def launch_action_viz(config: AppConfig) -> None:
356
  renderer = UnifiedRenderer(client)
357
  sample_state: dict[str, LoadedSample | None] = {"loaded": None}
358
  playing = {"enabled": False, "last_frame_time": _time.time()}
 
359
  generation_thread = {"thread": None}
360
  generation_lock = threading.Lock()
361
  control_handles: dict[int, Any] = {}
@@ -1161,10 +1162,8 @@ def launch_action_viz(config: AppConfig) -> None:
1161
  if loaded is not None:
1162
  _set_action_plot(plot_handle, loaded.baked_action, channel_visible, loaded.adapter.channel_names)
1163
 
1164
- load_sample(int(datasets[config.default_dataset].initial_index))
1165
-
1166
  def _play_loop() -> None:
1167
- while True:
1168
  loaded = sample_state["loaded"]
1169
  if loaded is not None and playing["enabled"]:
1170
  now = _time.time()
@@ -1172,20 +1171,73 @@ def launch_action_viz(config: AppConfig) -> None:
1172
  if now - playing["last_frame_time"] >= frame_period:
1173
  time_slider.value = (int(time_slider.value) + 1) % max(len(loaded.baked_action), 1)
1174
  playing["last_frame_time"] = now
1175
- _time.sleep(0.02)
1176
 
1177
- threading.Thread(target=_play_loop, daemon=True).start()
 
 
 
 
 
 
 
 
1178
  with sessions_lock:
1179
- sessions[client.client_id] = {"client": client, "scene_update_worker": scene_update_worker}
 
 
 
 
 
 
 
 
 
1180
 
1181
  @server.on_client_disconnect
1182
  def _(client) -> None:
1183
  with sessions_lock:
1184
  session = sessions.pop(client.client_id, None)
1185
  if session is not None:
 
 
 
1186
  worker = session.get("scene_update_worker")
1187
  if isinstance(worker, _LatestOnlyIkWorker):
1188
  worker.stop()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1189
 
1190
  log.info(f"Action viz ready at http://0.0.0.0:{config.port}")
1191
  if config.share:
 
356
  renderer = UnifiedRenderer(client)
357
  sample_state: dict[str, LoadedSample | None] = {"loaded": None}
358
  playing = {"enabled": False, "last_frame_time": _time.time()}
359
+ play_stop_event = threading.Event()
360
  generation_thread = {"thread": None}
361
  generation_lock = threading.Lock()
362
  control_handles: dict[int, Any] = {}
 
1162
  if loaded is not None:
1163
  _set_action_plot(plot_handle, loaded.baked_action, channel_visible, loaded.adapter.channel_names)
1164
 
 
 
1165
  def _play_loop() -> None:
1166
+ while not play_stop_event.wait(timeout=0.02):
1167
  loaded = sample_state["loaded"]
1168
  if loaded is not None and playing["enabled"]:
1169
  now = _time.time()
 
1171
  if now - playing["last_frame_time"] >= frame_period:
1172
  time_slider.value = (int(time_slider.value) + 1) % max(len(loaded.baked_action), 1)
1173
  playing["last_frame_time"] = now
 
1174
 
1175
+ play_thread = threading.Thread(
1176
+ target=_play_loop,
1177
+ name=f"action-viz-playback-client-{client.client_id}",
1178
+ daemon=True,
1179
+ )
1180
+ # Register the session before starting the thread so a disconnect that
1181
+ # races ahead of this handler can always find the session, set the stop
1182
+ # event, and stop the workers. Otherwise the play thread and IK worker
1183
+ # leak permanently for that client.
1184
  with sessions_lock:
1185
+ sessions[client.client_id] = {
1186
+ "client": client,
1187
+ "scene_update_worker": scene_update_worker,
1188
+ "play_stop_event": play_stop_event,
1189
+ "play_thread": play_thread,
1190
+ }
1191
+ play_thread.start()
1192
+
1193
+ if not play_stop_event.is_set():
1194
+ load_sample(int(datasets[config.default_dataset].initial_index))
1195
 
1196
  @server.on_client_disconnect
1197
  def _(client) -> None:
1198
  with sessions_lock:
1199
  session = sessions.pop(client.client_id, None)
1200
  if session is not None:
1201
+ play_stop_event = session.get("play_stop_event")
1202
+ if isinstance(play_stop_event, threading.Event):
1203
+ play_stop_event.set()
1204
  worker = session.get("scene_update_worker")
1205
  if isinstance(worker, _LatestOnlyIkWorker):
1206
  worker.stop()
1207
+ play_thread = session.get("play_thread")
1208
+ # is_alive() guards against a disconnect that races in before the
1209
+ # connect handler called start(): join() would raise RuntimeError on
1210
+ # an unstarted thread. Once started, the thread exits promptly since
1211
+ # the stop event is already set, so it needs no join here.
1212
+ if isinstance(play_thread, threading.Thread) and play_thread.is_alive():
1213
+ play_thread.join(timeout=1.0)
1214
+
1215
+ # Orphan-leak watchdog. A disconnect that lands before the connect
1216
+ # handler registers its session (sessions.pop -> None above) can't stop
1217
+ # the per-client IK/playback threads it would otherwise own, leaking an
1218
+ # idle daemon thread. We chose not to close that narrow race, so instead
1219
+ # surface accumulation: count live "action-viz-*client-<id>" threads whose
1220
+ # client_id has no active session. A handful are normal transients from
1221
+ # clients mid-connect (the IK thread starts before registration); a
1222
+ # sustained count above the threshold means the race is being hit often
1223
+ # enough to revisit the two-phase-registration fix. Runs after the
1224
+ # joins above so this client's own threads are gone, not miscounted.
1225
+ orphan_warn_threshold = 8
1226
+ with sessions_lock:
1227
+ active_ids = {str(client_id) for client_id in sessions}
1228
+ orphan_threads = [
1229
+ thread.name
1230
+ for thread in threading.enumerate()
1231
+ if thread.is_alive()
1232
+ and thread.name.startswith("action-viz-")
1233
+ and thread.name.rsplit("client-", 1)[-1] not in active_ids
1234
+ ]
1235
+ if len(orphan_threads) >= orphan_warn_threshold:
1236
+ log.warning(
1237
+ f"action-viz: {len(orphan_threads)} orphaned client threads alive with "
1238
+ f"no active session (e.g. {orphan_threads[:4]}) across {len(active_ids)} "
1239
+ f"active sessions; connect/disconnect race thread leak may be accumulating."
1240
+ )
1241
 
1242
  log.info(f"Action viz ready at http://0.0.0.0:{config.port}")
1243
  if config.share: