Spaces:
Running on L40S
Running on L40S
fix thread leaks
Browse files
cosmos-framework/cosmos_framework/data/vfm/action/action_viz/app.py
CHANGED
|
@@ -356,6 +356,7 @@ def launch_action_viz(config: AppConfig) -> None:
|
|
| 356 |
renderer = UnifiedRenderer(client)
|
| 357 |
sample_state: dict[str, LoadedSample | None] = {"loaded": None}
|
| 358 |
playing = {"enabled": False, "last_frame_time": _time.time()}
|
|
|
|
| 359 |
generation_thread = {"thread": None}
|
| 360 |
generation_lock = threading.Lock()
|
| 361 |
control_handles: dict[int, Any] = {}
|
|
@@ -1161,10 +1162,8 @@ def launch_action_viz(config: AppConfig) -> None:
|
|
| 1161 |
if loaded is not None:
|
| 1162 |
_set_action_plot(plot_handle, loaded.baked_action, channel_visible, loaded.adapter.channel_names)
|
| 1163 |
|
| 1164 |
-
load_sample(int(datasets[config.default_dataset].initial_index))
|
| 1165 |
-
|
| 1166 |
def _play_loop() -> None:
|
| 1167 |
-
while
|
| 1168 |
loaded = sample_state["loaded"]
|
| 1169 |
if loaded is not None and playing["enabled"]:
|
| 1170 |
now = _time.time()
|
|
@@ -1172,20 +1171,73 @@ def launch_action_viz(config: AppConfig) -> None:
|
|
| 1172 |
if now - playing["last_frame_time"] >= frame_period:
|
| 1173 |
time_slider.value = (int(time_slider.value) + 1) % max(len(loaded.baked_action), 1)
|
| 1174 |
playing["last_frame_time"] = now
|
| 1175 |
-
_time.sleep(0.02)
|
| 1176 |
|
| 1177 |
-
threading.Thread(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1178 |
with sessions_lock:
|
| 1179 |
-
sessions[client.client_id] = {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1180 |
|
| 1181 |
@server.on_client_disconnect
|
| 1182 |
def _(client) -> None:
|
| 1183 |
with sessions_lock:
|
| 1184 |
session = sessions.pop(client.client_id, None)
|
| 1185 |
if session is not None:
|
|
|
|
|
|
|
|
|
|
| 1186 |
worker = session.get("scene_update_worker")
|
| 1187 |
if isinstance(worker, _LatestOnlyIkWorker):
|
| 1188 |
worker.stop()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1189 |
|
| 1190 |
log.info(f"Action viz ready at http://0.0.0.0:{config.port}")
|
| 1191 |
if config.share:
|
|
|
|
| 356 |
renderer = UnifiedRenderer(client)
|
| 357 |
sample_state: dict[str, LoadedSample | None] = {"loaded": None}
|
| 358 |
playing = {"enabled": False, "last_frame_time": _time.time()}
|
| 359 |
+
play_stop_event = threading.Event()
|
| 360 |
generation_thread = {"thread": None}
|
| 361 |
generation_lock = threading.Lock()
|
| 362 |
control_handles: dict[int, Any] = {}
|
|
|
|
| 1162 |
if loaded is not None:
|
| 1163 |
_set_action_plot(plot_handle, loaded.baked_action, channel_visible, loaded.adapter.channel_names)
|
| 1164 |
|
|
|
|
|
|
|
| 1165 |
def _play_loop() -> None:
|
| 1166 |
+
while not play_stop_event.wait(timeout=0.02):
|
| 1167 |
loaded = sample_state["loaded"]
|
| 1168 |
if loaded is not None and playing["enabled"]:
|
| 1169 |
now = _time.time()
|
|
|
|
| 1171 |
if now - playing["last_frame_time"] >= frame_period:
|
| 1172 |
time_slider.value = (int(time_slider.value) + 1) % max(len(loaded.baked_action), 1)
|
| 1173 |
playing["last_frame_time"] = now
|
|
|
|
| 1174 |
|
| 1175 |
+
play_thread = threading.Thread(
|
| 1176 |
+
target=_play_loop,
|
| 1177 |
+
name=f"action-viz-playback-client-{client.client_id}",
|
| 1178 |
+
daemon=True,
|
| 1179 |
+
)
|
| 1180 |
+
# Register the session before starting the thread so a disconnect that
|
| 1181 |
+
# races ahead of this handler can always find the session, set the stop
|
| 1182 |
+
# event, and stop the workers. Otherwise the play thread and IK worker
|
| 1183 |
+
# leak permanently for that client.
|
| 1184 |
with sessions_lock:
|
| 1185 |
+
sessions[client.client_id] = {
|
| 1186 |
+
"client": client,
|
| 1187 |
+
"scene_update_worker": scene_update_worker,
|
| 1188 |
+
"play_stop_event": play_stop_event,
|
| 1189 |
+
"play_thread": play_thread,
|
| 1190 |
+
}
|
| 1191 |
+
play_thread.start()
|
| 1192 |
+
|
| 1193 |
+
if not play_stop_event.is_set():
|
| 1194 |
+
load_sample(int(datasets[config.default_dataset].initial_index))
|
| 1195 |
|
| 1196 |
@server.on_client_disconnect
|
| 1197 |
def _(client) -> None:
|
| 1198 |
with sessions_lock:
|
| 1199 |
session = sessions.pop(client.client_id, None)
|
| 1200 |
if session is not None:
|
| 1201 |
+
play_stop_event = session.get("play_stop_event")
|
| 1202 |
+
if isinstance(play_stop_event, threading.Event):
|
| 1203 |
+
play_stop_event.set()
|
| 1204 |
worker = session.get("scene_update_worker")
|
| 1205 |
if isinstance(worker, _LatestOnlyIkWorker):
|
| 1206 |
worker.stop()
|
| 1207 |
+
play_thread = session.get("play_thread")
|
| 1208 |
+
# is_alive() guards against a disconnect that races in before the
|
| 1209 |
+
# connect handler called start(): join() would raise RuntimeError on
|
| 1210 |
+
# an unstarted thread. Once started, the thread exits promptly since
|
| 1211 |
+
# the stop event is already set, so it needs no join here.
|
| 1212 |
+
if isinstance(play_thread, threading.Thread) and play_thread.is_alive():
|
| 1213 |
+
play_thread.join(timeout=1.0)
|
| 1214 |
+
|
| 1215 |
+
# Orphan-leak watchdog. A disconnect that lands before the connect
|
| 1216 |
+
# handler registers its session (sessions.pop -> None above) can't stop
|
| 1217 |
+
# the per-client IK/playback threads it would otherwise own, leaking an
|
| 1218 |
+
# idle daemon thread. We chose not to close that narrow race, so instead
|
| 1219 |
+
# surface accumulation: count live "action-viz-*client-<id>" threads whose
|
| 1220 |
+
# client_id has no active session. A handful are normal transients from
|
| 1221 |
+
# clients mid-connect (the IK thread starts before registration); a
|
| 1222 |
+
# sustained count above the threshold means the race is being hit often
|
| 1223 |
+
# enough to revisit the two-phase-registration fix. Runs after the
|
| 1224 |
+
# joins above so this client's own threads are gone, not miscounted.
|
| 1225 |
+
orphan_warn_threshold = 8
|
| 1226 |
+
with sessions_lock:
|
| 1227 |
+
active_ids = {str(client_id) for client_id in sessions}
|
| 1228 |
+
orphan_threads = [
|
| 1229 |
+
thread.name
|
| 1230 |
+
for thread in threading.enumerate()
|
| 1231 |
+
if thread.is_alive()
|
| 1232 |
+
and thread.name.startswith("action-viz-")
|
| 1233 |
+
and thread.name.rsplit("client-", 1)[-1] not in active_ids
|
| 1234 |
+
]
|
| 1235 |
+
if len(orphan_threads) >= orphan_warn_threshold:
|
| 1236 |
+
log.warning(
|
| 1237 |
+
f"action-viz: {len(orphan_threads)} orphaned client threads alive with "
|
| 1238 |
+
f"no active session (e.g. {orphan_threads[:4]}) across {len(active_ids)} "
|
| 1239 |
+
f"active sessions; connect/disconnect race thread leak may be accumulating."
|
| 1240 |
+
)
|
| 1241 |
|
| 1242 |
log.info(f"Action viz ready at http://0.0.0.0:{config.port}")
|
| 1243 |
if config.share:
|