Upload folder using huggingface_hub
Browse files
wandb/debug-internal.log
CHANGED
|
@@ -2278,3 +2278,253 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
|
|
| 2278 |
2024-07-03 16:44:08,755 INFO Thread-12 :120 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240703_150631-xd2jm096/files/wandb-summary.json
|
| 2279 |
2024-07-03 16:44:09,246 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2280 |
2024-07-03 16:44:14,253 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2278 |
2024-07-03 16:44:08,755 INFO Thread-12 :120 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240703_150631-xd2jm096/files/wandb-summary.json
|
| 2279 |
2024-07-03 16:44:09,246 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2280 |
2024-07-03 16:44:14,253 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2281 |
+
2024-07-03 16:44:14,757 INFO Thread-12 :120 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240703_150631-xd2jm096/files/config.yaml
|
| 2282 |
+
2024-07-03 16:44:16,316 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2283 |
+
2024-07-03 16:44:16,316 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2284 |
+
2024-07-03 16:44:17,129 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: pause
|
| 2285 |
+
2024-07-03 16:44:17,129 INFO HandlerThread:120 [handler.py:handle_request_pause():724] stopping system metrics thread
|
| 2286 |
+
2024-07-03 16:44:17,129 INFO HandlerThread:120 [system_monitor.py:finish():203] Stopping system monitor
|
| 2287 |
+
2024-07-03 16:44:17,130 INFO HandlerThread:120 [interfaces.py:finish():200] Joined cpu monitor
|
| 2288 |
+
2024-07-03 16:44:17,130 INFO HandlerThread:120 [interfaces.py:finish():200] Joined disk monitor
|
| 2289 |
+
2024-07-03 16:44:17,130 DEBUG SystemMonitor:120 [system_monitor.py:_start():172] Starting system metrics aggregation loop
|
| 2290 |
+
2024-07-03 16:44:17,130 DEBUG SystemMonitor:120 [system_monitor.py:_start():179] Finished system metrics aggregation loop
|
| 2291 |
+
2024-07-03 16:44:17,130 DEBUG SystemMonitor:120 [system_monitor.py:_start():183] Publishing last batch of metrics
|
| 2292 |
+
2024-07-03 16:44:17,135 INFO HandlerThread:120 [interfaces.py:finish():200] Joined gpu monitor
|
| 2293 |
+
2024-07-03 16:44:17,136 INFO HandlerThread:120 [interfaces.py:finish():200] Joined memory monitor
|
| 2294 |
+
2024-07-03 16:44:17,136 INFO HandlerThread:120 [interfaces.py:finish():200] Joined network monitor
|
| 2295 |
+
2024-07-03 16:44:17,136 DEBUG SenderThread:120 [sender.py:send():378] send: stats
|
| 2296 |
+
2024-07-03 16:44:20,137 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2297 |
+
2024-07-03 16:44:25,138 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2298 |
+
2024-07-03 16:44:30,139 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2299 |
+
2024-07-03 16:44:31,316 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2300 |
+
2024-07-03 16:44:31,316 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2301 |
+
2024-07-03 16:44:35,550 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2302 |
+
2024-07-03 16:44:40,551 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2303 |
+
2024-07-03 16:44:45,552 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2304 |
+
2024-07-03 16:44:46,316 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2305 |
+
2024-07-03 16:44:46,317 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2306 |
+
2024-07-03 16:44:51,474 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2307 |
+
2024-07-03 16:44:56,475 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2308 |
+
2024-07-03 16:45:01,316 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2309 |
+
2024-07-03 16:45:01,317 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2310 |
+
2024-07-03 16:45:01,506 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2311 |
+
2024-07-03 16:45:06,507 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2312 |
+
2024-07-03 16:45:11,508 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2313 |
+
2024-07-03 16:45:16,317 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2314 |
+
2024-07-03 16:45:16,317 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2315 |
+
2024-07-03 16:45:17,496 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2316 |
+
2024-07-03 16:45:22,498 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2317 |
+
2024-07-03 16:45:27,499 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2318 |
+
2024-07-03 16:45:31,317 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2319 |
+
2024-07-03 16:45:31,317 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2320 |
+
2024-07-03 16:45:33,470 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2321 |
+
2024-07-03 16:45:38,472 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2322 |
+
2024-07-03 16:45:43,473 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2323 |
+
2024-07-03 16:45:46,317 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2324 |
+
2024-07-03 16:45:46,317 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2325 |
+
2024-07-03 16:45:48,531 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2326 |
+
2024-07-03 16:45:53,532 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2327 |
+
2024-07-03 16:45:58,533 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2328 |
+
2024-07-03 16:46:01,317 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2329 |
+
2024-07-03 16:46:01,318 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2330 |
+
2024-07-03 16:46:04,486 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2331 |
+
2024-07-03 16:46:09,487 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2332 |
+
2024-07-03 16:46:14,488 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2333 |
+
2024-07-03 16:46:16,317 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2334 |
+
2024-07-03 16:46:16,318 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2335 |
+
2024-07-03 16:46:19,536 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2336 |
+
2024-07-03 16:46:24,537 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2337 |
+
2024-07-03 16:46:29,538 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2338 |
+
2024-07-03 16:46:31,318 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2339 |
+
2024-07-03 16:46:31,318 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2340 |
+
2024-07-03 16:46:35,461 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2341 |
+
2024-07-03 16:46:40,462 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2342 |
+
2024-07-03 16:46:45,463 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2343 |
+
2024-07-03 16:46:46,318 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2344 |
+
2024-07-03 16:46:46,318 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2345 |
+
2024-07-03 16:46:50,476 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2346 |
+
2024-07-03 16:46:55,477 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2347 |
+
2024-07-03 16:47:00,478 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2348 |
+
2024-07-03 16:47:01,318 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2349 |
+
2024-07-03 16:47:01,318 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2350 |
+
2024-07-03 16:47:06,460 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2351 |
+
2024-07-03 16:47:11,461 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2352 |
+
2024-07-03 16:47:16,318 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2353 |
+
2024-07-03 16:47:16,319 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2354 |
+
2024-07-03 16:47:17,458 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2355 |
+
2024-07-03 16:47:22,459 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2356 |
+
2024-07-03 16:47:27,460 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2357 |
+
2024-07-03 16:47:31,318 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2358 |
+
2024-07-03 16:47:31,319 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2359 |
+
2024-07-03 16:47:33,459 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2360 |
+
2024-07-03 16:47:38,460 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2361 |
+
2024-07-03 16:47:43,461 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2362 |
+
2024-07-03 16:47:46,318 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2363 |
+
2024-07-03 16:47:46,319 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2364 |
+
2024-07-03 16:47:48,503 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2365 |
+
2024-07-03 16:47:53,504 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2366 |
+
2024-07-03 16:47:58,505 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2367 |
+
2024-07-03 16:48:01,319 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2368 |
+
2024-07-03 16:48:01,319 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2369 |
+
2024-07-03 16:48:03,532 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2370 |
+
2024-07-03 16:48:08,533 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2371 |
+
2024-07-03 16:48:13,534 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2372 |
+
2024-07-03 16:48:16,319 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2373 |
+
2024-07-03 16:48:16,319 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2374 |
+
2024-07-03 16:48:19,501 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2375 |
+
2024-07-03 16:48:24,502 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2376 |
+
2024-07-03 16:48:29,503 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2377 |
+
2024-07-03 16:48:31,319 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2378 |
+
2024-07-03 16:48:31,320 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2379 |
+
2024-07-03 16:48:34,531 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2380 |
+
2024-07-03 16:48:39,532 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2381 |
+
2024-07-03 16:48:44,533 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2382 |
+
2024-07-03 16:48:46,319 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2383 |
+
2024-07-03 16:48:46,320 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2384 |
+
2024-07-03 16:48:50,494 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2385 |
+
2024-07-03 16:48:55,495 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2386 |
+
2024-07-03 16:49:00,497 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2387 |
+
2024-07-03 16:49:01,319 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2388 |
+
2024-07-03 16:49:01,320 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2389 |
+
2024-07-03 16:49:05,515 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2390 |
+
2024-07-03 16:49:10,516 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2391 |
+
2024-07-03 16:49:15,517 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2392 |
+
2024-07-03 16:49:16,320 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2393 |
+
2024-07-03 16:49:16,320 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2394 |
+
2024-07-03 16:49:20,527 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2395 |
+
2024-07-03 16:49:25,528 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2396 |
+
2024-07-03 16:49:30,529 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2397 |
+
2024-07-03 16:49:31,320 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2398 |
+
2024-07-03 16:49:31,320 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2399 |
+
2024-07-03 16:49:35,564 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2400 |
+
2024-07-03 16:49:40,565 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2401 |
+
2024-07-03 16:49:45,566 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2402 |
+
2024-07-03 16:49:46,320 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2403 |
+
2024-07-03 16:49:46,320 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2404 |
+
2024-07-03 16:49:51,500 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2405 |
+
2024-07-03 16:49:56,501 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2406 |
+
2024-07-03 16:50:01,320 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2407 |
+
2024-07-03 16:50:01,321 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2408 |
+
2024-07-03 16:50:01,504 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2409 |
+
2024-07-03 16:50:06,505 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2410 |
+
2024-07-03 16:50:11,506 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2411 |
+
2024-07-03 16:50:16,320 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2412 |
+
2024-07-03 16:50:16,321 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2413 |
+
2024-07-03 16:50:16,521 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2414 |
+
2024-07-03 16:50:21,522 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2415 |
+
2024-07-03 16:50:26,524 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2416 |
+
2024-07-03 16:50:31,320 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2417 |
+
2024-07-03 16:50:31,321 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2418 |
+
2024-07-03 16:50:32,470 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2419 |
+
2024-07-03 16:50:37,471 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2420 |
+
2024-07-03 16:50:42,472 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2421 |
+
2024-07-03 16:50:46,321 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2422 |
+
2024-07-03 16:50:46,321 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2423 |
+
2024-07-03 16:50:47,533 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2424 |
+
2024-07-03 16:50:52,534 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2425 |
+
2024-07-03 16:50:57,536 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2426 |
+
2024-07-03 16:51:01,321 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2427 |
+
2024-07-03 16:51:01,321 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2428 |
+
2024-07-03 16:51:03,519 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2429 |
+
2024-07-03 16:51:08,521 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2430 |
+
2024-07-03 16:51:13,522 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2431 |
+
2024-07-03 16:51:16,321 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2432 |
+
2024-07-03 16:51:16,321 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2433 |
+
2024-07-03 16:51:19,463 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2434 |
+
2024-07-03 16:51:24,464 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2435 |
+
2024-07-03 16:51:29,465 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2436 |
+
2024-07-03 16:51:31,321 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2437 |
+
2024-07-03 16:51:31,322 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2438 |
+
2024-07-03 16:51:34,497 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2439 |
+
2024-07-03 16:51:39,498 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2440 |
+
2024-07-03 16:51:44,499 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2441 |
+
2024-07-03 16:51:46,322 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2442 |
+
2024-07-03 16:51:46,322 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2443 |
+
2024-07-03 16:51:49,514 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2444 |
+
2024-07-03 16:51:54,515 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2445 |
+
2024-07-03 16:51:59,516 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2446 |
+
2024-07-03 16:52:01,322 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2447 |
+
2024-07-03 16:52:01,322 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2448 |
+
2024-07-03 16:52:04,572 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2449 |
+
2024-07-03 16:52:09,573 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2450 |
+
2024-07-03 16:52:14,574 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2451 |
+
2024-07-03 16:52:16,322 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2452 |
+
2024-07-03 16:52:16,322 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2453 |
+
2024-07-03 16:52:20,512 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2454 |
+
2024-07-03 16:52:25,513 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2455 |
+
2024-07-03 16:52:30,515 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2456 |
+
2024-07-03 16:52:31,322 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2457 |
+
2024-07-03 16:52:31,323 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2458 |
+
2024-07-03 16:52:36,464 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2459 |
+
2024-07-03 16:52:41,465 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2460 |
+
2024-07-03 16:52:46,322 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2461 |
+
2024-07-03 16:52:46,323 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2462 |
+
2024-07-03 16:52:46,504 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2463 |
+
2024-07-03 16:52:51,505 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2464 |
+
2024-07-03 16:52:56,506 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2465 |
+
2024-07-03 16:53:01,322 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2466 |
+
2024-07-03 16:53:01,323 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2467 |
+
2024-07-03 16:53:02,493 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2468 |
+
2024-07-03 16:53:07,494 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2469 |
+
2024-07-03 16:53:12,495 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2470 |
+
2024-07-03 16:53:16,323 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2471 |
+
2024-07-03 16:53:16,323 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2472 |
+
2024-07-03 16:53:17,539 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2473 |
+
2024-07-03 16:53:22,541 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2474 |
+
2024-07-03 16:53:27,542 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2475 |
+
2024-07-03 16:53:31,323 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2476 |
+
2024-07-03 16:53:31,323 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2477 |
+
2024-07-03 16:53:33,500 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2478 |
+
2024-07-03 16:53:38,501 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2479 |
+
2024-07-03 16:53:43,502 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2480 |
+
2024-07-03 16:53:46,323 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2481 |
+
2024-07-03 16:53:46,323 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2482 |
+
2024-07-03 16:53:49,486 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2483 |
+
2024-07-03 16:53:54,487 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2484 |
+
2024-07-03 16:53:59,488 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2485 |
+
2024-07-03 16:54:01,323 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2486 |
+
2024-07-03 16:54:01,324 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2487 |
+
2024-07-03 16:54:04,546 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2488 |
+
2024-07-03 16:54:09,547 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2489 |
+
2024-07-03 16:54:14,548 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2490 |
+
2024-07-03 16:54:16,324 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2491 |
+
2024-07-03 16:54:16,324 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2492 |
+
2024-07-03 16:54:20,502 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2493 |
+
2024-07-03 16:54:25,503 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2494 |
+
2024-07-03 16:54:30,504 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2495 |
+
2024-07-03 16:54:31,324 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2496 |
+
2024-07-03 16:54:31,324 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2497 |
+
2024-07-03 16:54:36,474 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2498 |
+
2024-07-03 16:54:41,475 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2499 |
+
2024-07-03 16:54:46,324 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2500 |
+
2024-07-03 16:54:46,324 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2501 |
+
2024-07-03 16:54:46,571 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2502 |
+
2024-07-03 16:54:51,571 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2503 |
+
2024-07-03 16:54:56,572 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2504 |
+
2024-07-03 16:55:01,324 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2505 |
+
2024-07-03 16:55:01,324 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2506 |
+
2024-07-03 16:55:02,566 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2507 |
+
2024-07-03 16:55:07,567 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2508 |
+
2024-07-03 16:55:12,568 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2509 |
+
2024-07-03 16:55:16,324 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2510 |
+
2024-07-03 16:55:16,325 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2511 |
+
2024-07-03 16:55:18,501 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2512 |
+
2024-07-03 16:55:23,502 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2513 |
+
2024-07-03 16:55:28,504 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2514 |
+
2024-07-03 16:55:31,324 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2515 |
+
2024-07-03 16:55:31,325 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2516 |
+
2024-07-03 16:55:33,532 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2517 |
+
2024-07-03 16:55:38,533 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2518 |
+
2024-07-03 16:55:43,534 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2519 |
+
2024-07-03 16:55:46,325 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2520 |
+
2024-07-03 16:55:46,325 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2521 |
+
2024-07-03 16:55:49,533 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2522 |
+
2024-07-03 16:55:50,141 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: resume
|
| 2523 |
+
2024-07-03 16:55:50,142 INFO HandlerThread:120 [handler.py:handle_request_resume():715] starting system metrics thread
|
| 2524 |
+
2024-07-03 16:55:50,142 INFO HandlerThread:120 [system_monitor.py:start():194] Starting system monitor
|
| 2525 |
+
2024-07-03 16:55:50,142 INFO SystemMonitor:120 [system_monitor.py:_start():158] Starting system asset monitoring threads
|
| 2526 |
+
2024-07-03 16:55:50,143 INFO SystemMonitor:120 [interfaces.py:start():188] Started cpu monitoring
|
| 2527 |
+
2024-07-03 16:55:50,143 INFO SystemMonitor:120 [interfaces.py:start():188] Started disk monitoring
|
| 2528 |
+
2024-07-03 16:55:50,145 INFO SystemMonitor:120 [interfaces.py:start():188] Started gpu monitoring
|
| 2529 |
+
2024-07-03 16:55:50,146 INFO SystemMonitor:120 [interfaces.py:start():188] Started memory monitoring
|
| 2530 |
+
2024-07-03 16:55:50,146 INFO SystemMonitor:120 [interfaces.py:start():188] Started network monitoring
|
wandb/debug.log
CHANGED
|
@@ -33,3 +33,6 @@ config: {}
|
|
| 33 |
2024-07-03 16:44:08,233 INFO MainThread:34 [jupyter.py:save_ipynb():373] not saving jupyter notebook
|
| 34 |
2024-07-03 16:44:08,234 INFO MainThread:34 [wandb_init.py:_pause_backend():431] pausing backend
|
| 35 |
2024-07-03 16:44:08,243 INFO MainThread:34 [wandb_init.py:_resume_backend():436] resuming backend
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
2024-07-03 16:44:08,233 INFO MainThread:34 [jupyter.py:save_ipynb():373] not saving jupyter notebook
|
| 34 |
2024-07-03 16:44:08,234 INFO MainThread:34 [wandb_init.py:_pause_backend():431] pausing backend
|
| 35 |
2024-07-03 16:44:08,243 INFO MainThread:34 [wandb_init.py:_resume_backend():436] resuming backend
|
| 36 |
+
2024-07-03 16:44:17,128 INFO MainThread:34 [jupyter.py:save_ipynb():373] not saving jupyter notebook
|
| 37 |
+
2024-07-03 16:44:17,128 INFO MainThread:34 [wandb_init.py:_pause_backend():431] pausing backend
|
| 38 |
+
2024-07-03 16:55:50,140 INFO MainThread:34 [wandb_init.py:_resume_backend():436] resuming backend
|
wandb/run-20240703_150631-xd2jm096/logs/debug-internal.log
CHANGED
|
@@ -2278,3 +2278,253 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
|
|
| 2278 |
2024-07-03 16:44:08,755 INFO Thread-12 :120 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240703_150631-xd2jm096/files/wandb-summary.json
|
| 2279 |
2024-07-03 16:44:09,246 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2280 |
2024-07-03 16:44:14,253 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2278 |
2024-07-03 16:44:08,755 INFO Thread-12 :120 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240703_150631-xd2jm096/files/wandb-summary.json
|
| 2279 |
2024-07-03 16:44:09,246 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2280 |
2024-07-03 16:44:14,253 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2281 |
+
2024-07-03 16:44:14,757 INFO Thread-12 :120 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240703_150631-xd2jm096/files/config.yaml
|
| 2282 |
+
2024-07-03 16:44:16,316 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2283 |
+
2024-07-03 16:44:16,316 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2284 |
+
2024-07-03 16:44:17,129 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: pause
|
| 2285 |
+
2024-07-03 16:44:17,129 INFO HandlerThread:120 [handler.py:handle_request_pause():724] stopping system metrics thread
|
| 2286 |
+
2024-07-03 16:44:17,129 INFO HandlerThread:120 [system_monitor.py:finish():203] Stopping system monitor
|
| 2287 |
+
2024-07-03 16:44:17,130 INFO HandlerThread:120 [interfaces.py:finish():200] Joined cpu monitor
|
| 2288 |
+
2024-07-03 16:44:17,130 INFO HandlerThread:120 [interfaces.py:finish():200] Joined disk monitor
|
| 2289 |
+
2024-07-03 16:44:17,130 DEBUG SystemMonitor:120 [system_monitor.py:_start():172] Starting system metrics aggregation loop
|
| 2290 |
+
2024-07-03 16:44:17,130 DEBUG SystemMonitor:120 [system_monitor.py:_start():179] Finished system metrics aggregation loop
|
| 2291 |
+
2024-07-03 16:44:17,130 DEBUG SystemMonitor:120 [system_monitor.py:_start():183] Publishing last batch of metrics
|
| 2292 |
+
2024-07-03 16:44:17,135 INFO HandlerThread:120 [interfaces.py:finish():200] Joined gpu monitor
|
| 2293 |
+
2024-07-03 16:44:17,136 INFO HandlerThread:120 [interfaces.py:finish():200] Joined memory monitor
|
| 2294 |
+
2024-07-03 16:44:17,136 INFO HandlerThread:120 [interfaces.py:finish():200] Joined network monitor
|
| 2295 |
+
2024-07-03 16:44:17,136 DEBUG SenderThread:120 [sender.py:send():378] send: stats
|
| 2296 |
+
2024-07-03 16:44:20,137 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2297 |
+
2024-07-03 16:44:25,138 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2298 |
+
2024-07-03 16:44:30,139 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2299 |
+
2024-07-03 16:44:31,316 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2300 |
+
2024-07-03 16:44:31,316 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2301 |
+
2024-07-03 16:44:35,550 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2302 |
+
2024-07-03 16:44:40,551 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2303 |
+
2024-07-03 16:44:45,552 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2304 |
+
2024-07-03 16:44:46,316 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2305 |
+
2024-07-03 16:44:46,317 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2306 |
+
2024-07-03 16:44:51,474 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2307 |
+
2024-07-03 16:44:56,475 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2308 |
+
2024-07-03 16:45:01,316 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2309 |
+
2024-07-03 16:45:01,317 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2310 |
+
2024-07-03 16:45:01,506 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2311 |
+
2024-07-03 16:45:06,507 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2312 |
+
2024-07-03 16:45:11,508 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2313 |
+
2024-07-03 16:45:16,317 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2314 |
+
2024-07-03 16:45:16,317 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2315 |
+
2024-07-03 16:45:17,496 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2316 |
+
2024-07-03 16:45:22,498 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2317 |
+
2024-07-03 16:45:27,499 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2318 |
+
2024-07-03 16:45:31,317 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2319 |
+
2024-07-03 16:45:31,317 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2320 |
+
2024-07-03 16:45:33,470 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2321 |
+
2024-07-03 16:45:38,472 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2322 |
+
2024-07-03 16:45:43,473 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2323 |
+
2024-07-03 16:45:46,317 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2324 |
+
2024-07-03 16:45:46,317 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2325 |
+
2024-07-03 16:45:48,531 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2326 |
+
2024-07-03 16:45:53,532 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2327 |
+
2024-07-03 16:45:58,533 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2328 |
+
2024-07-03 16:46:01,317 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2329 |
+
2024-07-03 16:46:01,318 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2330 |
+
2024-07-03 16:46:04,486 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2331 |
+
2024-07-03 16:46:09,487 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2332 |
+
2024-07-03 16:46:14,488 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2333 |
+
2024-07-03 16:46:16,317 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2334 |
+
2024-07-03 16:46:16,318 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2335 |
+
2024-07-03 16:46:19,536 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2336 |
+
2024-07-03 16:46:24,537 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2337 |
+
2024-07-03 16:46:29,538 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2338 |
+
2024-07-03 16:46:31,318 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2339 |
+
2024-07-03 16:46:31,318 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2340 |
+
2024-07-03 16:46:35,461 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2341 |
+
2024-07-03 16:46:40,462 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2342 |
+
2024-07-03 16:46:45,463 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2343 |
+
2024-07-03 16:46:46,318 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2344 |
+
2024-07-03 16:46:46,318 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2345 |
+
2024-07-03 16:46:50,476 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2346 |
+
2024-07-03 16:46:55,477 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2347 |
+
2024-07-03 16:47:00,478 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2348 |
+
2024-07-03 16:47:01,318 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2349 |
+
2024-07-03 16:47:01,318 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2350 |
+
2024-07-03 16:47:06,460 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2351 |
+
2024-07-03 16:47:11,461 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2352 |
+
2024-07-03 16:47:16,318 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2353 |
+
2024-07-03 16:47:16,319 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2354 |
+
2024-07-03 16:47:17,458 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2355 |
+
2024-07-03 16:47:22,459 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2356 |
+
2024-07-03 16:47:27,460 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2357 |
+
2024-07-03 16:47:31,318 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2358 |
+
2024-07-03 16:47:31,319 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2359 |
+
2024-07-03 16:47:33,459 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2360 |
+
2024-07-03 16:47:38,460 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2361 |
+
2024-07-03 16:47:43,461 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2362 |
+
2024-07-03 16:47:46,318 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2363 |
+
2024-07-03 16:47:46,319 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2364 |
+
2024-07-03 16:47:48,503 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2365 |
+
2024-07-03 16:47:53,504 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2366 |
+
2024-07-03 16:47:58,505 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2367 |
+
2024-07-03 16:48:01,319 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2368 |
+
2024-07-03 16:48:01,319 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2369 |
+
2024-07-03 16:48:03,532 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2370 |
+
2024-07-03 16:48:08,533 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2371 |
+
2024-07-03 16:48:13,534 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2372 |
+
2024-07-03 16:48:16,319 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2373 |
+
2024-07-03 16:48:16,319 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2374 |
+
2024-07-03 16:48:19,501 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2375 |
+
2024-07-03 16:48:24,502 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2376 |
+
2024-07-03 16:48:29,503 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2377 |
+
2024-07-03 16:48:31,319 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2378 |
+
2024-07-03 16:48:31,320 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2379 |
+
2024-07-03 16:48:34,531 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2380 |
+
2024-07-03 16:48:39,532 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2381 |
+
2024-07-03 16:48:44,533 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2382 |
+
2024-07-03 16:48:46,319 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2383 |
+
2024-07-03 16:48:46,320 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2384 |
+
2024-07-03 16:48:50,494 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2385 |
+
2024-07-03 16:48:55,495 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2386 |
+
2024-07-03 16:49:00,497 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2387 |
+
2024-07-03 16:49:01,319 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2388 |
+
2024-07-03 16:49:01,320 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2389 |
+
2024-07-03 16:49:05,515 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2390 |
+
2024-07-03 16:49:10,516 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2391 |
+
2024-07-03 16:49:15,517 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2392 |
+
2024-07-03 16:49:16,320 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2393 |
+
2024-07-03 16:49:16,320 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2394 |
+
2024-07-03 16:49:20,527 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2395 |
+
2024-07-03 16:49:25,528 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2396 |
+
2024-07-03 16:49:30,529 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2397 |
+
2024-07-03 16:49:31,320 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2398 |
+
2024-07-03 16:49:31,320 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2399 |
+
2024-07-03 16:49:35,564 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2400 |
+
2024-07-03 16:49:40,565 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2401 |
+
2024-07-03 16:49:45,566 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2402 |
+
2024-07-03 16:49:46,320 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2403 |
+
2024-07-03 16:49:46,320 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2404 |
+
2024-07-03 16:49:51,500 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2405 |
+
2024-07-03 16:49:56,501 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2406 |
+
2024-07-03 16:50:01,320 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2407 |
+
2024-07-03 16:50:01,321 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2408 |
+
2024-07-03 16:50:01,504 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2409 |
+
2024-07-03 16:50:06,505 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2410 |
+
2024-07-03 16:50:11,506 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2411 |
+
2024-07-03 16:50:16,320 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2412 |
+
2024-07-03 16:50:16,321 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2413 |
+
2024-07-03 16:50:16,521 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2414 |
+
2024-07-03 16:50:21,522 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2415 |
+
2024-07-03 16:50:26,524 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2416 |
+
2024-07-03 16:50:31,320 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2417 |
+
2024-07-03 16:50:31,321 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2418 |
+
2024-07-03 16:50:32,470 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2419 |
+
2024-07-03 16:50:37,471 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2420 |
+
2024-07-03 16:50:42,472 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2421 |
+
2024-07-03 16:50:46,321 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2422 |
+
2024-07-03 16:50:46,321 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2423 |
+
2024-07-03 16:50:47,533 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2424 |
+
2024-07-03 16:50:52,534 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2425 |
+
2024-07-03 16:50:57,536 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2426 |
+
2024-07-03 16:51:01,321 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2427 |
+
2024-07-03 16:51:01,321 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2428 |
+
2024-07-03 16:51:03,519 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2429 |
+
2024-07-03 16:51:08,521 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2430 |
+
2024-07-03 16:51:13,522 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2431 |
+
2024-07-03 16:51:16,321 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2432 |
+
2024-07-03 16:51:16,321 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2433 |
+
2024-07-03 16:51:19,463 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2434 |
+
2024-07-03 16:51:24,464 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2435 |
+
2024-07-03 16:51:29,465 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2436 |
+
2024-07-03 16:51:31,321 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2437 |
+
2024-07-03 16:51:31,322 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2438 |
+
2024-07-03 16:51:34,497 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2439 |
+
2024-07-03 16:51:39,498 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2440 |
+
2024-07-03 16:51:44,499 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2441 |
+
2024-07-03 16:51:46,322 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2442 |
+
2024-07-03 16:51:46,322 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2443 |
+
2024-07-03 16:51:49,514 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2444 |
+
2024-07-03 16:51:54,515 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2445 |
+
2024-07-03 16:51:59,516 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2446 |
+
2024-07-03 16:52:01,322 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2447 |
+
2024-07-03 16:52:01,322 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2448 |
+
2024-07-03 16:52:04,572 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2449 |
+
2024-07-03 16:52:09,573 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2450 |
+
2024-07-03 16:52:14,574 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2451 |
+
2024-07-03 16:52:16,322 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2452 |
+
2024-07-03 16:52:16,322 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2453 |
+
2024-07-03 16:52:20,512 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2454 |
+
2024-07-03 16:52:25,513 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2455 |
+
2024-07-03 16:52:30,515 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2456 |
+
2024-07-03 16:52:31,322 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2457 |
+
2024-07-03 16:52:31,323 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2458 |
+
2024-07-03 16:52:36,464 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2459 |
+
2024-07-03 16:52:41,465 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2460 |
+
2024-07-03 16:52:46,322 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2461 |
+
2024-07-03 16:52:46,323 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2462 |
+
2024-07-03 16:52:46,504 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2463 |
+
2024-07-03 16:52:51,505 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2464 |
+
2024-07-03 16:52:56,506 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2465 |
+
2024-07-03 16:53:01,322 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2466 |
+
2024-07-03 16:53:01,323 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2467 |
+
2024-07-03 16:53:02,493 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2468 |
+
2024-07-03 16:53:07,494 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2469 |
+
2024-07-03 16:53:12,495 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2470 |
+
2024-07-03 16:53:16,323 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2471 |
+
2024-07-03 16:53:16,323 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2472 |
+
2024-07-03 16:53:17,539 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2473 |
+
2024-07-03 16:53:22,541 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2474 |
+
2024-07-03 16:53:27,542 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2475 |
+
2024-07-03 16:53:31,323 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2476 |
+
2024-07-03 16:53:31,323 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2477 |
+
2024-07-03 16:53:33,500 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2478 |
+
2024-07-03 16:53:38,501 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2479 |
+
2024-07-03 16:53:43,502 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2480 |
+
2024-07-03 16:53:46,323 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2481 |
+
2024-07-03 16:53:46,323 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2482 |
+
2024-07-03 16:53:49,486 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2483 |
+
2024-07-03 16:53:54,487 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2484 |
+
2024-07-03 16:53:59,488 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2485 |
+
2024-07-03 16:54:01,323 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2486 |
+
2024-07-03 16:54:01,324 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2487 |
+
2024-07-03 16:54:04,546 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2488 |
+
2024-07-03 16:54:09,547 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2489 |
+
2024-07-03 16:54:14,548 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2490 |
+
2024-07-03 16:54:16,324 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2491 |
+
2024-07-03 16:54:16,324 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2492 |
+
2024-07-03 16:54:20,502 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2493 |
+
2024-07-03 16:54:25,503 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2494 |
+
2024-07-03 16:54:30,504 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2495 |
+
2024-07-03 16:54:31,324 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2496 |
+
2024-07-03 16:54:31,324 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2497 |
+
2024-07-03 16:54:36,474 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2498 |
+
2024-07-03 16:54:41,475 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2499 |
+
2024-07-03 16:54:46,324 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2500 |
+
2024-07-03 16:54:46,324 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2501 |
+
2024-07-03 16:54:46,571 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2502 |
+
2024-07-03 16:54:51,571 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2503 |
+
2024-07-03 16:54:56,572 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2504 |
+
2024-07-03 16:55:01,324 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2505 |
+
2024-07-03 16:55:01,324 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2506 |
+
2024-07-03 16:55:02,566 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2507 |
+
2024-07-03 16:55:07,567 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2508 |
+
2024-07-03 16:55:12,568 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2509 |
+
2024-07-03 16:55:16,324 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2510 |
+
2024-07-03 16:55:16,325 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2511 |
+
2024-07-03 16:55:18,501 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2512 |
+
2024-07-03 16:55:23,502 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2513 |
+
2024-07-03 16:55:28,504 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2514 |
+
2024-07-03 16:55:31,324 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2515 |
+
2024-07-03 16:55:31,325 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2516 |
+
2024-07-03 16:55:33,532 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2517 |
+
2024-07-03 16:55:38,533 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2518 |
+
2024-07-03 16:55:43,534 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2519 |
+
2024-07-03 16:55:46,325 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: stop_status
|
| 2520 |
+
2024-07-03 16:55:46,325 DEBUG SenderThread:120 [sender.py:send_request():405] send_request: stop_status
|
| 2521 |
+
2024-07-03 16:55:49,533 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: status_report
|
| 2522 |
+
2024-07-03 16:55:50,141 DEBUG HandlerThread:120 [handler.py:handle_request():158] handle_request: resume
|
| 2523 |
+
2024-07-03 16:55:50,142 INFO HandlerThread:120 [handler.py:handle_request_resume():715] starting system metrics thread
|
| 2524 |
+
2024-07-03 16:55:50,142 INFO HandlerThread:120 [system_monitor.py:start():194] Starting system monitor
|
| 2525 |
+
2024-07-03 16:55:50,142 INFO SystemMonitor:120 [system_monitor.py:_start():158] Starting system asset monitoring threads
|
| 2526 |
+
2024-07-03 16:55:50,143 INFO SystemMonitor:120 [interfaces.py:start():188] Started cpu monitoring
|
| 2527 |
+
2024-07-03 16:55:50,143 INFO SystemMonitor:120 [interfaces.py:start():188] Started disk monitoring
|
| 2528 |
+
2024-07-03 16:55:50,145 INFO SystemMonitor:120 [interfaces.py:start():188] Started gpu monitoring
|
| 2529 |
+
2024-07-03 16:55:50,146 INFO SystemMonitor:120 [interfaces.py:start():188] Started memory monitoring
|
| 2530 |
+
2024-07-03 16:55:50,146 INFO SystemMonitor:120 [interfaces.py:start():188] Started network monitoring
|
wandb/run-20240703_150631-xd2jm096/logs/debug.log
CHANGED
|
@@ -33,3 +33,6 @@ config: {}
|
|
| 33 |
2024-07-03 16:44:08,233 INFO MainThread:34 [jupyter.py:save_ipynb():373] not saving jupyter notebook
|
| 34 |
2024-07-03 16:44:08,234 INFO MainThread:34 [wandb_init.py:_pause_backend():431] pausing backend
|
| 35 |
2024-07-03 16:44:08,243 INFO MainThread:34 [wandb_init.py:_resume_backend():436] resuming backend
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
2024-07-03 16:44:08,233 INFO MainThread:34 [jupyter.py:save_ipynb():373] not saving jupyter notebook
|
| 34 |
2024-07-03 16:44:08,234 INFO MainThread:34 [wandb_init.py:_pause_backend():431] pausing backend
|
| 35 |
2024-07-03 16:44:08,243 INFO MainThread:34 [wandb_init.py:_resume_backend():436] resuming backend
|
| 36 |
+
2024-07-03 16:44:17,128 INFO MainThread:34 [jupyter.py:save_ipynb():373] not saving jupyter notebook
|
| 37 |
+
2024-07-03 16:44:17,128 INFO MainThread:34 [wandb_init.py:_pause_backend():431] pausing backend
|
| 38 |
+
2024-07-03 16:55:50,140 INFO MainThread:34 [wandb_init.py:_resume_backend():436] resuming backend
|