Training in progress, step 10
Browse files- model.safetensors +1 -1
- training_args.bin +1 -1
- wandb/debug-internal.log +673 -0
- wandb/debug.log +22 -0
- wandb/run-20240720_011510-w16tb06h/files/config.yaml +15 -3
- wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json +1 -0
- wandb/run-20240720_011510-w16tb06h/logs/debug-internal.log +673 -0
- wandb/run-20240720_011510-w16tb06h/logs/debug.log +22 -0
- wandb/run-20240720_011510-w16tb06h/run-w16tb06h.wandb +0 -0
model.safetensors
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 4537938567
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4c71f2dd7db627aa3b3b3625b62d026cf2dc0fcf66248bb1611ff9bc3331ec09
|
| 3 |
size 4537938567
|
training_args.bin
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 5112
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:73f52b0a6c2a3d83dcdc1d07bdcf27ce58e4a6e6bd6b6c497bd91cdc22845a98
|
| 3 |
size 5112
|
wandb/debug-internal.log
CHANGED
|
@@ -454,3 +454,676 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
|
|
| 454 |
2024-07-20 01:18:46,934 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 455 |
2024-07-20 01:18:47,935 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 456 |
2024-07-20 01:18:48,935 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 454 |
2024-07-20 01:18:46,934 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 455 |
2024-07-20 01:18:47,935 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 456 |
2024-07-20 01:18:48,935 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 457 |
+
2024-07-20 01:18:49,935 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 458 |
+
2024-07-20 01:18:50,373 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 459 |
+
2024-07-20 01:18:50,374 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause
|
| 460 |
+
2024-07-20 01:18:50,374 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread
|
| 461 |
+
2024-07-20 01:18:50,374 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor
|
| 462 |
+
2024-07-20 01:18:50,375 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop
|
| 463 |
+
2024-07-20 01:18:50,375 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics
|
| 464 |
+
2024-07-20 01:18:50,376 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor
|
| 465 |
+
2024-07-20 01:18:50,377 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor
|
| 466 |
+
2024-07-20 01:18:50,384 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor
|
| 467 |
+
2024-07-20 01:18:50,384 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor
|
| 468 |
+
2024-07-20 01:18:50,384 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor
|
| 469 |
+
2024-07-20 01:18:50,385 DEBUG SenderThread:179 [sender.py:send():379] send: stats
|
| 470 |
+
2024-07-20 01:18:50,935 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 471 |
+
2024-07-20 01:18:51,935 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 472 |
+
2024-07-20 01:18:52,935 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 473 |
+
2024-07-20 01:18:53,936 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 474 |
+
2024-07-20 01:18:54,936 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 475 |
+
2024-07-20 01:18:55,386 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 476 |
+
2024-07-20 01:18:55,936 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 477 |
+
2024-07-20 01:18:56,936 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 478 |
+
2024-07-20 01:18:57,231 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 479 |
+
2024-07-20 01:18:57,231 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 480 |
+
2024-07-20 01:18:57,936 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 481 |
+
2024-07-20 01:18:58,936 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 482 |
+
2024-07-20 01:18:59,937 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 483 |
+
2024-07-20 01:19:00,937 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 484 |
+
2024-07-20 01:19:01,315 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 485 |
+
2024-07-20 01:19:01,937 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 486 |
+
2024-07-20 01:19:02,937 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 487 |
+
2024-07-20 01:19:03,937 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 488 |
+
2024-07-20 01:19:04,937 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 489 |
+
2024-07-20 01:19:05,938 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 490 |
+
2024-07-20 01:19:06,316 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 491 |
+
2024-07-20 01:19:06,938 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 492 |
+
2024-07-20 01:19:07,938 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 493 |
+
2024-07-20 01:19:08,938 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 494 |
+
2024-07-20 01:19:09,938 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 495 |
+
2024-07-20 01:19:10,938 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 496 |
+
2024-07-20 01:19:11,317 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 497 |
+
2024-07-20 01:19:11,938 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 498 |
+
2024-07-20 01:19:12,231 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 499 |
+
2024-07-20 01:19:12,231 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 500 |
+
2024-07-20 01:19:12,938 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 501 |
+
2024-07-20 01:19:13,939 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 502 |
+
2024-07-20 01:19:14,939 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 503 |
+
2024-07-20 01:19:15,939 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 504 |
+
2024-07-20 01:19:16,939 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 505 |
+
2024-07-20 01:19:17,309 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 506 |
+
2024-07-20 01:19:17,939 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 507 |
+
2024-07-20 01:19:18,939 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 508 |
+
2024-07-20 01:19:19,939 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 509 |
+
2024-07-20 01:19:20,940 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 510 |
+
2024-07-20 01:19:21,940 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 511 |
+
2024-07-20 01:19:22,310 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 512 |
+
2024-07-20 01:19:22,940 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 513 |
+
2024-07-20 01:19:23,940 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 514 |
+
2024-07-20 01:19:24,940 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 515 |
+
2024-07-20 01:19:25,940 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 516 |
+
2024-07-20 01:19:26,941 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 517 |
+
2024-07-20 01:19:27,231 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 518 |
+
2024-07-20 01:19:27,231 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 519 |
+
2024-07-20 01:19:27,310 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 520 |
+
2024-07-20 01:19:27,941 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 521 |
+
2024-07-20 01:19:28,941 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 522 |
+
2024-07-20 01:19:29,941 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 523 |
+
2024-07-20 01:19:30,941 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 524 |
+
2024-07-20 01:19:31,941 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 525 |
+
2024-07-20 01:19:32,311 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 526 |
+
2024-07-20 01:19:32,941 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 527 |
+
2024-07-20 01:19:33,942 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 528 |
+
2024-07-20 01:19:34,942 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 529 |
+
2024-07-20 01:19:35,942 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 530 |
+
2024-07-20 01:19:36,942 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 531 |
+
2024-07-20 01:19:37,312 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 532 |
+
2024-07-20 01:19:37,942 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 533 |
+
2024-07-20 01:19:38,942 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 534 |
+
2024-07-20 01:19:39,942 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 535 |
+
2024-07-20 01:19:40,943 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 536 |
+
2024-07-20 01:19:41,943 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 537 |
+
2024-07-20 01:19:42,231 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 538 |
+
2024-07-20 01:19:42,231 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 539 |
+
2024-07-20 01:19:42,943 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 540 |
+
2024-07-20 01:19:43,309 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 541 |
+
2024-07-20 01:19:43,943 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 542 |
+
2024-07-20 01:19:44,943 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 543 |
+
2024-07-20 01:19:45,943 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 544 |
+
2024-07-20 01:19:46,943 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 545 |
+
2024-07-20 01:19:47,944 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 546 |
+
2024-07-20 01:19:48,310 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 547 |
+
2024-07-20 01:19:48,944 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 548 |
+
2024-07-20 01:19:49,944 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 549 |
+
2024-07-20 01:19:50,944 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 550 |
+
2024-07-20 01:19:51,944 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 551 |
+
2024-07-20 01:19:52,945 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 552 |
+
2024-07-20 01:19:53,311 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 553 |
+
2024-07-20 01:19:53,945 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 554 |
+
2024-07-20 01:19:54,945 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 555 |
+
2024-07-20 01:19:55,945 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 556 |
+
2024-07-20 01:19:56,945 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 557 |
+
2024-07-20 01:19:57,231 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 558 |
+
2024-07-20 01:19:57,232 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 559 |
+
2024-07-20 01:19:57,945 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 560 |
+
2024-07-20 01:19:58,358 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 561 |
+
2024-07-20 01:19:58,946 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 562 |
+
2024-07-20 01:19:59,946 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 563 |
+
2024-07-20 01:20:00,946 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 564 |
+
2024-07-20 01:20:01,946 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 565 |
+
2024-07-20 01:20:02,946 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 566 |
+
2024-07-20 01:20:03,359 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 567 |
+
2024-07-20 01:20:03,946 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 568 |
+
2024-07-20 01:20:04,947 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 569 |
+
2024-07-20 01:20:05,947 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 570 |
+
2024-07-20 01:20:06,947 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 571 |
+
2024-07-20 01:20:07,947 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 572 |
+
2024-07-20 01:20:08,360 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 573 |
+
2024-07-20 01:20:08,947 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 574 |
+
2024-07-20 01:20:09,947 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 575 |
+
2024-07-20 01:20:10,948 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 576 |
+
2024-07-20 01:20:11,948 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 577 |
+
2024-07-20 01:20:12,231 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 578 |
+
2024-07-20 01:20:12,232 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 579 |
+
2024-07-20 01:20:12,948 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 580 |
+
2024-07-20 01:20:13,948 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 581 |
+
2024-07-20 01:20:14,353 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 582 |
+
2024-07-20 01:20:14,948 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 583 |
+
2024-07-20 01:20:15,948 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 584 |
+
2024-07-20 01:20:16,949 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 585 |
+
2024-07-20 01:20:17,949 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 586 |
+
2024-07-20 01:20:18,949 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 587 |
+
2024-07-20 01:20:19,354 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 588 |
+
2024-07-20 01:20:19,949 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 589 |
+
2024-07-20 01:20:20,949 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 590 |
+
2024-07-20 01:20:21,950 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 591 |
+
2024-07-20 01:20:22,950 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 592 |
+
2024-07-20 01:20:23,950 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 593 |
+
2024-07-20 01:20:24,355 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 594 |
+
2024-07-20 01:20:24,950 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 595 |
+
2024-07-20 01:20:25,950 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 596 |
+
2024-07-20 01:20:26,950 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 597 |
+
2024-07-20 01:20:27,231 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 598 |
+
2024-07-20 01:20:27,232 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 599 |
+
2024-07-20 01:20:27,950 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 600 |
+
2024-07-20 01:20:28,951 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 601 |
+
2024-07-20 01:20:29,951 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 602 |
+
2024-07-20 01:20:30,295 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 603 |
+
2024-07-20 01:20:30,951 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 604 |
+
2024-07-20 01:20:31,951 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 605 |
+
2024-07-20 01:20:32,951 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 606 |
+
2024-07-20 01:20:33,951 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 607 |
+
2024-07-20 01:20:34,951 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 608 |
+
2024-07-20 01:20:35,295 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 609 |
+
2024-07-20 01:20:35,952 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 610 |
+
2024-07-20 01:20:36,952 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 611 |
+
2024-07-20 01:20:37,952 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 612 |
+
2024-07-20 01:20:38,952 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 613 |
+
2024-07-20 01:20:39,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 614 |
+
2024-07-20 01:20:40,296 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 615 |
+
2024-07-20 01:20:40,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 616 |
+
2024-07-20 01:20:41,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 617 |
+
2024-07-20 01:20:42,231 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 618 |
+
2024-07-20 01:20:42,232 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 619 |
+
2024-07-20 01:20:42,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 620 |
+
2024-07-20 01:20:43,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 621 |
+
2024-07-20 01:20:44,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 622 |
+
2024-07-20 01:20:45,393 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 623 |
+
2024-07-20 01:20:45,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 624 |
+
2024-07-20 01:20:46,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 625 |
+
2024-07-20 01:20:47,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 626 |
+
2024-07-20 01:20:48,954 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 627 |
+
2024-07-20 01:20:49,954 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 628 |
+
2024-07-20 01:20:50,394 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 629 |
+
2024-07-20 01:20:50,954 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 630 |
+
2024-07-20 01:20:51,954 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 631 |
+
2024-07-20 01:20:52,954 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 632 |
+
2024-07-20 01:20:53,954 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 633 |
+
2024-07-20 01:20:54,954 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 634 |
+
2024-07-20 01:20:55,395 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 635 |
+
2024-07-20 01:20:55,954 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 636 |
+
2024-07-20 01:20:56,955 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 637 |
+
2024-07-20 01:20:57,232 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 638 |
+
2024-07-20 01:20:57,232 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 639 |
+
2024-07-20 01:20:57,955 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 640 |
+
2024-07-20 01:20:58,955 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 641 |
+
2024-07-20 01:20:59,955 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 642 |
+
2024-07-20 01:21:00,955 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 643 |
+
2024-07-20 01:21:01,341 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 644 |
+
2024-07-20 01:21:01,955 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 645 |
+
2024-07-20 01:21:02,955 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 646 |
+
2024-07-20 01:21:03,956 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 647 |
+
2024-07-20 01:21:04,956 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 648 |
+
2024-07-20 01:21:05,956 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 649 |
+
2024-07-20 01:21:06,342 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 650 |
+
2024-07-20 01:21:06,956 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 651 |
+
2024-07-20 01:21:07,956 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 652 |
+
2024-07-20 01:21:08,290 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume
|
| 653 |
+
2024-07-20 01:21:08,291 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread
|
| 654 |
+
2024-07-20 01:21:08,291 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor
|
| 655 |
+
2024-07-20 01:21:08,291 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads
|
| 656 |
+
2024-07-20 01:21:08,291 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring
|
| 657 |
+
2024-07-20 01:21:08,292 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring
|
| 658 |
+
2024-07-20 01:21:08,293 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring
|
| 659 |
+
2024-07-20 01:21:08,294 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring
|
| 660 |
+
2024-07-20 01:21:08,295 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring
|
| 661 |
+
2024-07-20 01:21:09,123 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 662 |
+
2024-07-20 01:21:09,256 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause
|
| 663 |
+
2024-07-20 01:21:09,257 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread
|
| 664 |
+
2024-07-20 01:21:09,257 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor
|
| 665 |
+
2024-07-20 01:21:09,257 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor
|
| 666 |
+
2024-07-20 01:21:09,257 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor
|
| 667 |
+
2024-07-20 01:21:09,258 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop
|
| 668 |
+
2024-07-20 01:21:09,258 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop
|
| 669 |
+
2024-07-20 01:21:09,258 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics
|
| 670 |
+
2024-07-20 01:21:09,265 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor
|
| 671 |
+
2024-07-20 01:21:09,266 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor
|
| 672 |
+
2024-07-20 01:21:09,266 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor
|
| 673 |
+
2024-07-20 01:21:09,266 DEBUG SenderThread:179 [sender.py:send():379] send: stats
|
| 674 |
+
2024-07-20 01:21:09,967 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 675 |
+
2024-07-20 01:21:10,967 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 676 |
+
2024-07-20 01:21:11,967 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 677 |
+
2024-07-20 01:21:12,232 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 678 |
+
2024-07-20 01:21:12,232 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 679 |
+
2024-07-20 01:21:12,360 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 680 |
+
2024-07-20 01:21:12,967 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 681 |
+
2024-07-20 01:21:13,967 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 682 |
+
2024-07-20 01:21:14,968 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 683 |
+
2024-07-20 01:21:15,968 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 684 |
+
2024-07-20 01:21:16,968 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 685 |
+
2024-07-20 01:21:17,361 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 686 |
+
2024-07-20 01:21:17,968 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 687 |
+
2024-07-20 01:21:18,968 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 688 |
+
2024-07-20 01:21:19,968 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 689 |
+
2024-07-20 01:21:20,968 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 690 |
+
2024-07-20 01:21:21,969 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 691 |
+
2024-07-20 01:21:22,362 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 692 |
+
2024-07-20 01:21:22,969 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 693 |
+
2024-07-20 01:21:23,969 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 694 |
+
2024-07-20 01:21:24,969 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 695 |
+
2024-07-20 01:21:25,969 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 696 |
+
2024-07-20 01:21:26,969 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 697 |
+
2024-07-20 01:21:27,232 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 698 |
+
2024-07-20 01:21:27,232 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 699 |
+
2024-07-20 01:21:27,404 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 700 |
+
2024-07-20 01:21:27,969 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 701 |
+
2024-07-20 01:21:28,969 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 702 |
+
2024-07-20 01:21:29,970 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 703 |
+
2024-07-20 01:21:30,970 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 704 |
+
2024-07-20 01:21:31,970 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 705 |
+
2024-07-20 01:21:32,405 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 706 |
+
2024-07-20 01:21:32,970 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 707 |
+
2024-07-20 01:21:33,970 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 708 |
+
2024-07-20 01:21:34,970 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 709 |
+
2024-07-20 01:21:35,970 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 710 |
+
2024-07-20 01:21:36,971 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 711 |
+
2024-07-20 01:21:37,406 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 712 |
+
2024-07-20 01:21:37,971 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 713 |
+
2024-07-20 01:21:38,971 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 714 |
+
2024-07-20 01:21:39,971 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 715 |
+
2024-07-20 01:21:40,971 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 716 |
+
2024-07-20 01:21:41,971 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 717 |
+
2024-07-20 01:21:42,232 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 718 |
+
2024-07-20 01:21:42,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 719 |
+
2024-07-20 01:21:42,972 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 720 |
+
2024-07-20 01:21:43,326 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 721 |
+
2024-07-20 01:21:43,972 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 722 |
+
2024-07-20 01:21:44,972 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 723 |
+
2024-07-20 01:21:45,972 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 724 |
+
2024-07-20 01:21:46,972 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 725 |
+
2024-07-20 01:21:47,972 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 726 |
+
2024-07-20 01:21:48,327 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 727 |
+
2024-07-20 01:21:48,972 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 728 |
+
2024-07-20 01:21:49,972 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 729 |
+
2024-07-20 01:21:50,973 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 730 |
+
2024-07-20 01:21:51,973 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 731 |
+
2024-07-20 01:21:52,973 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 732 |
+
2024-07-20 01:21:53,328 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 733 |
+
2024-07-20 01:21:53,973 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 734 |
+
2024-07-20 01:21:54,973 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 735 |
+
2024-07-20 01:21:55,974 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 736 |
+
2024-07-20 01:21:56,974 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 737 |
+
2024-07-20 01:21:57,232 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 738 |
+
2024-07-20 01:21:57,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 739 |
+
2024-07-20 01:21:57,974 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 740 |
+
2024-07-20 01:21:58,974 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 741 |
+
2024-07-20 01:21:59,290 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 742 |
+
2024-07-20 01:21:59,974 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 743 |
+
2024-07-20 01:22:00,974 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 744 |
+
2024-07-20 01:22:01,974 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 745 |
+
2024-07-20 01:22:02,975 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 746 |
+
2024-07-20 01:22:03,975 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 747 |
+
2024-07-20 01:22:04,290 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 748 |
+
2024-07-20 01:22:04,975 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 749 |
+
2024-07-20 01:22:05,975 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 750 |
+
2024-07-20 01:22:06,975 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 751 |
+
2024-07-20 01:22:07,975 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 752 |
+
2024-07-20 01:22:08,976 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 753 |
+
2024-07-20 01:22:09,291 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 754 |
+
2024-07-20 01:22:09,976 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 755 |
+
2024-07-20 01:22:10,976 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 756 |
+
2024-07-20 01:22:11,976 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 757 |
+
2024-07-20 01:22:12,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 758 |
+
2024-07-20 01:22:12,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 759 |
+
2024-07-20 01:22:12,976 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 760 |
+
2024-07-20 01:22:13,976 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 761 |
+
2024-07-20 01:22:14,326 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 762 |
+
2024-07-20 01:22:14,976 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 763 |
+
2024-07-20 01:22:15,977 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 764 |
+
2024-07-20 01:22:16,977 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 765 |
+
2024-07-20 01:22:17,977 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 766 |
+
2024-07-20 01:22:18,977 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 767 |
+
2024-07-20 01:22:19,327 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 768 |
+
2024-07-20 01:22:19,977 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 769 |
+
2024-07-20 01:22:20,977 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 770 |
+
2024-07-20 01:22:21,978 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 771 |
+
2024-07-20 01:22:22,978 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 772 |
+
2024-07-20 01:22:23,978 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 773 |
+
2024-07-20 01:22:24,328 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 774 |
+
2024-07-20 01:22:24,978 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 775 |
+
2024-07-20 01:22:25,978 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 776 |
+
2024-07-20 01:22:26,978 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 777 |
+
2024-07-20 01:22:27,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 778 |
+
2024-07-20 01:22:27,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 779 |
+
2024-07-20 01:22:27,978 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 780 |
+
2024-07-20 01:22:28,979 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 781 |
+
2024-07-20 01:22:29,398 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 782 |
+
2024-07-20 01:22:29,979 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 783 |
+
2024-07-20 01:22:30,979 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 784 |
+
2024-07-20 01:22:31,979 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 785 |
+
2024-07-20 01:22:32,979 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 786 |
+
2024-07-20 01:22:33,979 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 787 |
+
2024-07-20 01:22:34,399 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 788 |
+
2024-07-20 01:22:34,979 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 789 |
+
2024-07-20 01:22:35,980 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 790 |
+
2024-07-20 01:22:36,980 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 791 |
+
2024-07-20 01:22:37,980 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 792 |
+
2024-07-20 01:22:38,980 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 793 |
+
2024-07-20 01:22:39,400 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 794 |
+
2024-07-20 01:22:39,980 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 795 |
+
2024-07-20 01:22:40,981 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 796 |
+
2024-07-20 01:22:41,981 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 797 |
+
2024-07-20 01:22:42,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 798 |
+
2024-07-20 01:22:42,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 799 |
+
2024-07-20 01:22:42,981 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 800 |
+
2024-07-20 01:22:43,981 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 801 |
+
2024-07-20 01:22:44,981 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 802 |
+
2024-07-20 01:22:45,310 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 803 |
+
2024-07-20 01:22:45,981 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 804 |
+
2024-07-20 01:22:46,982 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 805 |
+
2024-07-20 01:22:47,982 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 806 |
+
2024-07-20 01:22:48,982 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 807 |
+
2024-07-20 01:22:49,982 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 808 |
+
2024-07-20 01:22:50,311 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 809 |
+
2024-07-20 01:22:50,982 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 810 |
+
2024-07-20 01:22:51,982 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 811 |
+
2024-07-20 01:22:52,983 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 812 |
+
2024-07-20 01:22:53,983 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 813 |
+
2024-07-20 01:22:54,983 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 814 |
+
2024-07-20 01:22:55,312 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 815 |
+
2024-07-20 01:22:55,983 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 816 |
+
2024-07-20 01:22:56,983 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 817 |
+
2024-07-20 01:22:57,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 818 |
+
2024-07-20 01:22:57,234 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 819 |
+
2024-07-20 01:22:57,984 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 820 |
+
2024-07-20 01:22:58,984 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 821 |
+
2024-07-20 01:22:59,714 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume
|
| 822 |
+
2024-07-20 01:22:59,715 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread
|
| 823 |
+
2024-07-20 01:22:59,715 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor
|
| 824 |
+
2024-07-20 01:22:59,715 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads
|
| 825 |
+
2024-07-20 01:22:59,716 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring
|
| 826 |
+
2024-07-20 01:22:59,717 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring
|
| 827 |
+
2024-07-20 01:22:59,717 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring
|
| 828 |
+
2024-07-20 01:22:59,720 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring
|
| 829 |
+
2024-07-20 01:22:59,721 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring
|
| 830 |
+
2024-07-20 01:22:59,756 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause
|
| 831 |
+
2024-07-20 01:22:59,756 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread
|
| 832 |
+
2024-07-20 01:22:59,756 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor
|
| 833 |
+
2024-07-20 01:22:59,757 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor
|
| 834 |
+
2024-07-20 01:22:59,757 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop
|
| 835 |
+
2024-07-20 01:22:59,757 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor
|
| 836 |
+
2024-07-20 01:22:59,757 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop
|
| 837 |
+
2024-07-20 01:22:59,758 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics
|
| 838 |
+
2024-07-20 01:22:59,763 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor
|
| 839 |
+
2024-07-20 01:22:59,763 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor
|
| 840 |
+
2024-07-20 01:22:59,763 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor
|
| 841 |
+
2024-07-20 01:22:59,764 DEBUG SenderThread:179 [sender.py:send():379] send: stats
|
| 842 |
+
2024-07-20 01:22:59,984 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 843 |
+
2024-07-20 01:23:00,463 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume
|
| 844 |
+
2024-07-20 01:23:00,463 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread
|
| 845 |
+
2024-07-20 01:23:00,463 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor
|
| 846 |
+
2024-07-20 01:23:00,463 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads
|
| 847 |
+
2024-07-20 01:23:00,464 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring
|
| 848 |
+
2024-07-20 01:23:00,464 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring
|
| 849 |
+
2024-07-20 01:23:00,465 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring
|
| 850 |
+
2024-07-20 01:23:00,470 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring
|
| 851 |
+
2024-07-20 01:23:00,470 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring
|
| 852 |
+
2024-07-20 01:23:00,503 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause
|
| 853 |
+
2024-07-20 01:23:00,503 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread
|
| 854 |
+
2024-07-20 01:23:00,503 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor
|
| 855 |
+
2024-07-20 01:23:00,503 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor
|
| 856 |
+
2024-07-20 01:23:00,504 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor
|
| 857 |
+
2024-07-20 01:23:00,504 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop
|
| 858 |
+
2024-07-20 01:23:00,504 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop
|
| 859 |
+
2024-07-20 01:23:00,504 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics
|
| 860 |
+
2024-07-20 01:23:00,509 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor
|
| 861 |
+
2024-07-20 01:23:00,510 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor
|
| 862 |
+
2024-07-20 01:23:00,510 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor
|
| 863 |
+
2024-07-20 01:23:00,510 DEBUG SenderThread:179 [sender.py:send():379] send: stats
|
| 864 |
+
2024-07-20 01:23:00,511 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 865 |
+
2024-07-20 01:23:00,984 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 866 |
+
2024-07-20 01:23:01,882 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume
|
| 867 |
+
2024-07-20 01:23:01,882 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread
|
| 868 |
+
2024-07-20 01:23:01,883 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor
|
| 869 |
+
2024-07-20 01:23:01,883 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads
|
| 870 |
+
2024-07-20 01:23:01,883 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring
|
| 871 |
+
2024-07-20 01:23:01,884 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring
|
| 872 |
+
2024-07-20 01:23:01,884 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring
|
| 873 |
+
2024-07-20 01:23:01,885 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring
|
| 874 |
+
2024-07-20 01:23:01,885 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring
|
| 875 |
+
2024-07-20 01:23:01,923 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause
|
| 876 |
+
2024-07-20 01:23:01,923 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread
|
| 877 |
+
2024-07-20 01:23:01,923 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor
|
| 878 |
+
2024-07-20 01:23:01,924 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor
|
| 879 |
+
2024-07-20 01:23:01,924 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor
|
| 880 |
+
2024-07-20 01:23:01,924 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop
|
| 881 |
+
2024-07-20 01:23:01,924 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop
|
| 882 |
+
2024-07-20 01:23:01,924 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics
|
| 883 |
+
2024-07-20 01:23:01,930 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor
|
| 884 |
+
2024-07-20 01:23:01,930 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor
|
| 885 |
+
2024-07-20 01:23:01,930 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor
|
| 886 |
+
2024-07-20 01:23:01,930 DEBUG SenderThread:179 [sender.py:send():379] send: stats
|
| 887 |
+
2024-07-20 01:23:01,984 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 888 |
+
2024-07-20 01:23:02,980 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume
|
| 889 |
+
2024-07-20 01:23:02,980 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread
|
| 890 |
+
2024-07-20 01:23:02,980 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor
|
| 891 |
+
2024-07-20 01:23:02,980 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads
|
| 892 |
+
2024-07-20 01:23:02,981 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring
|
| 893 |
+
2024-07-20 01:23:02,981 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring
|
| 894 |
+
2024-07-20 01:23:02,982 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring
|
| 895 |
+
2024-07-20 01:23:02,983 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring
|
| 896 |
+
2024-07-20 01:23:02,984 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring
|
| 897 |
+
2024-07-20 01:23:03,020 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause
|
| 898 |
+
2024-07-20 01:23:03,020 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread
|
| 899 |
+
2024-07-20 01:23:03,020 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor
|
| 900 |
+
2024-07-20 01:23:03,020 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop
|
| 901 |
+
2024-07-20 01:23:03,021 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop
|
| 902 |
+
2024-07-20 01:23:03,021 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics
|
| 903 |
+
2024-07-20 01:23:03,021 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor
|
| 904 |
+
2024-07-20 01:23:03,021 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor
|
| 905 |
+
2024-07-20 01:23:03,026 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor
|
| 906 |
+
2024-07-20 01:23:03,027 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor
|
| 907 |
+
2024-07-20 01:23:03,027 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor
|
| 908 |
+
2024-07-20 01:23:03,027 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 909 |
+
2024-07-20 01:23:03,027 DEBUG SenderThread:179 [sender.py:send():379] send: stats
|
| 910 |
+
2024-07-20 01:23:03,185 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume
|
| 911 |
+
2024-07-20 01:23:03,185 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread
|
| 912 |
+
2024-07-20 01:23:03,185 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor
|
| 913 |
+
2024-07-20 01:23:03,185 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads
|
| 914 |
+
2024-07-20 01:23:03,186 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring
|
| 915 |
+
2024-07-20 01:23:03,187 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring
|
| 916 |
+
2024-07-20 01:23:03,188 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring
|
| 917 |
+
2024-07-20 01:23:03,189 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring
|
| 918 |
+
2024-07-20 01:23:03,190 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring
|
| 919 |
+
2024-07-20 01:23:03,327 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause
|
| 920 |
+
2024-07-20 01:23:03,327 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread
|
| 921 |
+
2024-07-20 01:23:03,327 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor
|
| 922 |
+
2024-07-20 01:23:03,328 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor
|
| 923 |
+
2024-07-20 01:23:03,328 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop
|
| 924 |
+
2024-07-20 01:23:03,328 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor
|
| 925 |
+
2024-07-20 01:23:03,329 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop
|
| 926 |
+
2024-07-20 01:23:03,329 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics
|
| 927 |
+
2024-07-20 01:23:03,335 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor
|
| 928 |
+
2024-07-20 01:23:03,335 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor
|
| 929 |
+
2024-07-20 01:23:03,335 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor
|
| 930 |
+
2024-07-20 01:23:03,336 DEBUG SenderThread:179 [sender.py:send():379] send: stats
|
| 931 |
+
2024-07-20 01:23:03,985 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 932 |
+
2024-07-20 01:23:04,985 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 933 |
+
2024-07-20 01:23:05,985 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 934 |
+
2024-07-20 01:23:06,149 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume
|
| 935 |
+
2024-07-20 01:23:06,149 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread
|
| 936 |
+
2024-07-20 01:23:06,149 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor
|
| 937 |
+
2024-07-20 01:23:06,150 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads
|
| 938 |
+
2024-07-20 01:23:06,150 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring
|
| 939 |
+
2024-07-20 01:23:06,151 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring
|
| 940 |
+
2024-07-20 01:23:06,152 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring
|
| 941 |
+
2024-07-20 01:23:06,153 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring
|
| 942 |
+
2024-07-20 01:23:06,153 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring
|
| 943 |
+
2024-07-20 01:23:06,337 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 944 |
+
2024-07-20 01:23:06,874 DEBUG SenderThread:179 [sender.py:send():379] send: config
|
| 945 |
+
2024-07-20 01:23:06,875 DEBUG SenderThread:179 [sender.py:send():379] send: metric
|
| 946 |
+
2024-07-20 01:23:06,875 DEBUG SenderThread:179 [sender.py:send():379] send: metric
|
| 947 |
+
2024-07-20 01:23:06,875 WARNING SenderThread:179 [sender.py:send_metric():1405] Seen metric with glob (shouldn't happen)
|
| 948 |
+
2024-07-20 01:23:06,967 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: log_artifact
|
| 949 |
+
2024-07-20 01:23:06,967 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: log_artifact
|
| 950 |
+
2024-07-20 01:23:06,985 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 951 |
+
2024-07-20 01:23:07,078 INFO SenderThread:179 [sender.py:send_request_log_artifact():1518] logged artifact model-w16tb06h - {'id': 'QXJ0aWZhY3Q6MTAwNDIwNjIwMg==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjI5MzEzNDE4MA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTAwNDIwNjIwMg==', 'versionIndex': 0}}}
|
| 952 |
+
2024-07-20 01:23:07,986 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 953 |
+
2024-07-20 01:23:08,903 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history
|
| 954 |
+
2024-07-20 01:23:08,905 DEBUG SenderThread:179 [sender.py:send():379] send: metric
|
| 955 |
+
2024-07-20 01:23:08,905 DEBUG SenderThread:179 [sender.py:send():379] send: metric
|
| 956 |
+
2024-07-20 01:23:08,905 DEBUG SenderThread:179 [sender.py:send():379] send: metric
|
| 957 |
+
2024-07-20 01:23:08,905 DEBUG SenderThread:179 [sender.py:send():379] send: history
|
| 958 |
+
2024-07-20 01:23:08,906 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 959 |
+
2024-07-20 01:23:08,906 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 960 |
+
2024-07-20 01:23:08,985 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 961 |
+
2024-07-20 01:23:09,285 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json
|
| 962 |
+
2024-07-20 01:23:09,986 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 963 |
+
2024-07-20 01:23:10,247 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history
|
| 964 |
+
2024-07-20 01:23:10,249 DEBUG SenderThread:179 [sender.py:send():379] send: history
|
| 965 |
+
2024-07-20 01:23:10,249 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 966 |
+
2024-07-20 01:23:10,249 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 967 |
+
2024-07-20 01:23:10,285 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json
|
| 968 |
+
2024-07-20 01:23:10,986 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 969 |
+
2024-07-20 01:23:11,576 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history
|
| 970 |
+
2024-07-20 01:23:11,578 DEBUG SenderThread:179 [sender.py:send():379] send: history
|
| 971 |
+
2024-07-20 01:23:11,578 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 972 |
+
2024-07-20 01:23:11,578 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 973 |
+
2024-07-20 01:23:11,579 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 974 |
+
2024-07-20 01:23:11,986 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 975 |
+
2024-07-20 01:23:12,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 976 |
+
2024-07-20 01:23:12,234 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 977 |
+
2024-07-20 01:23:12,286 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json
|
| 978 |
+
2024-07-20 01:23:12,868 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history
|
| 979 |
+
2024-07-20 01:23:12,869 DEBUG SenderThread:179 [sender.py:send():379] send: history
|
| 980 |
+
2024-07-20 01:23:12,870 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 981 |
+
2024-07-20 01:23:12,870 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 982 |
+
2024-07-20 01:23:12,986 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 983 |
+
2024-07-20 01:23:13,287 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json
|
| 984 |
+
2024-07-20 01:23:13,987 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 985 |
+
2024-07-20 01:23:14,247 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history
|
| 986 |
+
2024-07-20 01:23:14,248 DEBUG SenderThread:179 [sender.py:send():379] send: history
|
| 987 |
+
2024-07-20 01:23:14,249 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 988 |
+
2024-07-20 01:23:14,249 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 989 |
+
2024-07-20 01:23:14,287 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json
|
| 990 |
+
2024-07-20 01:23:14,987 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 991 |
+
2024-07-20 01:23:15,711 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history
|
| 992 |
+
2024-07-20 01:23:15,712 DEBUG SenderThread:179 [sender.py:send():379] send: history
|
| 993 |
+
2024-07-20 01:23:15,712 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 994 |
+
2024-07-20 01:23:15,713 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 995 |
+
2024-07-20 01:23:15,987 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 996 |
+
2024-07-20 01:23:16,288 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json
|
| 997 |
+
2024-07-20 01:23:16,713 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 998 |
+
2024-07-20 01:23:16,987 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 999 |
+
2024-07-20 01:23:17,034 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history
|
| 1000 |
+
2024-07-20 01:23:17,035 DEBUG SenderThread:179 [sender.py:send():379] send: history
|
| 1001 |
+
2024-07-20 01:23:17,036 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 1002 |
+
2024-07-20 01:23:17,036 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 1003 |
+
2024-07-20 01:23:17,289 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json
|
| 1004 |
+
2024-07-20 01:23:17,987 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 1005 |
+
2024-07-20 01:23:18,377 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history
|
| 1006 |
+
2024-07-20 01:23:18,378 DEBUG SenderThread:179 [sender.py:send():379] send: history
|
| 1007 |
+
2024-07-20 01:23:18,379 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 1008 |
+
2024-07-20 01:23:18,379 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 1009 |
+
2024-07-20 01:23:18,987 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 1010 |
+
2024-07-20 01:23:19,289 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json
|
| 1011 |
+
2024-07-20 01:23:19,834 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history
|
| 1012 |
+
2024-07-20 01:23:19,836 DEBUG SenderThread:179 [sender.py:send():379] send: history
|
| 1013 |
+
2024-07-20 01:23:19,836 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 1014 |
+
2024-07-20 01:23:19,836 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 1015 |
+
2024-07-20 01:23:19,987 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 1016 |
+
2024-07-20 01:23:20,290 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json
|
| 1017 |
+
2024-07-20 01:23:20,725 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history
|
| 1018 |
+
2024-07-20 01:23:20,727 DEBUG SenderThread:179 [sender.py:send():379] send: history
|
| 1019 |
+
2024-07-20 01:23:20,727 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 1020 |
+
2024-07-20 01:23:20,727 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 1021 |
+
2024-07-20 01:23:20,988 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 1022 |
+
2024-07-20 01:23:21,290 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json
|
| 1023 |
+
2024-07-20 01:23:21,728 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1024 |
+
2024-07-20 01:23:22,470 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 1025 |
+
2024-07-20 01:23:23,470 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 1026 |
+
2024-07-20 01:23:24,473 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 1027 |
+
2024-07-20 01:23:25,473 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 1028 |
+
2024-07-20 01:23:26,729 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1029 |
+
2024-07-20 01:23:26,939 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 1030 |
+
2024-07-20 01:23:31,234 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 1031 |
+
2024-07-20 01:23:31,234 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 1032 |
+
2024-07-20 01:23:31,274 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 1033 |
+
2024-07-20 01:23:32,356 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1034 |
+
2024-07-20 01:23:37,072 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1035 |
+
2024-07-20 01:23:37,364 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1036 |
+
2024-07-20 01:23:38,298 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/config.yaml
|
| 1037 |
+
2024-07-20 01:23:42,074 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1038 |
+
2024-07-20 01:23:46,232 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 1039 |
+
2024-07-20 01:23:50,052 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 1040 |
+
2024-07-20 01:23:52,076 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1041 |
+
2024-07-20 01:23:55,225 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1042 |
+
2024-07-20 01:23:57,082 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1043 |
+
2024-07-20 01:24:00,225 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1044 |
+
2024-07-20 01:24:01,232 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 1045 |
+
2024-07-20 01:24:01,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 1046 |
+
2024-07-20 01:24:05,395 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1047 |
+
2024-07-20 01:24:06,153 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop
|
| 1048 |
+
2024-07-20 01:24:06,154 DEBUG SenderThread:179 [sender.py:send():379] send: stats
|
| 1049 |
+
2024-07-20 01:24:07,084 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1050 |
+
2024-07-20 01:24:11,155 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1051 |
+
2024-07-20 01:24:12,085 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1052 |
+
2024-07-20 01:24:16,156 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1053 |
+
2024-07-20 01:24:16,232 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 1054 |
+
2024-07-20 01:24:16,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 1055 |
+
2024-07-20 01:24:16,568 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 1056 |
+
2024-07-20 01:24:16,568 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 1057 |
+
2024-07-20 01:24:16,571 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history
|
| 1058 |
+
2024-07-20 01:24:16,572 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 1059 |
+
2024-07-20 01:24:16,584 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 1060 |
+
2024-07-20 01:24:16,584 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 1061 |
+
2024-07-20 01:24:16,584 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 1062 |
+
2024-07-20 01:24:16,585 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 1063 |
+
2024-07-20 01:24:16,585 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 1064 |
+
2024-07-20 01:24:16,585 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 1065 |
+
2024-07-20 01:24:16,586 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 1066 |
+
2024-07-20 01:24:16,586 DEBUG SenderThread:179 [sender.py:send():379] send: history
|
| 1067 |
+
2024-07-20 01:24:16,586 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 1068 |
+
2024-07-20 01:24:16,586 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 1069 |
+
2024-07-20 01:24:17,312 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json
|
| 1070 |
+
2024-07-20 01:24:21,587 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1071 |
+
2024-07-20 01:24:22,087 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1072 |
+
2024-07-20 01:24:26,588 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1073 |
+
2024-07-20 01:24:27,088 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1074 |
+
2024-07-20 01:24:31,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 1075 |
+
2024-07-20 01:24:31,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 1076 |
+
2024-07-20 01:24:32,396 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1077 |
+
2024-07-20 01:24:36,155 DEBUG SenderThread:179 [sender.py:send():379] send: stats
|
| 1078 |
+
2024-07-20 01:24:37,090 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1079 |
+
2024-07-20 01:24:38,156 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1080 |
+
2024-07-20 01:24:42,091 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1081 |
+
2024-07-20 01:24:43,157 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1082 |
+
2024-07-20 01:24:46,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 1083 |
+
2024-07-20 01:24:46,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 1084 |
+
2024-07-20 01:24:48,393 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1085 |
+
2024-07-20 01:24:52,094 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1086 |
+
2024-07-20 01:24:53,394 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1087 |
+
2024-07-20 01:24:57,095 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1088 |
+
2024-07-20 01:24:58,395 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1089 |
+
2024-07-20 01:25:01,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 1090 |
+
2024-07-20 01:25:01,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 1091 |
+
2024-07-20 01:25:04,381 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1092 |
+
2024-07-20 01:25:06,156 DEBUG SenderThread:179 [sender.py:send():379] send: stats
|
| 1093 |
+
2024-07-20 01:25:07,098 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1094 |
+
2024-07-20 01:25:10,157 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1095 |
+
2024-07-20 01:25:12,099 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1096 |
+
2024-07-20 01:25:15,158 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1097 |
+
2024-07-20 01:25:16,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 1098 |
+
2024-07-20 01:25:16,234 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 1099 |
+
2024-07-20 01:25:20,394 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1100 |
+
2024-07-20 01:25:22,101 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1101 |
+
2024-07-20 01:25:25,395 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1102 |
+
2024-07-20 01:25:27,102 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1103 |
+
2024-07-20 01:25:30,396 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1104 |
+
2024-07-20 01:25:31,234 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 1105 |
+
2024-07-20 01:25:31,234 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 1106 |
+
2024-07-20 01:25:36,157 DEBUG SenderThread:179 [sender.py:send():379] send: stats
|
| 1107 |
+
2024-07-20 01:25:36,157 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1108 |
+
2024-07-20 01:25:37,104 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1109 |
+
2024-07-20 01:25:41,158 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1110 |
+
2024-07-20 01:25:42,106 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1111 |
+
2024-07-20 01:25:46,159 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1112 |
+
2024-07-20 01:25:46,234 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 1113 |
+
2024-07-20 01:25:46,234 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 1114 |
+
2024-07-20 01:25:51,327 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1115 |
+
2024-07-20 01:25:52,108 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1116 |
+
2024-07-20 01:25:56,328 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1117 |
+
2024-07-20 01:25:57,110 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1118 |
+
2024-07-20 01:26:01,234 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 1119 |
+
2024-07-20 01:26:01,234 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 1120 |
+
2024-07-20 01:26:02,310 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1121 |
+
2024-07-20 01:26:06,157 DEBUG SenderThread:179 [sender.py:send():379] send: stats
|
| 1122 |
+
2024-07-20 01:26:07,112 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1123 |
+
2024-07-20 01:26:08,158 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1124 |
+
2024-07-20 01:26:12,113 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1125 |
+
2024-07-20 01:26:13,159 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1126 |
+
2024-07-20 01:26:16,234 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 1127 |
+
2024-07-20 01:26:16,235 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 1128 |
+
2024-07-20 01:26:18,386 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1129 |
+
2024-07-20 01:26:22,115 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
wandb/debug.log
CHANGED
|
@@ -45,3 +45,25 @@ config: {}
|
|
| 45 |
2024-07-20 01:15:55,259 INFO MainThread:34 [wandb_init.py:_pause_backend():440] pausing backend
|
| 46 |
2024-07-20 01:15:58,284 INFO MainThread:34 [wandb_init.py:_resume_backend():445] resuming backend
|
| 47 |
2024-07-20 01:15:58,845 INFO MainThread:34 [wandb_run.py:_config_callback():1382] config_cb None None {'output_dir': './', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'eval_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 10, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 1e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 1, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './runs/Jul20_01-15-42_3bc9163d6d69', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 10, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 500, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': 'VQA2', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': None, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
2024-07-20 01:15:55,259 INFO MainThread:34 [wandb_init.py:_pause_backend():440] pausing backend
|
| 46 |
2024-07-20 01:15:58,284 INFO MainThread:34 [wandb_init.py:_resume_backend():445] resuming backend
|
| 47 |
2024-07-20 01:15:58,845 INFO MainThread:34 [wandb_run.py:_config_callback():1382] config_cb None None {'output_dir': './', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'eval_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 10, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 1e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 1, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './runs/Jul20_01-15-42_3bc9163d6d69', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 10, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 500, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': 'VQA2', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': None, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False}
|
| 48 |
+
2024-07-20 01:18:50,373 INFO MainThread:34 [jupyter.py:save_ipynb():372] not saving jupyter notebook
|
| 49 |
+
2024-07-20 01:18:50,373 INFO MainThread:34 [wandb_init.py:_pause_backend():440] pausing backend
|
| 50 |
+
2024-07-20 01:21:08,290 INFO MainThread:34 [wandb_init.py:_resume_backend():445] resuming backend
|
| 51 |
+
2024-07-20 01:21:09,256 INFO MainThread:34 [jupyter.py:save_ipynb():372] not saving jupyter notebook
|
| 52 |
+
2024-07-20 01:21:09,256 INFO MainThread:34 [wandb_init.py:_pause_backend():440] pausing backend
|
| 53 |
+
2024-07-20 01:22:59,714 INFO MainThread:34 [wandb_init.py:_resume_backend():445] resuming backend
|
| 54 |
+
2024-07-20 01:22:59,740 INFO MainThread:34 [jupyter.py:save_ipynb():372] not saving jupyter notebook
|
| 55 |
+
2024-07-20 01:22:59,740 INFO MainThread:34 [wandb_init.py:_pause_backend():440] pausing backend
|
| 56 |
+
2024-07-20 01:23:00,462 INFO MainThread:34 [wandb_init.py:_resume_backend():445] resuming backend
|
| 57 |
+
2024-07-20 01:23:00,464 INFO MainThread:34 [jupyter.py:save_ipynb():372] not saving jupyter notebook
|
| 58 |
+
2024-07-20 01:23:00,464 INFO MainThread:34 [wandb_init.py:_pause_backend():440] pausing backend
|
| 59 |
+
2024-07-20 01:23:01,882 INFO MainThread:34 [wandb_init.py:_resume_backend():445] resuming backend
|
| 60 |
+
2024-07-20 01:23:01,885 INFO MainThread:34 [jupyter.py:save_ipynb():372] not saving jupyter notebook
|
| 61 |
+
2024-07-20 01:23:01,885 INFO MainThread:34 [wandb_init.py:_pause_backend():440] pausing backend
|
| 62 |
+
2024-07-20 01:23:02,979 INFO MainThread:34 [wandb_init.py:_resume_backend():445] resuming backend
|
| 63 |
+
2024-07-20 01:23:02,980 INFO MainThread:34 [jupyter.py:save_ipynb():372] not saving jupyter notebook
|
| 64 |
+
2024-07-20 01:23:02,981 INFO MainThread:34 [wandb_init.py:_pause_backend():440] pausing backend
|
| 65 |
+
2024-07-20 01:23:03,184 INFO MainThread:34 [wandb_init.py:_resume_backend():445] resuming backend
|
| 66 |
+
2024-07-20 01:23:03,326 INFO MainThread:34 [jupyter.py:save_ipynb():372] not saving jupyter notebook
|
| 67 |
+
2024-07-20 01:23:03,327 INFO MainThread:34 [wandb_init.py:_pause_backend():440] pausing backend
|
| 68 |
+
2024-07-20 01:23:06,148 INFO MainThread:34 [wandb_init.py:_resume_backend():445] resuming backend
|
| 69 |
+
2024-07-20 01:23:06,870 INFO MainThread:34 [wandb_run.py:_config_callback():1382] config_cb None None {'output_dir': './', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'eval_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 1e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 1, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './runs/Jul20_01-22-59_3bc9163d6d69', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 500, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': 'VQA2', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': None, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False}
|
wandb/run-20240720_011510-w16tb06h/files/config.yaml
CHANGED
|
@@ -64,6 +64,18 @@ _wandb:
|
|
| 64 |
5: 1
|
| 65 |
6:
|
| 66 |
- 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
output_dir:
|
| 68 |
desc: null
|
| 69 |
value: ./
|
|
@@ -99,7 +111,7 @@ per_gpu_eval_batch_size:
|
|
| 99 |
value: null
|
| 100 |
gradient_accumulation_steps:
|
| 101 |
desc: null
|
| 102 |
-
value:
|
| 103 |
eval_accumulation_steps:
|
| 104 |
desc: null
|
| 105 |
value: null
|
|
@@ -153,7 +165,7 @@ log_on_each_node:
|
|
| 153 |
value: true
|
| 154 |
logging_dir:
|
| 155 |
desc: null
|
| 156 |
-
value: ./runs/Jul20_01-
|
| 157 |
logging_strategy:
|
| 158 |
desc: null
|
| 159 |
value: steps
|
|
@@ -162,7 +174,7 @@ logging_first_step:
|
|
| 162 |
value: false
|
| 163 |
logging_steps:
|
| 164 |
desc: null
|
| 165 |
-
value:
|
| 166 |
logging_nan_inf_filter:
|
| 167 |
desc: null
|
| 168 |
value: true
|
|
|
|
| 64 |
5: 1
|
| 65 |
6:
|
| 66 |
- 1
|
| 67 |
+
- 1: train/loss
|
| 68 |
+
5: 1
|
| 69 |
+
6:
|
| 70 |
+
- 1
|
| 71 |
+
- 1: train/grad_norm
|
| 72 |
+
5: 1
|
| 73 |
+
6:
|
| 74 |
+
- 1
|
| 75 |
+
- 1: train/learning_rate
|
| 76 |
+
5: 1
|
| 77 |
+
6:
|
| 78 |
+
- 1
|
| 79 |
output_dir:
|
| 80 |
desc: null
|
| 81 |
value: ./
|
|
|
|
| 111 |
value: null
|
| 112 |
gradient_accumulation_steps:
|
| 113 |
desc: null
|
| 114 |
+
value: 1
|
| 115 |
eval_accumulation_steps:
|
| 116 |
desc: null
|
| 117 |
value: null
|
|
|
|
| 165 |
value: true
|
| 166 |
logging_dir:
|
| 167 |
desc: null
|
| 168 |
+
value: ./runs/Jul20_01-22-59_3bc9163d6d69
|
| 169 |
logging_strategy:
|
| 170 |
desc: null
|
| 171 |
value: steps
|
|
|
|
| 174 |
value: false
|
| 175 |
logging_steps:
|
| 176 |
desc: null
|
| 177 |
+
value: 1
|
| 178 |
logging_nan_inf_filter:
|
| 179 |
desc: null
|
| 180 |
value: true
|
wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"train_runtime": 69.7044, "train_samples_per_second": 0.143, "train_steps_per_second": 0.143, "total_flos": 0.0, "train_loss": 13.572482585906982, "train/epoch": 1.0, "train/global_step": 10, "_timestamp": 1721438656.5692015, "_runtime": 546.3884704113007, "_step": 11, "train/loss": 13.7642, "train/grad_norm": 1432.1019287109375, "train/learning_rate": 0.0}
|
wandb/run-20240720_011510-w16tb06h/logs/debug-internal.log
CHANGED
|
@@ -454,3 +454,676 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
|
|
| 454 |
2024-07-20 01:18:46,934 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 455 |
2024-07-20 01:18:47,935 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 456 |
2024-07-20 01:18:48,935 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 454 |
2024-07-20 01:18:46,934 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 455 |
2024-07-20 01:18:47,935 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 456 |
2024-07-20 01:18:48,935 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 457 |
+
2024-07-20 01:18:49,935 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 458 |
+
2024-07-20 01:18:50,373 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 459 |
+
2024-07-20 01:18:50,374 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause
|
| 460 |
+
2024-07-20 01:18:50,374 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread
|
| 461 |
+
2024-07-20 01:18:50,374 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor
|
| 462 |
+
2024-07-20 01:18:50,375 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop
|
| 463 |
+
2024-07-20 01:18:50,375 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics
|
| 464 |
+
2024-07-20 01:18:50,376 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor
|
| 465 |
+
2024-07-20 01:18:50,377 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor
|
| 466 |
+
2024-07-20 01:18:50,384 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor
|
| 467 |
+
2024-07-20 01:18:50,384 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor
|
| 468 |
+
2024-07-20 01:18:50,384 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor
|
| 469 |
+
2024-07-20 01:18:50,385 DEBUG SenderThread:179 [sender.py:send():379] send: stats
|
| 470 |
+
2024-07-20 01:18:50,935 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 471 |
+
2024-07-20 01:18:51,935 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 472 |
+
2024-07-20 01:18:52,935 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 473 |
+
2024-07-20 01:18:53,936 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 474 |
+
2024-07-20 01:18:54,936 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 475 |
+
2024-07-20 01:18:55,386 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 476 |
+
2024-07-20 01:18:55,936 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 477 |
+
2024-07-20 01:18:56,936 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 478 |
+
2024-07-20 01:18:57,231 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 479 |
+
2024-07-20 01:18:57,231 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 480 |
+
2024-07-20 01:18:57,936 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 481 |
+
2024-07-20 01:18:58,936 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 482 |
+
2024-07-20 01:18:59,937 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 483 |
+
2024-07-20 01:19:00,937 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 484 |
+
2024-07-20 01:19:01,315 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 485 |
+
2024-07-20 01:19:01,937 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 486 |
+
2024-07-20 01:19:02,937 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 487 |
+
2024-07-20 01:19:03,937 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 488 |
+
2024-07-20 01:19:04,937 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 489 |
+
2024-07-20 01:19:05,938 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 490 |
+
2024-07-20 01:19:06,316 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 491 |
+
2024-07-20 01:19:06,938 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 492 |
+
2024-07-20 01:19:07,938 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 493 |
+
2024-07-20 01:19:08,938 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 494 |
+
2024-07-20 01:19:09,938 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 495 |
+
2024-07-20 01:19:10,938 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 496 |
+
2024-07-20 01:19:11,317 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 497 |
+
2024-07-20 01:19:11,938 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 498 |
+
2024-07-20 01:19:12,231 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 499 |
+
2024-07-20 01:19:12,231 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 500 |
+
2024-07-20 01:19:12,938 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 501 |
+
2024-07-20 01:19:13,939 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 502 |
+
2024-07-20 01:19:14,939 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 503 |
+
2024-07-20 01:19:15,939 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 504 |
+
2024-07-20 01:19:16,939 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 505 |
+
2024-07-20 01:19:17,309 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 506 |
+
2024-07-20 01:19:17,939 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 507 |
+
2024-07-20 01:19:18,939 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 508 |
+
2024-07-20 01:19:19,939 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 509 |
+
2024-07-20 01:19:20,940 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 510 |
+
2024-07-20 01:19:21,940 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 511 |
+
2024-07-20 01:19:22,310 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 512 |
+
2024-07-20 01:19:22,940 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 513 |
+
2024-07-20 01:19:23,940 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 514 |
+
2024-07-20 01:19:24,940 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 515 |
+
2024-07-20 01:19:25,940 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 516 |
+
2024-07-20 01:19:26,941 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 517 |
+
2024-07-20 01:19:27,231 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 518 |
+
2024-07-20 01:19:27,231 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 519 |
+
2024-07-20 01:19:27,310 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 520 |
+
2024-07-20 01:19:27,941 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 521 |
+
2024-07-20 01:19:28,941 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 522 |
+
2024-07-20 01:19:29,941 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 523 |
+
2024-07-20 01:19:30,941 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 524 |
+
2024-07-20 01:19:31,941 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 525 |
+
2024-07-20 01:19:32,311 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 526 |
+
2024-07-20 01:19:32,941 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 527 |
+
2024-07-20 01:19:33,942 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 528 |
+
2024-07-20 01:19:34,942 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 529 |
+
2024-07-20 01:19:35,942 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 530 |
+
2024-07-20 01:19:36,942 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 531 |
+
2024-07-20 01:19:37,312 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 532 |
+
2024-07-20 01:19:37,942 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 533 |
+
2024-07-20 01:19:38,942 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 534 |
+
2024-07-20 01:19:39,942 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 535 |
+
2024-07-20 01:19:40,943 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 536 |
+
2024-07-20 01:19:41,943 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 537 |
+
2024-07-20 01:19:42,231 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 538 |
+
2024-07-20 01:19:42,231 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 539 |
+
2024-07-20 01:19:42,943 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 540 |
+
2024-07-20 01:19:43,309 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 541 |
+
2024-07-20 01:19:43,943 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 542 |
+
2024-07-20 01:19:44,943 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 543 |
+
2024-07-20 01:19:45,943 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 544 |
+
2024-07-20 01:19:46,943 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 545 |
+
2024-07-20 01:19:47,944 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 546 |
+
2024-07-20 01:19:48,310 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 547 |
+
2024-07-20 01:19:48,944 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 548 |
+
2024-07-20 01:19:49,944 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 549 |
+
2024-07-20 01:19:50,944 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 550 |
+
2024-07-20 01:19:51,944 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 551 |
+
2024-07-20 01:19:52,945 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 552 |
+
2024-07-20 01:19:53,311 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 553 |
+
2024-07-20 01:19:53,945 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 554 |
+
2024-07-20 01:19:54,945 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 555 |
+
2024-07-20 01:19:55,945 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 556 |
+
2024-07-20 01:19:56,945 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 557 |
+
2024-07-20 01:19:57,231 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 558 |
+
2024-07-20 01:19:57,232 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 559 |
+
2024-07-20 01:19:57,945 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 560 |
+
2024-07-20 01:19:58,358 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 561 |
+
2024-07-20 01:19:58,946 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 562 |
+
2024-07-20 01:19:59,946 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 563 |
+
2024-07-20 01:20:00,946 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 564 |
+
2024-07-20 01:20:01,946 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 565 |
+
2024-07-20 01:20:02,946 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 566 |
+
2024-07-20 01:20:03,359 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 567 |
+
2024-07-20 01:20:03,946 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 568 |
+
2024-07-20 01:20:04,947 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 569 |
+
2024-07-20 01:20:05,947 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 570 |
+
2024-07-20 01:20:06,947 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 571 |
+
2024-07-20 01:20:07,947 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 572 |
+
2024-07-20 01:20:08,360 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 573 |
+
2024-07-20 01:20:08,947 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 574 |
+
2024-07-20 01:20:09,947 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 575 |
+
2024-07-20 01:20:10,948 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 576 |
+
2024-07-20 01:20:11,948 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 577 |
+
2024-07-20 01:20:12,231 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 578 |
+
2024-07-20 01:20:12,232 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 579 |
+
2024-07-20 01:20:12,948 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 580 |
+
2024-07-20 01:20:13,948 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 581 |
+
2024-07-20 01:20:14,353 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 582 |
+
2024-07-20 01:20:14,948 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 583 |
+
2024-07-20 01:20:15,948 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 584 |
+
2024-07-20 01:20:16,949 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 585 |
+
2024-07-20 01:20:17,949 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 586 |
+
2024-07-20 01:20:18,949 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 587 |
+
2024-07-20 01:20:19,354 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 588 |
+
2024-07-20 01:20:19,949 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 589 |
+
2024-07-20 01:20:20,949 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 590 |
+
2024-07-20 01:20:21,950 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 591 |
+
2024-07-20 01:20:22,950 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 592 |
+
2024-07-20 01:20:23,950 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 593 |
+
2024-07-20 01:20:24,355 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 594 |
+
2024-07-20 01:20:24,950 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 595 |
+
2024-07-20 01:20:25,950 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 596 |
+
2024-07-20 01:20:26,950 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 597 |
+
2024-07-20 01:20:27,231 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 598 |
+
2024-07-20 01:20:27,232 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 599 |
+
2024-07-20 01:20:27,950 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 600 |
+
2024-07-20 01:20:28,951 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 601 |
+
2024-07-20 01:20:29,951 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 602 |
+
2024-07-20 01:20:30,295 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 603 |
+
2024-07-20 01:20:30,951 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 604 |
+
2024-07-20 01:20:31,951 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 605 |
+
2024-07-20 01:20:32,951 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 606 |
+
2024-07-20 01:20:33,951 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 607 |
+
2024-07-20 01:20:34,951 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 608 |
+
2024-07-20 01:20:35,295 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 609 |
+
2024-07-20 01:20:35,952 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 610 |
+
2024-07-20 01:20:36,952 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 611 |
+
2024-07-20 01:20:37,952 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 612 |
+
2024-07-20 01:20:38,952 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 613 |
+
2024-07-20 01:20:39,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 614 |
+
2024-07-20 01:20:40,296 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 615 |
+
2024-07-20 01:20:40,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 616 |
+
2024-07-20 01:20:41,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 617 |
+
2024-07-20 01:20:42,231 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 618 |
+
2024-07-20 01:20:42,232 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 619 |
+
2024-07-20 01:20:42,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 620 |
+
2024-07-20 01:20:43,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 621 |
+
2024-07-20 01:20:44,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 622 |
+
2024-07-20 01:20:45,393 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 623 |
+
2024-07-20 01:20:45,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 624 |
+
2024-07-20 01:20:46,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 625 |
+
2024-07-20 01:20:47,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 626 |
+
2024-07-20 01:20:48,954 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 627 |
+
2024-07-20 01:20:49,954 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 628 |
+
2024-07-20 01:20:50,394 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 629 |
+
2024-07-20 01:20:50,954 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 630 |
+
2024-07-20 01:20:51,954 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 631 |
+
2024-07-20 01:20:52,954 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 632 |
+
2024-07-20 01:20:53,954 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 633 |
+
2024-07-20 01:20:54,954 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 634 |
+
2024-07-20 01:20:55,395 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 635 |
+
2024-07-20 01:20:55,954 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 636 |
+
2024-07-20 01:20:56,955 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 637 |
+
2024-07-20 01:20:57,232 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 638 |
+
2024-07-20 01:20:57,232 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 639 |
+
2024-07-20 01:20:57,955 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 640 |
+
2024-07-20 01:20:58,955 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 641 |
+
2024-07-20 01:20:59,955 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 642 |
+
2024-07-20 01:21:00,955 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 643 |
+
2024-07-20 01:21:01,341 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 644 |
+
2024-07-20 01:21:01,955 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 645 |
+
2024-07-20 01:21:02,955 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 646 |
+
2024-07-20 01:21:03,956 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 647 |
+
2024-07-20 01:21:04,956 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 648 |
+
2024-07-20 01:21:05,956 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 649 |
+
2024-07-20 01:21:06,342 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 650 |
+
2024-07-20 01:21:06,956 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 651 |
+
2024-07-20 01:21:07,956 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 652 |
+
2024-07-20 01:21:08,290 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume
|
| 653 |
+
2024-07-20 01:21:08,291 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread
|
| 654 |
+
2024-07-20 01:21:08,291 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor
|
| 655 |
+
2024-07-20 01:21:08,291 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads
|
| 656 |
+
2024-07-20 01:21:08,291 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring
|
| 657 |
+
2024-07-20 01:21:08,292 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring
|
| 658 |
+
2024-07-20 01:21:08,293 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring
|
| 659 |
+
2024-07-20 01:21:08,294 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring
|
| 660 |
+
2024-07-20 01:21:08,295 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring
|
| 661 |
+
2024-07-20 01:21:09,123 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 662 |
+
2024-07-20 01:21:09,256 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause
|
| 663 |
+
2024-07-20 01:21:09,257 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread
|
| 664 |
+
2024-07-20 01:21:09,257 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor
|
| 665 |
+
2024-07-20 01:21:09,257 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor
|
| 666 |
+
2024-07-20 01:21:09,257 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor
|
| 667 |
+
2024-07-20 01:21:09,258 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop
|
| 668 |
+
2024-07-20 01:21:09,258 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop
|
| 669 |
+
2024-07-20 01:21:09,258 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics
|
| 670 |
+
2024-07-20 01:21:09,265 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor
|
| 671 |
+
2024-07-20 01:21:09,266 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor
|
| 672 |
+
2024-07-20 01:21:09,266 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor
|
| 673 |
+
2024-07-20 01:21:09,266 DEBUG SenderThread:179 [sender.py:send():379] send: stats
|
| 674 |
+
2024-07-20 01:21:09,967 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 675 |
+
2024-07-20 01:21:10,967 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 676 |
+
2024-07-20 01:21:11,967 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 677 |
+
2024-07-20 01:21:12,232 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 678 |
+
2024-07-20 01:21:12,232 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 679 |
+
2024-07-20 01:21:12,360 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 680 |
+
2024-07-20 01:21:12,967 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 681 |
+
2024-07-20 01:21:13,967 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 682 |
+
2024-07-20 01:21:14,968 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 683 |
+
2024-07-20 01:21:15,968 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 684 |
+
2024-07-20 01:21:16,968 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 685 |
+
2024-07-20 01:21:17,361 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 686 |
+
2024-07-20 01:21:17,968 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 687 |
+
2024-07-20 01:21:18,968 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 688 |
+
2024-07-20 01:21:19,968 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 689 |
+
2024-07-20 01:21:20,968 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 690 |
+
2024-07-20 01:21:21,969 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 691 |
+
2024-07-20 01:21:22,362 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 692 |
+
2024-07-20 01:21:22,969 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 693 |
+
2024-07-20 01:21:23,969 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 694 |
+
2024-07-20 01:21:24,969 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 695 |
+
2024-07-20 01:21:25,969 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 696 |
+
2024-07-20 01:21:26,969 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 697 |
+
2024-07-20 01:21:27,232 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 698 |
+
2024-07-20 01:21:27,232 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 699 |
+
2024-07-20 01:21:27,404 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 700 |
+
2024-07-20 01:21:27,969 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 701 |
+
2024-07-20 01:21:28,969 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 702 |
+
2024-07-20 01:21:29,970 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 703 |
+
2024-07-20 01:21:30,970 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 704 |
+
2024-07-20 01:21:31,970 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 705 |
+
2024-07-20 01:21:32,405 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 706 |
+
2024-07-20 01:21:32,970 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 707 |
+
2024-07-20 01:21:33,970 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 708 |
+
2024-07-20 01:21:34,970 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 709 |
+
2024-07-20 01:21:35,970 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 710 |
+
2024-07-20 01:21:36,971 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 711 |
+
2024-07-20 01:21:37,406 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 712 |
+
2024-07-20 01:21:37,971 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 713 |
+
2024-07-20 01:21:38,971 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 714 |
+
2024-07-20 01:21:39,971 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 715 |
+
2024-07-20 01:21:40,971 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 716 |
+
2024-07-20 01:21:41,971 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 717 |
+
2024-07-20 01:21:42,232 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 718 |
+
2024-07-20 01:21:42,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 719 |
+
2024-07-20 01:21:42,972 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 720 |
+
2024-07-20 01:21:43,326 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 721 |
+
2024-07-20 01:21:43,972 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 722 |
+
2024-07-20 01:21:44,972 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 723 |
+
2024-07-20 01:21:45,972 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 724 |
+
2024-07-20 01:21:46,972 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 725 |
+
2024-07-20 01:21:47,972 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 726 |
+
2024-07-20 01:21:48,327 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 727 |
+
2024-07-20 01:21:48,972 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 728 |
+
2024-07-20 01:21:49,972 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 729 |
+
2024-07-20 01:21:50,973 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 730 |
+
2024-07-20 01:21:51,973 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 731 |
+
2024-07-20 01:21:52,973 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 732 |
+
2024-07-20 01:21:53,328 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 733 |
+
2024-07-20 01:21:53,973 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 734 |
+
2024-07-20 01:21:54,973 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 735 |
+
2024-07-20 01:21:55,974 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 736 |
+
2024-07-20 01:21:56,974 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 737 |
+
2024-07-20 01:21:57,232 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 738 |
+
2024-07-20 01:21:57,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 739 |
+
2024-07-20 01:21:57,974 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 740 |
+
2024-07-20 01:21:58,974 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 741 |
+
2024-07-20 01:21:59,290 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 742 |
+
2024-07-20 01:21:59,974 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 743 |
+
2024-07-20 01:22:00,974 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 744 |
+
2024-07-20 01:22:01,974 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 745 |
+
2024-07-20 01:22:02,975 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 746 |
+
2024-07-20 01:22:03,975 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 747 |
+
2024-07-20 01:22:04,290 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 748 |
+
2024-07-20 01:22:04,975 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 749 |
+
2024-07-20 01:22:05,975 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 750 |
+
2024-07-20 01:22:06,975 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 751 |
+
2024-07-20 01:22:07,975 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 752 |
+
2024-07-20 01:22:08,976 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 753 |
+
2024-07-20 01:22:09,291 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 754 |
+
2024-07-20 01:22:09,976 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 755 |
+
2024-07-20 01:22:10,976 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 756 |
+
2024-07-20 01:22:11,976 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 757 |
+
2024-07-20 01:22:12,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 758 |
+
2024-07-20 01:22:12,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 759 |
+
2024-07-20 01:22:12,976 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 760 |
+
2024-07-20 01:22:13,976 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 761 |
+
2024-07-20 01:22:14,326 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 762 |
+
2024-07-20 01:22:14,976 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 763 |
+
2024-07-20 01:22:15,977 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 764 |
+
2024-07-20 01:22:16,977 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 765 |
+
2024-07-20 01:22:17,977 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 766 |
+
2024-07-20 01:22:18,977 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 767 |
+
2024-07-20 01:22:19,327 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 768 |
+
2024-07-20 01:22:19,977 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 769 |
+
2024-07-20 01:22:20,977 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 770 |
+
2024-07-20 01:22:21,978 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 771 |
+
2024-07-20 01:22:22,978 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 772 |
+
2024-07-20 01:22:23,978 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 773 |
+
2024-07-20 01:22:24,328 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 774 |
+
2024-07-20 01:22:24,978 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 775 |
+
2024-07-20 01:22:25,978 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 776 |
+
2024-07-20 01:22:26,978 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 777 |
+
2024-07-20 01:22:27,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 778 |
+
2024-07-20 01:22:27,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 779 |
+
2024-07-20 01:22:27,978 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 780 |
+
2024-07-20 01:22:28,979 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 781 |
+
2024-07-20 01:22:29,398 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 782 |
+
2024-07-20 01:22:29,979 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 783 |
+
2024-07-20 01:22:30,979 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 784 |
+
2024-07-20 01:22:31,979 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 785 |
+
2024-07-20 01:22:32,979 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 786 |
+
2024-07-20 01:22:33,979 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 787 |
+
2024-07-20 01:22:34,399 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 788 |
+
2024-07-20 01:22:34,979 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 789 |
+
2024-07-20 01:22:35,980 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 790 |
+
2024-07-20 01:22:36,980 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 791 |
+
2024-07-20 01:22:37,980 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 792 |
+
2024-07-20 01:22:38,980 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 793 |
+
2024-07-20 01:22:39,400 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 794 |
+
2024-07-20 01:22:39,980 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 795 |
+
2024-07-20 01:22:40,981 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 796 |
+
2024-07-20 01:22:41,981 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 797 |
+
2024-07-20 01:22:42,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 798 |
+
2024-07-20 01:22:42,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 799 |
+
2024-07-20 01:22:42,981 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 800 |
+
2024-07-20 01:22:43,981 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 801 |
+
2024-07-20 01:22:44,981 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 802 |
+
2024-07-20 01:22:45,310 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 803 |
+
2024-07-20 01:22:45,981 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 804 |
+
2024-07-20 01:22:46,982 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 805 |
+
2024-07-20 01:22:47,982 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 806 |
+
2024-07-20 01:22:48,982 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 807 |
+
2024-07-20 01:22:49,982 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 808 |
+
2024-07-20 01:22:50,311 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 809 |
+
2024-07-20 01:22:50,982 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 810 |
+
2024-07-20 01:22:51,982 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 811 |
+
2024-07-20 01:22:52,983 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 812 |
+
2024-07-20 01:22:53,983 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 813 |
+
2024-07-20 01:22:54,983 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 814 |
+
2024-07-20 01:22:55,312 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 815 |
+
2024-07-20 01:22:55,983 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 816 |
+
2024-07-20 01:22:56,983 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 817 |
+
2024-07-20 01:22:57,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 818 |
+
2024-07-20 01:22:57,234 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 819 |
+
2024-07-20 01:22:57,984 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 820 |
+
2024-07-20 01:22:58,984 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 821 |
+
2024-07-20 01:22:59,714 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume
|
| 822 |
+
2024-07-20 01:22:59,715 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread
|
| 823 |
+
2024-07-20 01:22:59,715 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor
|
| 824 |
+
2024-07-20 01:22:59,715 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads
|
| 825 |
+
2024-07-20 01:22:59,716 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring
|
| 826 |
+
2024-07-20 01:22:59,717 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring
|
| 827 |
+
2024-07-20 01:22:59,717 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring
|
| 828 |
+
2024-07-20 01:22:59,720 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring
|
| 829 |
+
2024-07-20 01:22:59,721 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring
|
| 830 |
+
2024-07-20 01:22:59,756 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause
|
| 831 |
+
2024-07-20 01:22:59,756 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread
|
| 832 |
+
2024-07-20 01:22:59,756 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor
|
| 833 |
+
2024-07-20 01:22:59,757 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor
|
| 834 |
+
2024-07-20 01:22:59,757 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop
|
| 835 |
+
2024-07-20 01:22:59,757 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor
|
| 836 |
+
2024-07-20 01:22:59,757 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop
|
| 837 |
+
2024-07-20 01:22:59,758 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics
|
| 838 |
+
2024-07-20 01:22:59,763 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor
|
| 839 |
+
2024-07-20 01:22:59,763 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor
|
| 840 |
+
2024-07-20 01:22:59,763 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor
|
| 841 |
+
2024-07-20 01:22:59,764 DEBUG SenderThread:179 [sender.py:send():379] send: stats
|
| 842 |
+
2024-07-20 01:22:59,984 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 843 |
+
2024-07-20 01:23:00,463 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume
|
| 844 |
+
2024-07-20 01:23:00,463 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread
|
| 845 |
+
2024-07-20 01:23:00,463 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor
|
| 846 |
+
2024-07-20 01:23:00,463 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads
|
| 847 |
+
2024-07-20 01:23:00,464 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring
|
| 848 |
+
2024-07-20 01:23:00,464 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring
|
| 849 |
+
2024-07-20 01:23:00,465 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring
|
| 850 |
+
2024-07-20 01:23:00,470 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring
|
| 851 |
+
2024-07-20 01:23:00,470 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring
|
| 852 |
+
2024-07-20 01:23:00,503 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause
|
| 853 |
+
2024-07-20 01:23:00,503 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread
|
| 854 |
+
2024-07-20 01:23:00,503 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor
|
| 855 |
+
2024-07-20 01:23:00,503 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor
|
| 856 |
+
2024-07-20 01:23:00,504 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor
|
| 857 |
+
2024-07-20 01:23:00,504 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop
|
| 858 |
+
2024-07-20 01:23:00,504 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop
|
| 859 |
+
2024-07-20 01:23:00,504 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics
|
| 860 |
+
2024-07-20 01:23:00,509 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor
|
| 861 |
+
2024-07-20 01:23:00,510 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor
|
| 862 |
+
2024-07-20 01:23:00,510 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor
|
| 863 |
+
2024-07-20 01:23:00,510 DEBUG SenderThread:179 [sender.py:send():379] send: stats
|
| 864 |
+
2024-07-20 01:23:00,511 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 865 |
+
2024-07-20 01:23:00,984 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 866 |
+
2024-07-20 01:23:01,882 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume
|
| 867 |
+
2024-07-20 01:23:01,882 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread
|
| 868 |
+
2024-07-20 01:23:01,883 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor
|
| 869 |
+
2024-07-20 01:23:01,883 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads
|
| 870 |
+
2024-07-20 01:23:01,883 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring
|
| 871 |
+
2024-07-20 01:23:01,884 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring
|
| 872 |
+
2024-07-20 01:23:01,884 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring
|
| 873 |
+
2024-07-20 01:23:01,885 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring
|
| 874 |
+
2024-07-20 01:23:01,885 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring
|
| 875 |
+
2024-07-20 01:23:01,923 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause
|
| 876 |
+
2024-07-20 01:23:01,923 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread
|
| 877 |
+
2024-07-20 01:23:01,923 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor
|
| 878 |
+
2024-07-20 01:23:01,924 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor
|
| 879 |
+
2024-07-20 01:23:01,924 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor
|
| 880 |
+
2024-07-20 01:23:01,924 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop
|
| 881 |
+
2024-07-20 01:23:01,924 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop
|
| 882 |
+
2024-07-20 01:23:01,924 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics
|
| 883 |
+
2024-07-20 01:23:01,930 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor
|
| 884 |
+
2024-07-20 01:23:01,930 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor
|
| 885 |
+
2024-07-20 01:23:01,930 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor
|
| 886 |
+
2024-07-20 01:23:01,930 DEBUG SenderThread:179 [sender.py:send():379] send: stats
|
| 887 |
+
2024-07-20 01:23:01,984 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 888 |
+
2024-07-20 01:23:02,980 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume
|
| 889 |
+
2024-07-20 01:23:02,980 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread
|
| 890 |
+
2024-07-20 01:23:02,980 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor
|
| 891 |
+
2024-07-20 01:23:02,980 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads
|
| 892 |
+
2024-07-20 01:23:02,981 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring
|
| 893 |
+
2024-07-20 01:23:02,981 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring
|
| 894 |
+
2024-07-20 01:23:02,982 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring
|
| 895 |
+
2024-07-20 01:23:02,983 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring
|
| 896 |
+
2024-07-20 01:23:02,984 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring
|
| 897 |
+
2024-07-20 01:23:03,020 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause
|
| 898 |
+
2024-07-20 01:23:03,020 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread
|
| 899 |
+
2024-07-20 01:23:03,020 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor
|
| 900 |
+
2024-07-20 01:23:03,020 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop
|
| 901 |
+
2024-07-20 01:23:03,021 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop
|
| 902 |
+
2024-07-20 01:23:03,021 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics
|
| 903 |
+
2024-07-20 01:23:03,021 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor
|
| 904 |
+
2024-07-20 01:23:03,021 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor
|
| 905 |
+
2024-07-20 01:23:03,026 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor
|
| 906 |
+
2024-07-20 01:23:03,027 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor
|
| 907 |
+
2024-07-20 01:23:03,027 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor
|
| 908 |
+
2024-07-20 01:23:03,027 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 909 |
+
2024-07-20 01:23:03,027 DEBUG SenderThread:179 [sender.py:send():379] send: stats
|
| 910 |
+
2024-07-20 01:23:03,185 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume
|
| 911 |
+
2024-07-20 01:23:03,185 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread
|
| 912 |
+
2024-07-20 01:23:03,185 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor
|
| 913 |
+
2024-07-20 01:23:03,185 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads
|
| 914 |
+
2024-07-20 01:23:03,186 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring
|
| 915 |
+
2024-07-20 01:23:03,187 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring
|
| 916 |
+
2024-07-20 01:23:03,188 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring
|
| 917 |
+
2024-07-20 01:23:03,189 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring
|
| 918 |
+
2024-07-20 01:23:03,190 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring
|
| 919 |
+
2024-07-20 01:23:03,327 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause
|
| 920 |
+
2024-07-20 01:23:03,327 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread
|
| 921 |
+
2024-07-20 01:23:03,327 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor
|
| 922 |
+
2024-07-20 01:23:03,328 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor
|
| 923 |
+
2024-07-20 01:23:03,328 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop
|
| 924 |
+
2024-07-20 01:23:03,328 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor
|
| 925 |
+
2024-07-20 01:23:03,329 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop
|
| 926 |
+
2024-07-20 01:23:03,329 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics
|
| 927 |
+
2024-07-20 01:23:03,335 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor
|
| 928 |
+
2024-07-20 01:23:03,335 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor
|
| 929 |
+
2024-07-20 01:23:03,335 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor
|
| 930 |
+
2024-07-20 01:23:03,336 DEBUG SenderThread:179 [sender.py:send():379] send: stats
|
| 931 |
+
2024-07-20 01:23:03,985 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 932 |
+
2024-07-20 01:23:04,985 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 933 |
+
2024-07-20 01:23:05,985 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 934 |
+
2024-07-20 01:23:06,149 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume
|
| 935 |
+
2024-07-20 01:23:06,149 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread
|
| 936 |
+
2024-07-20 01:23:06,149 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor
|
| 937 |
+
2024-07-20 01:23:06,150 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads
|
| 938 |
+
2024-07-20 01:23:06,150 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring
|
| 939 |
+
2024-07-20 01:23:06,151 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring
|
| 940 |
+
2024-07-20 01:23:06,152 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring
|
| 941 |
+
2024-07-20 01:23:06,153 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring
|
| 942 |
+
2024-07-20 01:23:06,153 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring
|
| 943 |
+
2024-07-20 01:23:06,337 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 944 |
+
2024-07-20 01:23:06,874 DEBUG SenderThread:179 [sender.py:send():379] send: config
|
| 945 |
+
2024-07-20 01:23:06,875 DEBUG SenderThread:179 [sender.py:send():379] send: metric
|
| 946 |
+
2024-07-20 01:23:06,875 DEBUG SenderThread:179 [sender.py:send():379] send: metric
|
| 947 |
+
2024-07-20 01:23:06,875 WARNING SenderThread:179 [sender.py:send_metric():1405] Seen metric with glob (shouldn't happen)
|
| 948 |
+
2024-07-20 01:23:06,967 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: log_artifact
|
| 949 |
+
2024-07-20 01:23:06,967 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: log_artifact
|
| 950 |
+
2024-07-20 01:23:06,985 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 951 |
+
2024-07-20 01:23:07,078 INFO SenderThread:179 [sender.py:send_request_log_artifact():1518] logged artifact model-w16tb06h - {'id': 'QXJ0aWZhY3Q6MTAwNDIwNjIwMg==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjI5MzEzNDE4MA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTAwNDIwNjIwMg==', 'versionIndex': 0}}}
|
| 952 |
+
2024-07-20 01:23:07,986 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 953 |
+
2024-07-20 01:23:08,903 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history
|
| 954 |
+
2024-07-20 01:23:08,905 DEBUG SenderThread:179 [sender.py:send():379] send: metric
|
| 955 |
+
2024-07-20 01:23:08,905 DEBUG SenderThread:179 [sender.py:send():379] send: metric
|
| 956 |
+
2024-07-20 01:23:08,905 DEBUG SenderThread:179 [sender.py:send():379] send: metric
|
| 957 |
+
2024-07-20 01:23:08,905 DEBUG SenderThread:179 [sender.py:send():379] send: history
|
| 958 |
+
2024-07-20 01:23:08,906 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 959 |
+
2024-07-20 01:23:08,906 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 960 |
+
2024-07-20 01:23:08,985 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 961 |
+
2024-07-20 01:23:09,285 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json
|
| 962 |
+
2024-07-20 01:23:09,986 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 963 |
+
2024-07-20 01:23:10,247 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history
|
| 964 |
+
2024-07-20 01:23:10,249 DEBUG SenderThread:179 [sender.py:send():379] send: history
|
| 965 |
+
2024-07-20 01:23:10,249 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 966 |
+
2024-07-20 01:23:10,249 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 967 |
+
2024-07-20 01:23:10,285 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json
|
| 968 |
+
2024-07-20 01:23:10,986 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 969 |
+
2024-07-20 01:23:11,576 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history
|
| 970 |
+
2024-07-20 01:23:11,578 DEBUG SenderThread:179 [sender.py:send():379] send: history
|
| 971 |
+
2024-07-20 01:23:11,578 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 972 |
+
2024-07-20 01:23:11,578 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 973 |
+
2024-07-20 01:23:11,579 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 974 |
+
2024-07-20 01:23:11,986 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 975 |
+
2024-07-20 01:23:12,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 976 |
+
2024-07-20 01:23:12,234 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 977 |
+
2024-07-20 01:23:12,286 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json
|
| 978 |
+
2024-07-20 01:23:12,868 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history
|
| 979 |
+
2024-07-20 01:23:12,869 DEBUG SenderThread:179 [sender.py:send():379] send: history
|
| 980 |
+
2024-07-20 01:23:12,870 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 981 |
+
2024-07-20 01:23:12,870 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 982 |
+
2024-07-20 01:23:12,986 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 983 |
+
2024-07-20 01:23:13,287 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json
|
| 984 |
+
2024-07-20 01:23:13,987 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 985 |
+
2024-07-20 01:23:14,247 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history
|
| 986 |
+
2024-07-20 01:23:14,248 DEBUG SenderThread:179 [sender.py:send():379] send: history
|
| 987 |
+
2024-07-20 01:23:14,249 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 988 |
+
2024-07-20 01:23:14,249 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 989 |
+
2024-07-20 01:23:14,287 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json
|
| 990 |
+
2024-07-20 01:23:14,987 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 991 |
+
2024-07-20 01:23:15,711 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history
|
| 992 |
+
2024-07-20 01:23:15,712 DEBUG SenderThread:179 [sender.py:send():379] send: history
|
| 993 |
+
2024-07-20 01:23:15,712 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 994 |
+
2024-07-20 01:23:15,713 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 995 |
+
2024-07-20 01:23:15,987 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 996 |
+
2024-07-20 01:23:16,288 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json
|
| 997 |
+
2024-07-20 01:23:16,713 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 998 |
+
2024-07-20 01:23:16,987 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 999 |
+
2024-07-20 01:23:17,034 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history
|
| 1000 |
+
2024-07-20 01:23:17,035 DEBUG SenderThread:179 [sender.py:send():379] send: history
|
| 1001 |
+
2024-07-20 01:23:17,036 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 1002 |
+
2024-07-20 01:23:17,036 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 1003 |
+
2024-07-20 01:23:17,289 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json
|
| 1004 |
+
2024-07-20 01:23:17,987 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 1005 |
+
2024-07-20 01:23:18,377 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history
|
| 1006 |
+
2024-07-20 01:23:18,378 DEBUG SenderThread:179 [sender.py:send():379] send: history
|
| 1007 |
+
2024-07-20 01:23:18,379 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 1008 |
+
2024-07-20 01:23:18,379 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 1009 |
+
2024-07-20 01:23:18,987 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 1010 |
+
2024-07-20 01:23:19,289 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json
|
| 1011 |
+
2024-07-20 01:23:19,834 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history
|
| 1012 |
+
2024-07-20 01:23:19,836 DEBUG SenderThread:179 [sender.py:send():379] send: history
|
| 1013 |
+
2024-07-20 01:23:19,836 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 1014 |
+
2024-07-20 01:23:19,836 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 1015 |
+
2024-07-20 01:23:19,987 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 1016 |
+
2024-07-20 01:23:20,290 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json
|
| 1017 |
+
2024-07-20 01:23:20,725 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history
|
| 1018 |
+
2024-07-20 01:23:20,727 DEBUG SenderThread:179 [sender.py:send():379] send: history
|
| 1019 |
+
2024-07-20 01:23:20,727 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 1020 |
+
2024-07-20 01:23:20,727 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 1021 |
+
2024-07-20 01:23:20,988 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 1022 |
+
2024-07-20 01:23:21,290 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json
|
| 1023 |
+
2024-07-20 01:23:21,728 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1024 |
+
2024-07-20 01:23:22,470 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 1025 |
+
2024-07-20 01:23:23,470 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 1026 |
+
2024-07-20 01:23:24,473 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 1027 |
+
2024-07-20 01:23:25,473 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 1028 |
+
2024-07-20 01:23:26,729 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1029 |
+
2024-07-20 01:23:26,939 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 1030 |
+
2024-07-20 01:23:31,234 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 1031 |
+
2024-07-20 01:23:31,234 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 1032 |
+
2024-07-20 01:23:31,274 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages
|
| 1033 |
+
2024-07-20 01:23:32,356 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1034 |
+
2024-07-20 01:23:37,072 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1035 |
+
2024-07-20 01:23:37,364 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1036 |
+
2024-07-20 01:23:38,298 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/config.yaml
|
| 1037 |
+
2024-07-20 01:23:42,074 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1038 |
+
2024-07-20 01:23:46,232 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 1039 |
+
2024-07-20 01:23:50,052 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 1040 |
+
2024-07-20 01:23:52,076 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1041 |
+
2024-07-20 01:23:55,225 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1042 |
+
2024-07-20 01:23:57,082 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1043 |
+
2024-07-20 01:24:00,225 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1044 |
+
2024-07-20 01:24:01,232 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 1045 |
+
2024-07-20 01:24:01,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 1046 |
+
2024-07-20 01:24:05,395 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1047 |
+
2024-07-20 01:24:06,153 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop
|
| 1048 |
+
2024-07-20 01:24:06,154 DEBUG SenderThread:179 [sender.py:send():379] send: stats
|
| 1049 |
+
2024-07-20 01:24:07,084 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1050 |
+
2024-07-20 01:24:11,155 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1051 |
+
2024-07-20 01:24:12,085 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1052 |
+
2024-07-20 01:24:16,156 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1053 |
+
2024-07-20 01:24:16,232 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 1054 |
+
2024-07-20 01:24:16,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 1055 |
+
2024-07-20 01:24:16,568 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 1056 |
+
2024-07-20 01:24:16,568 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 1057 |
+
2024-07-20 01:24:16,571 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history
|
| 1058 |
+
2024-07-20 01:24:16,572 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 1059 |
+
2024-07-20 01:24:16,584 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 1060 |
+
2024-07-20 01:24:16,584 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 1061 |
+
2024-07-20 01:24:16,584 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 1062 |
+
2024-07-20 01:24:16,585 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 1063 |
+
2024-07-20 01:24:16,585 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 1064 |
+
2024-07-20 01:24:16,585 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 1065 |
+
2024-07-20 01:24:16,586 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 1066 |
+
2024-07-20 01:24:16,586 DEBUG SenderThread:179 [sender.py:send():379] send: history
|
| 1067 |
+
2024-07-20 01:24:16,586 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record
|
| 1068 |
+
2024-07-20 01:24:16,586 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
|
| 1069 |
+
2024-07-20 01:24:17,312 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json
|
| 1070 |
+
2024-07-20 01:24:21,587 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1071 |
+
2024-07-20 01:24:22,087 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1072 |
+
2024-07-20 01:24:26,588 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1073 |
+
2024-07-20 01:24:27,088 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1074 |
+
2024-07-20 01:24:31,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 1075 |
+
2024-07-20 01:24:31,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 1076 |
+
2024-07-20 01:24:32,396 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1077 |
+
2024-07-20 01:24:36,155 DEBUG SenderThread:179 [sender.py:send():379] send: stats
|
| 1078 |
+
2024-07-20 01:24:37,090 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1079 |
+
2024-07-20 01:24:38,156 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1080 |
+
2024-07-20 01:24:42,091 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1081 |
+
2024-07-20 01:24:43,157 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1082 |
+
2024-07-20 01:24:46,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 1083 |
+
2024-07-20 01:24:46,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 1084 |
+
2024-07-20 01:24:48,393 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1085 |
+
2024-07-20 01:24:52,094 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1086 |
+
2024-07-20 01:24:53,394 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1087 |
+
2024-07-20 01:24:57,095 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1088 |
+
2024-07-20 01:24:58,395 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1089 |
+
2024-07-20 01:25:01,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 1090 |
+
2024-07-20 01:25:01,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 1091 |
+
2024-07-20 01:25:04,381 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1092 |
+
2024-07-20 01:25:06,156 DEBUG SenderThread:179 [sender.py:send():379] send: stats
|
| 1093 |
+
2024-07-20 01:25:07,098 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1094 |
+
2024-07-20 01:25:10,157 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1095 |
+
2024-07-20 01:25:12,099 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1096 |
+
2024-07-20 01:25:15,158 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1097 |
+
2024-07-20 01:25:16,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 1098 |
+
2024-07-20 01:25:16,234 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 1099 |
+
2024-07-20 01:25:20,394 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1100 |
+
2024-07-20 01:25:22,101 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1101 |
+
2024-07-20 01:25:25,395 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1102 |
+
2024-07-20 01:25:27,102 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1103 |
+
2024-07-20 01:25:30,396 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1104 |
+
2024-07-20 01:25:31,234 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 1105 |
+
2024-07-20 01:25:31,234 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 1106 |
+
2024-07-20 01:25:36,157 DEBUG SenderThread:179 [sender.py:send():379] send: stats
|
| 1107 |
+
2024-07-20 01:25:36,157 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1108 |
+
2024-07-20 01:25:37,104 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1109 |
+
2024-07-20 01:25:41,158 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1110 |
+
2024-07-20 01:25:42,106 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1111 |
+
2024-07-20 01:25:46,159 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1112 |
+
2024-07-20 01:25:46,234 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 1113 |
+
2024-07-20 01:25:46,234 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 1114 |
+
2024-07-20 01:25:51,327 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1115 |
+
2024-07-20 01:25:52,108 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1116 |
+
2024-07-20 01:25:56,328 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1117 |
+
2024-07-20 01:25:57,110 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1118 |
+
2024-07-20 01:26:01,234 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 1119 |
+
2024-07-20 01:26:01,234 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 1120 |
+
2024-07-20 01:26:02,310 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1121 |
+
2024-07-20 01:26:06,157 DEBUG SenderThread:179 [sender.py:send():379] send: stats
|
| 1122 |
+
2024-07-20 01:26:07,112 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1123 |
+
2024-07-20 01:26:08,158 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1124 |
+
2024-07-20 01:26:12,113 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
| 1125 |
+
2024-07-20 01:26:13,159 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1126 |
+
2024-07-20 01:26:16,234 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status
|
| 1127 |
+
2024-07-20 01:26:16,235 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status
|
| 1128 |
+
2024-07-20 01:26:18,386 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report
|
| 1129 |
+
2024-07-20 01:26:22,115 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive
|
wandb/run-20240720_011510-w16tb06h/logs/debug.log
CHANGED
|
@@ -45,3 +45,25 @@ config: {}
|
|
| 45 |
2024-07-20 01:15:55,259 INFO MainThread:34 [wandb_init.py:_pause_backend():440] pausing backend
|
| 46 |
2024-07-20 01:15:58,284 INFO MainThread:34 [wandb_init.py:_resume_backend():445] resuming backend
|
| 47 |
2024-07-20 01:15:58,845 INFO MainThread:34 [wandb_run.py:_config_callback():1382] config_cb None None {'output_dir': './', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'eval_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 10, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 1e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 1, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './runs/Jul20_01-15-42_3bc9163d6d69', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 10, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 500, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': 'VQA2', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': None, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
2024-07-20 01:15:55,259 INFO MainThread:34 [wandb_init.py:_pause_backend():440] pausing backend
|
| 46 |
2024-07-20 01:15:58,284 INFO MainThread:34 [wandb_init.py:_resume_backend():445] resuming backend
|
| 47 |
2024-07-20 01:15:58,845 INFO MainThread:34 [wandb_run.py:_config_callback():1382] config_cb None None {'output_dir': './', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'eval_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 10, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 1e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 1, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './runs/Jul20_01-15-42_3bc9163d6d69', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 10, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 500, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': 'VQA2', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': None, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False}
|
| 48 |
+
2024-07-20 01:18:50,373 INFO MainThread:34 [jupyter.py:save_ipynb():372] not saving jupyter notebook
|
| 49 |
+
2024-07-20 01:18:50,373 INFO MainThread:34 [wandb_init.py:_pause_backend():440] pausing backend
|
| 50 |
+
2024-07-20 01:21:08,290 INFO MainThread:34 [wandb_init.py:_resume_backend():445] resuming backend
|
| 51 |
+
2024-07-20 01:21:09,256 INFO MainThread:34 [jupyter.py:save_ipynb():372] not saving jupyter notebook
|
| 52 |
+
2024-07-20 01:21:09,256 INFO MainThread:34 [wandb_init.py:_pause_backend():440] pausing backend
|
| 53 |
+
2024-07-20 01:22:59,714 INFO MainThread:34 [wandb_init.py:_resume_backend():445] resuming backend
|
| 54 |
+
2024-07-20 01:22:59,740 INFO MainThread:34 [jupyter.py:save_ipynb():372] not saving jupyter notebook
|
| 55 |
+
2024-07-20 01:22:59,740 INFO MainThread:34 [wandb_init.py:_pause_backend():440] pausing backend
|
| 56 |
+
2024-07-20 01:23:00,462 INFO MainThread:34 [wandb_init.py:_resume_backend():445] resuming backend
|
| 57 |
+
2024-07-20 01:23:00,464 INFO MainThread:34 [jupyter.py:save_ipynb():372] not saving jupyter notebook
|
| 58 |
+
2024-07-20 01:23:00,464 INFO MainThread:34 [wandb_init.py:_pause_backend():440] pausing backend
|
| 59 |
+
2024-07-20 01:23:01,882 INFO MainThread:34 [wandb_init.py:_resume_backend():445] resuming backend
|
| 60 |
+
2024-07-20 01:23:01,885 INFO MainThread:34 [jupyter.py:save_ipynb():372] not saving jupyter notebook
|
| 61 |
+
2024-07-20 01:23:01,885 INFO MainThread:34 [wandb_init.py:_pause_backend():440] pausing backend
|
| 62 |
+
2024-07-20 01:23:02,979 INFO MainThread:34 [wandb_init.py:_resume_backend():445] resuming backend
|
| 63 |
+
2024-07-20 01:23:02,980 INFO MainThread:34 [jupyter.py:save_ipynb():372] not saving jupyter notebook
|
| 64 |
+
2024-07-20 01:23:02,981 INFO MainThread:34 [wandb_init.py:_pause_backend():440] pausing backend
|
| 65 |
+
2024-07-20 01:23:03,184 INFO MainThread:34 [wandb_init.py:_resume_backend():445] resuming backend
|
| 66 |
+
2024-07-20 01:23:03,326 INFO MainThread:34 [jupyter.py:save_ipynb():372] not saving jupyter notebook
|
| 67 |
+
2024-07-20 01:23:03,327 INFO MainThread:34 [wandb_init.py:_pause_backend():440] pausing backend
|
| 68 |
+
2024-07-20 01:23:06,148 INFO MainThread:34 [wandb_init.py:_resume_backend():445] resuming backend
|
| 69 |
+
2024-07-20 01:23:06,870 INFO MainThread:34 [wandb_run.py:_config_callback():1382] config_cb None None {'output_dir': './', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'eval_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 1e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 1, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './runs/Jul20_01-22-59_3bc9163d6d69', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 500, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': 'VQA2', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': None, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False}
|
wandb/run-20240720_011510-w16tb06h/run-w16tb06h.wandb
CHANGED
|
Binary files a/wandb/run-20240720_011510-w16tb06h/run-w16tb06h.wandb and b/wandb/run-20240720_011510-w16tb06h/run-w16tb06h.wandb differ
|
|
|