diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" --- "a/wandb/debug-internal.log" +++ "b/wandb/debug-internal.log" @@ -1,420 +1,661 @@ -2024-02-19 19:09:40,372 INFO StreamThr :85 [internal.py:wandb_internal():86] W&B internal server running at pid: 85, started at: 2024-02-19 19:09:40.371490 -2024-02-19 19:09:40,373 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status -2024-02-19 19:09:40,376 INFO WriterThread:85 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240219_190940-ztflz27u/run-ztflz27u.wandb -2024-02-19 19:09:40,377 DEBUG SenderThread:85 [sender.py:send():382] send: header -2024-02-19 19:09:40,379 DEBUG SenderThread:85 [sender.py:send():382] send: run -2024-02-19 19:09:40,755 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: check_version -2024-02-19 19:09:40,756 INFO SenderThread:85 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240219_190940-ztflz27u/files -2024-02-19 19:09:40,756 INFO SenderThread:85 [sender.py:_start_run_threads():1136] run started: ztflz27u with start time 1708369780.372554 -2024-02-19 19:09:40,756 DEBUG SenderThread:85 [sender.py:send_request():409] send_request: check_version -2024-02-19 19:09:40,815 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: run_start -2024-02-19 19:09:40,827 DEBUG HandlerThread:85 [system_info.py:__init__():27] System info init -2024-02-19 19:09:40,827 DEBUG HandlerThread:85 [system_info.py:__init__():42] System info init done -2024-02-19 19:09:40,827 INFO HandlerThread:85 [system_monitor.py:start():194] Starting system monitor -2024-02-19 19:09:40,827 INFO SystemMonitor:85 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-02-19 19:09:40,827 INFO HandlerThread:85 [system_monitor.py:probe():214] Collecting system info -2024-02-19 19:09:40,828 INFO SystemMonitor:85 [interfaces.py:start():190] Started cpu monitoring -2024-02-19 19:09:40,828 INFO SystemMonitor:85 [interfaces.py:start():190] Started disk monitoring -2024-02-19 19:09:40,829 INFO SystemMonitor:85 [interfaces.py:start():190] Started gpu monitoring -2024-02-19 19:09:40,830 INFO SystemMonitor:85 [interfaces.py:start():190] Started memory monitoring -2024-02-19 19:09:40,831 INFO SystemMonitor:85 [interfaces.py:start():190] Started network monitoring -2024-02-19 19:09:40,841 DEBUG HandlerThread:85 [system_info.py:probe():151] Probing system -2024-02-19 19:09:40,843 DEBUG HandlerThread:85 [gitlib.py:_init_repo():56] git repository is invalid -2024-02-19 19:09:40,843 DEBUG HandlerThread:85 [system_info.py:probe():199] Probing system done -2024-02-19 19:09:40,843 DEBUG HandlerThread:85 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-02-19T19:09:40.841087', 'startedAt': '2024-02-19T19:09:40.366125', 'docker': None, 'cuda': None, 'args': ('-f', '/tmp/tmpcjzmy8d6.json', '--HistoryManager.hist_file=:memory:'), 'state': 'running', 'program': '', 'codePathLocal': None, 'host': 'ed7fe84f9951', 'username': 'root', 'executable': '/opt/conda/bin/python', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.176, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.176, 'min': 0.0, 'max': 0.0}, {'current': 2000.176, 'min': 0.0, 'max': 0.0}, {'current': 2000.176, 'min': 0.0, 'max': 0.0}, {'current': 2000.176, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5432.319198608398}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.357559204101562}} -2024-02-19 19:09:40,843 INFO HandlerThread:85 [system_monitor.py:probe():224] Finished collecting system info -2024-02-19 19:09:40,843 INFO HandlerThread:85 [system_monitor.py:probe():227] Publishing system info -2024-02-19 19:09:40,843 DEBUG HandlerThread:85 [system_info.py:_save_conda():208] Saving list of conda packages installed into the current environment -2024-02-19 19:09:41,758 INFO Thread-12 :85 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240219_190940-ztflz27u/files/conda-environment.yaml -2024-02-19 19:10:35,554 DEBUG HandlerThread:85 [system_info.py:_save_conda():220] Saving conda packages done -2024-02-19 19:10:35,554 INFO HandlerThread:85 [system_monitor.py:probe():229] Finished publishing system info -2024-02-19 19:10:35,563 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:10:35,564 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:10:35,564 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:10:35,564 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:10:35,564 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:10:35,564 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:10:35,565 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:10:35,565 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:10:35,565 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:10:35,565 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:10:35,565 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:10:35,565 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: python_packages -2024-02-19 19:10:35,565 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: stop_status -2024-02-19 19:10:35,565 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-19 19:10:35,566 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:10:35,566 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:10:35,566 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:10:35,566 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:10:35,566 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:10:35,566 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:10:35,566 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:10:35,566 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:10:35,567 DEBUG SenderThread:85 [sender.py:send_request():409] send_request: python_packages -2024-02-19 19:10:35,569 DEBUG SenderThread:85 [sender.py:send():382] send: telemetry -2024-02-19 19:10:35,569 DEBUG SenderThread:85 [sender.py:send():382] send: config -2024-02-19 19:10:35,570 DEBUG SenderThread:85 [sender.py:send():382] send: metric -2024-02-19 19:10:35,571 DEBUG SenderThread:85 [sender.py:send():382] send: telemetry -2024-02-19 19:10:35,571 DEBUG SenderThread:85 [sender.py:send():382] send: metric -2024-02-19 19:10:35,571 WARNING SenderThread:85 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) -2024-02-19 19:10:35,571 DEBUG SenderThread:85 [sender.py:send():382] send: telemetry -2024-02-19 19:10:35,571 DEBUG SenderThread:85 [sender.py:send_request():409] send_request: stop_status -2024-02-19 19:10:35,719 DEBUG SenderThread:85 [sender.py:send():382] send: files -2024-02-19 19:10:35,720 INFO SenderThread:85 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now -2024-02-19 19:10:35,774 INFO Thread-12 :85 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240219_190940-ztflz27u/files/conda-environment.yaml -2024-02-19 19:10:35,774 INFO Thread-12 :85 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240219_190940-ztflz27u/files/requirements.txt -2024-02-19 19:10:35,775 INFO Thread-12 :85 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240219_190940-ztflz27u/files/wandb-metadata.json -2024-02-19 19:10:36,226 INFO wandb-upload_0:85 [upload_job.py:push():131] Uploaded file /tmp/tmppbeiotzkwandb/q1ioc5b0-wandb-metadata.json -2024-02-19 19:10:36,720 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:10:38,595 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:10:40,831 DEBUG SystemMonitor:85 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-02-19 19:10:40,833 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:10:41,838 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:10:42,777 INFO Thread-12 :85 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240219_190940-ztflz27u/files/config.yaml -2024-02-19 19:10:43,736 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:10:47,126 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:10:49,360 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:10:52,127 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:10:56,197 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:10:57,128 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:11:01,443 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:11:02,129 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:11:07,073 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:11:07,130 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:11:10,833 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:11:12,737 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:11:12,834 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:11:17,835 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:11:17,862 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:11:22,836 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:11:23,690 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:11:27,837 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:11:28,737 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:11:32,838 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:11:35,288 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:11:37,839 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:11:40,834 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:11:41,115 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:11:43,835 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:11:46,281 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:11:48,836 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:11:51,892 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:11:53,837 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:11:57,737 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:11:58,838 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:12:03,839 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:12:04,584 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:12:08,840 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:12:10,670 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:12:10,835 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:12:14,836 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:12:16,279 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:12:19,837 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:12:21,369 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:12:24,838 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:12:26,737 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:12:26,737 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:12:29,839 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:12:31,808 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:12:34,840 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:12:37,662 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:12:39,840 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:12:40,836 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:12:44,414 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:12:44,414 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:12:45,838 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:12:50,226 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:12:50,838 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:12:55,839 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:12:56,075 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:13:00,840 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:13:01,388 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:13:01,388 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:13:05,841 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:13:08,068 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:13:10,837 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:13:11,838 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:13:13,263 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:13:16,839 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:13:19,188 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:13:21,840 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:13:24,812 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:13:26,840 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:13:30,012 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:13:31,841 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:13:35,918 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:13:36,842 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:13:40,838 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:13:42,424 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:13:42,839 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:13:47,840 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:13:48,493 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:13:52,841 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:13:54,329 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:13:57,842 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:14:00,013 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:14:02,843 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:14:05,693 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:14:07,843 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:14:10,839 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:14:12,463 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:14:13,840 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:14:17,640 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:14:18,841 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:14:23,257 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:14:23,842 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:14:28,264 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:14:28,843 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:14:33,844 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:14:33,997 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:14:38,844 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:14:40,790 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:14:40,840 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:14:44,841 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:14:45,878 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:14:49,842 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:14:51,506 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:14:54,843 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:14:57,391 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:14:59,844 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:15:02,594 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:15:04,844 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:15:08,228 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:15:09,845 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:15:10,840 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:15:13,266 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:15:15,842 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:15:19,939 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:15:20,842 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:15:25,783 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:15:25,843 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:15:30,844 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:15:31,659 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:15:31,659 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:15:35,845 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:15:37,596 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:15:40,841 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:15:41,842 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:15:44,378 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:15:44,625 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:15:46,843 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:15:50,453 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:15:51,843 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:15:56,309 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:15:56,844 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:16:01,659 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:16:01,845 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:16:06,846 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:16:06,886 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:16:10,842 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:16:12,765 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:16:12,843 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:16:17,844 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:16:19,551 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:16:19,551 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:16:22,845 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:16:25,399 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:16:27,845 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:16:30,504 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:16:32,846 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:16:36,174 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:16:37,847 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:16:40,843 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:16:42,993 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:16:43,844 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:16:48,845 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:16:48,937 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:16:53,846 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:16:54,842 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:16:58,847 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:17:00,660 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:17:03,847 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:17:06,492 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:17:08,848 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:17:10,844 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:17:13,329 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:17:14,845 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:17:18,409 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:17:19,846 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:17:24,290 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:17:24,847 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:17:29,379 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:17:29,847 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:17:34,848 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:17:35,611 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:17:39,849 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:17:40,845 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:17:42,319 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:17:45,846 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:17:48,362 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:17:50,847 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:17:53,381 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:17:55,848 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:17:58,430 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:18:00,848 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:18:03,481 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:18:05,849 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:18:09,048 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:18:10,845 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:18:11,846 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:18:14,696 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:18:14,947 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:18:14,947 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:18:16,847 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:18:20,612 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:18:21,848 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:18:26,238 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:18:26,849 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:18:31,850 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:18:32,046 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:18:32,046 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:18:36,850 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:18:37,741 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:18:40,846 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:18:42,847 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:18:45,389 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:18:45,639 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:18:45,639 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:18:47,848 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:18:51,273 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:18:52,849 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:18:56,960 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:18:57,849 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:19:02,781 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:19:02,781 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:19:02,850 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:19:07,852 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:19:08,441 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:19:10,847 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:19:13,848 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:19:14,311 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:19:18,849 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:19:20,317 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:19:20,589 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:19:23,850 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:19:26,490 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:19:28,851 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:19:32,345 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:19:32,598 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:19:32,599 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:19:33,852 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:19:38,853 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:19:39,058 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:19:40,848 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:19:44,849 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:19:45,820 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:19:49,850 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:19:51,489 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:19:51,740 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:19:54,851 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:19:58,168 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:19:59,852 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:20:04,201 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:20:04,202 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:20:04,853 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:20:09,854 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:20:10,551 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:20:10,849 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:20:15,850 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:20:16,358 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:20:20,851 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:20:22,059 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:20:25,852 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:20:27,919 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:20:30,853 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:20:33,618 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:20:33,865 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:20:35,854 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:20:39,430 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:20:40,850 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:20:41,851 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:20:45,306 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:20:46,852 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:20:50,868 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:20:50,868 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:20:51,852 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:20:56,554 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:20:56,853 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:21:01,854 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:21:03,268 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:21:04,186 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:21:06,855 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:21:10,043 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:21:10,851 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:21:12,852 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:21:15,876 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:21:17,853 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:21:21,817 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:21:21,817 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:21:22,854 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:21:27,554 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:21:27,855 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:21:32,855 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:21:33,437 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:21:37,856 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:21:39,240 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:21:39,489 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:21:40,852 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:21:43,853 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:21:45,147 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:21:48,854 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:21:50,846 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:21:51,096 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:21:53,854 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:21:56,752 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:21:58,856 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:22:02,956 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:22:03,856 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:22:08,857 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:22:08,978 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:22:10,853 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:22:14,658 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:22:14,854 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:22:19,855 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:22:20,482 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:22:24,856 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:22:26,115 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:22:26,364 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:22:29,857 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:22:32,006 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:22:34,857 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:22:37,761 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:22:39,858 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:22:40,854 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:22:43,611 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:22:45,855 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:22:49,429 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:22:50,856 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:22:54,824 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:22:55,857 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:22:59,032 INFO Thread-12 :85 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240219_190940-ztflz27u/files/output.log -2024-02-19 19:22:59,851 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:23:01,032 INFO Thread-12 :85 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240219_190940-ztflz27u/files/output.log -2024-02-19 19:23:01,539 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:23:04,883 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:23:06,540 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:23:06,835 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: partial_history -2024-02-19 19:23:06,839 DEBUG SenderThread:85 [sender.py:send():382] send: metric -2024-02-19 19:23:06,840 DEBUG SenderThread:85 [sender.py:send():382] send: metric -2024-02-19 19:23:06,840 DEBUG SenderThread:85 [sender.py:send():382] send: metric -2024-02-19 19:23:06,840 DEBUG SenderThread:85 [sender.py:send():382] send: metric -2024-02-19 19:23:06,840 DEBUG SenderThread:85 [sender.py:send():382] send: metric -2024-02-19 19:23:06,840 DEBUG SenderThread:85 [sender.py:send():382] send: metric -2024-02-19 19:23:06,840 DEBUG SenderThread:85 [sender.py:send():382] send: metric -2024-02-19 19:23:06,840 DEBUG SenderThread:85 [sender.py:send():382] send: metric -2024-02-19 19:23:06,840 DEBUG SenderThread:85 [sender.py:send():382] send: metric -2024-02-19 19:23:06,840 DEBUG SenderThread:85 [sender.py:send():382] send: metric -2024-02-19 19:23:06,840 DEBUG SenderThread:85 [sender.py:send():382] send: history -2024-02-19 19:23:06,841 DEBUG SenderThread:85 [sender.py:send_request():409] send_request: summary_record -2024-02-19 19:23:06,841 INFO SenderThread:85 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-02-19 19:23:07,035 INFO Thread-12 :85 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240219_190940-ztflz27u/files/wandb-summary.json -2024-02-19 19:23:10,315 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:23:10,316 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:23:10,855 DEBUG SenderThread:85 [sender.py:send():382] send: stats -2024-02-19 19:23:11,856 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:23:16,043 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:23:16,856 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:23:21,108 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:23:21,857 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report -2024-02-19 19:23:26,299 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: keepalive -2024-02-19 19:23:26,858 DEBUG HandlerThread:85 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:17:30,951 INFO StreamThr :235 [internal.py:wandb_internal():86] W&B internal server running at pid: 235, started at: 2024-02-21 17:17:30.951077 +2024-02-21 17:17:30,955 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status +2024-02-21 17:17:31,265 INFO WriterThread:235 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240221_171730-9wbsk0df/run-9wbsk0df.wandb +2024-02-21 17:17:31,265 DEBUG SenderThread:235 [sender.py:send():382] send: header +2024-02-21 17:17:31,268 DEBUG SenderThread:235 [sender.py:send():382] send: run +2024-02-21 17:17:31,450 INFO SenderThread:235 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240221_171730-9wbsk0df/files +2024-02-21 17:17:31,450 INFO SenderThread:235 [sender.py:_start_run_threads():1136] run started: 9wbsk0df with start time 1708535850.952207 +2024-02-21 17:17:31,458 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: check_version +2024-02-21 17:17:31,458 DEBUG SenderThread:235 [sender.py:send_request():409] send_request: check_version +2024-02-21 17:17:31,547 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: run_start +2024-02-21 17:17:31,559 DEBUG HandlerThread:235 [system_info.py:__init__():27] System info init +2024-02-21 17:17:31,559 DEBUG HandlerThread:235 [system_info.py:__init__():42] System info init done +2024-02-21 17:17:31,559 INFO HandlerThread:235 [system_monitor.py:start():194] Starting system monitor +2024-02-21 17:17:31,559 INFO SystemMonitor:235 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-21 17:17:31,560 INFO SystemMonitor:235 [interfaces.py:start():190] Started cpu monitoring +2024-02-21 17:17:31,561 INFO SystemMonitor:235 [interfaces.py:start():190] Started disk monitoring +2024-02-21 17:17:31,561 INFO SystemMonitor:235 [interfaces.py:start():190] Started gpu monitoring +2024-02-21 17:17:31,562 INFO HandlerThread:235 [system_monitor.py:probe():214] Collecting system info +2024-02-21 17:17:31,562 INFO SystemMonitor:235 [interfaces.py:start():190] Started memory monitoring +2024-02-21 17:17:31,563 INFO SystemMonitor:235 [interfaces.py:start():190] Started network monitoring +2024-02-21 17:17:31,575 DEBUG HandlerThread:235 [system_info.py:probe():151] Probing system +2024-02-21 17:17:31,578 DEBUG HandlerThread:235 [gitlib.py:_init_repo():56] git repository is invalid +2024-02-21 17:17:31,578 DEBUG HandlerThread:235 [system_info.py:probe():199] Probing system done +2024-02-21 17:17:31,578 DEBUG HandlerThread:235 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-02-21T17:17:31.575117', 'startedAt': '2024-02-21T17:17:30.944813', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': '047c4dfc8464', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.14, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.14, 'min': 0.0, 'max': 0.0}, {'current': 2000.14, 'min': 0.0, 'max': 0.0}, {'current': 2000.14, 'min': 0.0, 'max': 0.0}, {'current': 2000.14, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5432.354721069336}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.357559204101562}} +2024-02-21 17:17:31,578 INFO HandlerThread:235 [system_monitor.py:probe():224] Finished collecting system info +2024-02-21 17:17:31,578 INFO HandlerThread:235 [system_monitor.py:probe():227] Publishing system info +2024-02-21 17:17:31,578 DEBUG HandlerThread:235 [system_info.py:_save_conda():208] Saving list of conda packages installed into the current environment +2024-02-21 17:17:32,452 INFO Thread-12 :235 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240221_171730-9wbsk0df/files/conda-environment.yaml +2024-02-21 17:18:26,461 DEBUG HandlerThread:235 [system_info.py:_save_conda():220] Saving conda packages done +2024-02-21 17:18:26,461 INFO HandlerThread:235 [system_monitor.py:probe():229] Finished publishing system info +2024-02-21 17:18:26,467 INFO Thread-12 :235 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240221_171730-9wbsk0df/files/conda-environment.yaml +2024-02-21 17:18:26,468 INFO Thread-12 :235 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240221_171730-9wbsk0df/files/wandb-metadata.json +2024-02-21 17:18:26,471 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:18:26,471 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:18:26,471 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:18:26,471 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:18:26,471 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:18:26,471 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:18:26,471 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:18:26,471 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:18:26,471 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:18:26,472 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:18:26,472 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:18:26,472 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: python_packages +2024-02-21 17:18:26,472 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: stop_status +2024-02-21 17:18:26,473 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-21 17:18:26,473 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: pause +2024-02-21 17:18:26,473 INFO HandlerThread:235 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-21 17:18:26,473 INFO HandlerThread:235 [system_monitor.py:finish():203] Stopping system monitor +2024-02-21 17:18:26,474 DEBUG SystemMonitor:235 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-21 17:18:26,474 DEBUG SystemMonitor:235 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-21 17:18:26,474 DEBUG SystemMonitor:235 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-21 17:18:26,474 DEBUG SenderThread:235 [sender.py:send_request():409] send_request: python_packages +2024-02-21 17:18:26,474 INFO HandlerThread:235 [interfaces.py:finish():202] Joined cpu monitor +2024-02-21 17:18:26,477 INFO HandlerThread:235 [interfaces.py:finish():202] Joined disk monitor +2024-02-21 17:18:26,477 DEBUG SenderThread:235 [sender.py:send_request():409] send_request: stop_status +2024-02-21 17:18:26,490 INFO HandlerThread:235 [interfaces.py:finish():202] Joined gpu monitor +2024-02-21 17:18:26,490 INFO HandlerThread:235 [interfaces.py:finish():202] Joined memory monitor +2024-02-21 17:18:26,490 INFO HandlerThread:235 [interfaces.py:finish():202] Joined network monitor +2024-02-21 17:18:26,490 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:18:26,490 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:18:26,490 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:18:26,490 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:18:26,490 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:18:26,491 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:18:26,491 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:18:26,491 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:18:26,554 DEBUG SenderThread:235 [sender.py:send():382] send: telemetry +2024-02-21 17:18:26,555 DEBUG SenderThread:235 [sender.py:send():382] send: config +2024-02-21 17:18:26,555 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:18:26,556 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:18:26,556 DEBUG SenderThread:235 [sender.py:send():382] send: telemetry +2024-02-21 17:18:26,556 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:18:26,557 WARNING SenderThread:235 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) +2024-02-21 17:18:26,557 DEBUG SenderThread:235 [sender.py:send():382] send: telemetry +2024-02-21 17:18:26,557 DEBUG SenderThread:235 [sender.py:send():382] send: files +2024-02-21 17:18:26,557 INFO SenderThread:235 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-02-21 17:18:26,557 DEBUG SenderThread:235 [sender.py:send():382] send: stats +2024-02-21 17:18:26,766 INFO wandb-upload_0:235 [upload_job.py:push():131] Uploaded file /tmp/tmpgge_bdx8wandb/lv5ylz55-wandb-metadata.json +2024-02-21 17:18:27,468 INFO Thread-12 :235 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240221_171730-9wbsk0df/files/requirements.txt +2024-02-21 17:18:27,590 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:18:31,563 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:18:32,470 INFO Thread-12 :235 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240221_171730-9wbsk0df/files/config.yaml +2024-02-21 17:18:32,708 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:18:36,379 DEBUG SenderThread:235 [sender.py:send():382] send: telemetry +2024-02-21 17:18:36,471 INFO Thread-12 :235 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240221_171730-9wbsk0df/files/output.log +2024-02-21 17:18:37,380 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:18:37,485 DEBUG SenderThread:235 [sender.py:send():382] send: telemetry +2024-02-21 17:18:37,526 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: resume +2024-02-21 17:18:37,526 INFO HandlerThread:235 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-21 17:18:37,526 INFO HandlerThread:235 [system_monitor.py:start():194] Starting system monitor +2024-02-21 17:18:37,526 INFO SystemMonitor:235 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-21 17:18:37,528 INFO SystemMonitor:235 [interfaces.py:start():190] Started cpu monitoring +2024-02-21 17:18:37,529 INFO SystemMonitor:235 [interfaces.py:start():190] Started disk monitoring +2024-02-21 17:18:37,530 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: pause +2024-02-21 17:18:37,530 INFO HandlerThread:235 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-21 17:18:37,530 INFO HandlerThread:235 [system_monitor.py:finish():203] Stopping system monitor +2024-02-21 17:18:37,531 INFO HandlerThread:235 [interfaces.py:finish():202] Joined cpu monitor +2024-02-21 17:18:37,531 INFO SystemMonitor:235 [interfaces.py:start():190] Started gpu monitoring +2024-02-21 17:18:37,531 DEBUG SystemMonitor:235 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-21 17:18:37,531 DEBUG SystemMonitor:235 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-21 17:18:37,531 DEBUG SystemMonitor:235 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-21 17:18:37,533 INFO HandlerThread:235 [interfaces.py:finish():202] Joined disk monitor +2024-02-21 17:18:37,534 INFO HandlerThread:235 [interfaces.py:finish():202] Joined gpu monitor +2024-02-21 17:18:37,534 DEBUG SenderThread:235 [sender.py:send():382] send: stats +2024-02-21 17:18:37,725 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:18:38,472 INFO Thread-12 :235 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240221_171730-9wbsk0df/files/output.log +2024-02-21 17:18:40,523 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: resume +2024-02-21 17:18:40,523 INFO HandlerThread:235 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-21 17:18:40,523 INFO HandlerThread:235 [system_monitor.py:start():194] Starting system monitor +2024-02-21 17:18:40,524 INFO SystemMonitor:235 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-21 17:18:40,524 INFO SystemMonitor:235 [interfaces.py:start():190] Started cpu monitoring +2024-02-21 17:18:40,525 INFO SystemMonitor:235 [interfaces.py:start():190] Started disk monitoring +2024-02-21 17:18:40,526 INFO SystemMonitor:235 [interfaces.py:start():190] Started gpu monitoring +2024-02-21 17:18:40,528 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: pause +2024-02-21 17:18:40,528 INFO HandlerThread:235 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-21 17:18:40,529 INFO SystemMonitor:235 [interfaces.py:start():190] Started memory monitoring +2024-02-21 17:18:40,529 INFO HandlerThread:235 [system_monitor.py:finish():203] Stopping system monitor +2024-02-21 17:18:40,530 INFO SystemMonitor:235 [interfaces.py:start():190] Started network monitoring +2024-02-21 17:18:40,530 INFO HandlerThread:235 [interfaces.py:finish():202] Joined cpu monitor +2024-02-21 17:18:40,530 DEBUG SystemMonitor:235 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-21 17:18:40,531 DEBUG SystemMonitor:235 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-21 17:18:40,531 DEBUG SystemMonitor:235 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-21 17:18:40,531 INFO HandlerThread:235 [interfaces.py:finish():202] Joined disk monitor +2024-02-21 17:18:40,553 INFO HandlerThread:235 [interfaces.py:finish():202] Joined gpu monitor +2024-02-21 17:18:40,553 INFO HandlerThread:235 [interfaces.py:finish():202] Joined memory monitor +2024-02-21 17:18:40,553 INFO HandlerThread:235 [interfaces.py:finish():202] Joined network monitor +2024-02-21 17:18:40,553 DEBUG SenderThread:235 [sender.py:send():382] send: stats +2024-02-21 17:18:42,554 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:18:42,575 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: resume +2024-02-21 17:18:42,575 INFO HandlerThread:235 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-21 17:18:42,575 INFO HandlerThread:235 [system_monitor.py:start():194] Starting system monitor +2024-02-21 17:18:42,575 INFO SystemMonitor:235 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-21 17:18:42,576 INFO SystemMonitor:235 [interfaces.py:start():190] Started cpu monitoring +2024-02-21 17:18:42,577 INFO SystemMonitor:235 [interfaces.py:start():190] Started disk monitoring +2024-02-21 17:18:42,580 INFO SystemMonitor:235 [interfaces.py:start():190] Started gpu monitoring +2024-02-21 17:18:42,580 INFO SystemMonitor:235 [interfaces.py:start():190] Started memory monitoring +2024-02-21 17:18:42,581 INFO SystemMonitor:235 [interfaces.py:start():190] Started network monitoring +2024-02-21 17:18:43,598 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:18:43,617 DEBUG SenderThread:235 [sender.py:send():382] send: config +2024-02-21 17:18:43,619 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:18:43,619 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:18:43,619 WARNING SenderThread:235 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) +2024-02-21 17:18:44,545 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: pause +2024-02-21 17:18:44,546 INFO HandlerThread:235 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-21 17:18:44,546 INFO HandlerThread:235 [system_monitor.py:finish():203] Stopping system monitor +2024-02-21 17:18:44,546 DEBUG SystemMonitor:235 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-21 17:18:44,547 INFO HandlerThread:235 [interfaces.py:finish():202] Joined cpu monitor +2024-02-21 17:18:44,547 DEBUG SystemMonitor:235 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-21 17:18:44,547 INFO HandlerThread:235 [interfaces.py:finish():202] Joined disk monitor +2024-02-21 17:18:44,547 DEBUG SystemMonitor:235 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-21 17:18:44,559 INFO HandlerThread:235 [interfaces.py:finish():202] Joined gpu monitor +2024-02-21 17:18:44,559 INFO HandlerThread:235 [interfaces.py:finish():202] Joined memory monitor +2024-02-21 17:18:44,560 INFO HandlerThread:235 [interfaces.py:finish():202] Joined network monitor +2024-02-21 17:18:44,560 DEBUG SenderThread:235 [sender.py:send():382] send: stats +2024-02-21 17:18:46,421 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: resume +2024-02-21 17:18:46,422 INFO HandlerThread:235 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-21 17:18:46,422 INFO HandlerThread:235 [system_monitor.py:start():194] Starting system monitor +2024-02-21 17:18:46,422 INFO SystemMonitor:235 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-21 17:18:46,423 INFO SystemMonitor:235 [interfaces.py:start():190] Started cpu monitoring +2024-02-21 17:18:46,423 INFO SystemMonitor:235 [interfaces.py:start():190] Started disk monitoring +2024-02-21 17:18:46,424 INFO SystemMonitor:235 [interfaces.py:start():190] Started gpu monitoring +2024-02-21 17:18:46,425 INFO SystemMonitor:235 [interfaces.py:start():190] Started memory monitoring +2024-02-21 17:18:46,426 INFO SystemMonitor:235 [interfaces.py:start():190] Started network monitoring +2024-02-21 17:18:47,561 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:18:47,681 DEBUG SenderThread:235 [sender.py:send():382] send: config +2024-02-21 17:18:47,682 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:18:47,682 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:18:47,682 WARNING SenderThread:235 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) +2024-02-21 17:18:48,054 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: pause +2024-02-21 17:18:48,054 INFO HandlerThread:235 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-21 17:18:48,054 INFO HandlerThread:235 [system_monitor.py:finish():203] Stopping system monitor +2024-02-21 17:18:48,055 DEBUG SystemMonitor:235 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-21 17:18:48,055 INFO HandlerThread:235 [interfaces.py:finish():202] Joined cpu monitor +2024-02-21 17:18:48,055 DEBUG SystemMonitor:235 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-21 17:18:48,056 DEBUG SystemMonitor:235 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-21 17:18:48,056 INFO HandlerThread:235 [interfaces.py:finish():202] Joined disk monitor +2024-02-21 17:18:48,068 INFO HandlerThread:235 [interfaces.py:finish():202] Joined gpu monitor +2024-02-21 17:18:48,068 INFO HandlerThread:235 [interfaces.py:finish():202] Joined memory monitor +2024-02-21 17:18:48,068 INFO HandlerThread:235 [interfaces.py:finish():202] Joined network monitor +2024-02-21 17:18:48,068 DEBUG SenderThread:235 [sender.py:send():382] send: stats +2024-02-21 17:18:49,314 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:18:49,315 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:18:53,070 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:18:54,315 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:18:58,071 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:18:59,316 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:19:03,077 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:19:03,480 INFO Thread-12 :235 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240221_171730-9wbsk0df/files/config.yaml +2024-02-21 17:19:04,317 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:19:08,212 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:19:09,318 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:19:13,213 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:19:14,324 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:19:17,566 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: resume +2024-02-21 17:19:17,566 INFO HandlerThread:235 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-21 17:19:17,566 INFO HandlerThread:235 [system_monitor.py:start():194] Starting system monitor +2024-02-21 17:19:17,566 INFO SystemMonitor:235 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-21 17:19:17,567 INFO SystemMonitor:235 [interfaces.py:start():190] Started cpu monitoring +2024-02-21 17:19:17,568 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: pause +2024-02-21 17:19:17,568 INFO HandlerThread:235 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-21 17:19:17,568 INFO HandlerThread:235 [system_monitor.py:finish():203] Stopping system monitor +2024-02-21 17:19:17,568 INFO SystemMonitor:235 [interfaces.py:start():190] Started disk monitoring +2024-02-21 17:19:17,568 DEBUG SystemMonitor:235 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-21 17:19:17,569 DEBUG SystemMonitor:235 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-21 17:19:17,569 DEBUG SystemMonitor:235 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-21 17:19:17,570 INFO HandlerThread:235 [interfaces.py:finish():202] Joined cpu monitor +2024-02-21 17:19:17,572 INFO HandlerThread:235 [interfaces.py:finish():202] Joined disk monitor +2024-02-21 17:19:17,573 DEBUG SenderThread:235 [sender.py:send():382] send: stats +2024-02-21 17:19:18,573 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:19:19,325 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:19:22,720 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: resume +2024-02-21 17:19:22,720 INFO HandlerThread:235 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-21 17:19:22,720 INFO HandlerThread:235 [system_monitor.py:start():194] Starting system monitor +2024-02-21 17:19:22,720 INFO SystemMonitor:235 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-21 17:19:22,721 INFO SystemMonitor:235 [interfaces.py:start():190] Started cpu monitoring +2024-02-21 17:19:22,722 INFO SystemMonitor:235 [interfaces.py:start():190] Started disk monitoring +2024-02-21 17:19:22,723 INFO SystemMonitor:235 [interfaces.py:start():190] Started gpu monitoring +2024-02-21 17:19:22,723 INFO SystemMonitor:235 [interfaces.py:start():190] Started memory monitoring +2024-02-21 17:19:22,725 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: pause +2024-02-21 17:19:22,725 INFO HandlerThread:235 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-21 17:19:22,725 INFO SystemMonitor:235 [interfaces.py:start():190] Started network monitoring +2024-02-21 17:19:22,725 INFO HandlerThread:235 [system_monitor.py:finish():203] Stopping system monitor +2024-02-21 17:19:22,726 DEBUG SystemMonitor:235 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-21 17:19:22,726 DEBUG SystemMonitor:235 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-21 17:19:22,726 DEBUG SystemMonitor:235 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-21 17:19:22,727 INFO HandlerThread:235 [interfaces.py:finish():202] Joined cpu monitor +2024-02-21 17:19:22,729 INFO HandlerThread:235 [interfaces.py:finish():202] Joined disk monitor +2024-02-21 17:19:22,747 INFO HandlerThread:235 [interfaces.py:finish():202] Joined gpu monitor +2024-02-21 17:19:22,748 INFO HandlerThread:235 [interfaces.py:finish():202] Joined memory monitor +2024-02-21 17:19:22,748 INFO HandlerThread:235 [interfaces.py:finish():202] Joined network monitor +2024-02-21 17:19:22,748 DEBUG SenderThread:235 [sender.py:send():382] send: stats +2024-02-21 17:19:23,749 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:19:24,326 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:19:28,750 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:19:29,327 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:19:29,544 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: resume +2024-02-21 17:19:29,544 INFO HandlerThread:235 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-21 17:19:29,544 INFO HandlerThread:235 [system_monitor.py:start():194] Starting system monitor +2024-02-21 17:19:29,544 INFO SystemMonitor:235 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-21 17:19:29,545 INFO SystemMonitor:235 [interfaces.py:start():190] Started cpu monitoring +2024-02-21 17:19:29,545 INFO SystemMonitor:235 [interfaces.py:start():190] Started disk monitoring +2024-02-21 17:19:29,546 INFO SystemMonitor:235 [interfaces.py:start():190] Started gpu monitoring +2024-02-21 17:19:29,547 INFO SystemMonitor:235 [interfaces.py:start():190] Started memory monitoring +2024-02-21 17:19:29,547 INFO SystemMonitor:235 [interfaces.py:start():190] Started network monitoring +2024-02-21 17:19:29,548 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: pause +2024-02-21 17:19:29,548 INFO HandlerThread:235 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-21 17:19:29,549 INFO HandlerThread:235 [system_monitor.py:finish():203] Stopping system monitor +2024-02-21 17:19:29,549 DEBUG SystemMonitor:235 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-21 17:19:29,549 DEBUG SystemMonitor:235 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-21 17:19:29,549 DEBUG SystemMonitor:235 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-21 17:19:29,550 INFO HandlerThread:235 [interfaces.py:finish():202] Joined cpu monitor +2024-02-21 17:19:29,553 INFO HandlerThread:235 [interfaces.py:finish():202] Joined disk monitor +2024-02-21 17:19:29,572 INFO HandlerThread:235 [interfaces.py:finish():202] Joined gpu monitor +2024-02-21 17:19:29,572 INFO HandlerThread:235 [interfaces.py:finish():202] Joined memory monitor +2024-02-21 17:19:29,572 INFO HandlerThread:235 [interfaces.py:finish():202] Joined network monitor +2024-02-21 17:19:29,573 DEBUG SenderThread:235 [sender.py:send():382] send: stats +2024-02-21 17:19:33,815 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: resume +2024-02-21 17:19:33,815 INFO HandlerThread:235 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-21 17:19:33,815 INFO HandlerThread:235 [system_monitor.py:start():194] Starting system monitor +2024-02-21 17:19:33,815 INFO SystemMonitor:235 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-21 17:19:33,816 INFO SystemMonitor:235 [interfaces.py:start():190] Started cpu monitoring +2024-02-21 17:19:33,816 INFO SystemMonitor:235 [interfaces.py:start():190] Started disk monitoring +2024-02-21 17:19:33,817 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: pause +2024-02-21 17:19:33,818 INFO HandlerThread:235 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-21 17:19:33,818 INFO HandlerThread:235 [system_monitor.py:finish():203] Stopping system monitor +2024-02-21 17:19:33,818 INFO SystemMonitor:235 [interfaces.py:start():190] Started gpu monitoring +2024-02-21 17:19:33,818 DEBUG SystemMonitor:235 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-21 17:19:33,818 DEBUG SystemMonitor:235 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-21 17:19:33,818 DEBUG SystemMonitor:235 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-21 17:19:33,820 INFO HandlerThread:235 [interfaces.py:finish():202] Joined cpu monitor +2024-02-21 17:19:33,822 INFO HandlerThread:235 [interfaces.py:finish():202] Joined disk monitor +2024-02-21 17:19:33,843 INFO HandlerThread:235 [interfaces.py:finish():202] Joined gpu monitor +2024-02-21 17:19:33,844 DEBUG SenderThread:235 [sender.py:send():382] send: stats +2024-02-21 17:19:33,845 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:19:34,328 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:19:35,014 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: resume +2024-02-21 17:19:35,014 INFO HandlerThread:235 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-21 17:19:35,014 INFO HandlerThread:235 [system_monitor.py:start():194] Starting system monitor +2024-02-21 17:19:35,015 INFO SystemMonitor:235 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-21 17:19:35,015 INFO SystemMonitor:235 [interfaces.py:start():190] Started cpu monitoring +2024-02-21 17:19:35,016 INFO SystemMonitor:235 [interfaces.py:start():190] Started disk monitoring +2024-02-21 17:19:35,020 INFO SystemMonitor:235 [interfaces.py:start():190] Started gpu monitoring +2024-02-21 17:19:35,020 INFO SystemMonitor:235 [interfaces.py:start():190] Started memory monitoring +2024-02-21 17:19:35,021 INFO SystemMonitor:235 [interfaces.py:start():190] Started network monitoring +2024-02-21 17:19:35,941 DEBUG SenderThread:235 [sender.py:send():382] send: config +2024-02-21 17:19:35,942 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:19:35,942 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:19:35,942 WARNING SenderThread:235 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) +2024-02-21 17:19:36,341 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: pause +2024-02-21 17:19:36,341 INFO HandlerThread:235 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-21 17:19:36,341 INFO HandlerThread:235 [system_monitor.py:finish():203] Stopping system monitor +2024-02-21 17:19:36,342 DEBUG SystemMonitor:235 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-21 17:19:36,342 DEBUG SystemMonitor:235 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-21 17:19:36,342 DEBUG SystemMonitor:235 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-21 17:19:36,343 INFO HandlerThread:235 [interfaces.py:finish():202] Joined cpu monitor +2024-02-21 17:19:36,343 INFO HandlerThread:235 [interfaces.py:finish():202] Joined disk monitor +2024-02-21 17:19:36,354 INFO HandlerThread:235 [interfaces.py:finish():202] Joined gpu monitor +2024-02-21 17:19:36,354 INFO HandlerThread:235 [interfaces.py:finish():202] Joined memory monitor +2024-02-21 17:19:36,355 INFO HandlerThread:235 [interfaces.py:finish():202] Joined network monitor +2024-02-21 17:19:36,355 DEBUG SenderThread:235 [sender.py:send():382] send: stats +2024-02-21 17:19:37,546 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: resume +2024-02-21 17:19:37,546 INFO HandlerThread:235 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-21 17:19:37,546 INFO HandlerThread:235 [system_monitor.py:start():194] Starting system monitor +2024-02-21 17:19:37,547 INFO SystemMonitor:235 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-21 17:19:37,547 INFO SystemMonitor:235 [interfaces.py:start():190] Started cpu monitoring +2024-02-21 17:19:37,548 INFO SystemMonitor:235 [interfaces.py:start():190] Started disk monitoring +2024-02-21 17:19:37,549 INFO SystemMonitor:235 [interfaces.py:start():190] Started gpu monitoring +2024-02-21 17:19:37,550 INFO SystemMonitor:235 [interfaces.py:start():190] Started memory monitoring +2024-02-21 17:19:37,551 INFO SystemMonitor:235 [interfaces.py:start():190] Started network monitoring +2024-02-21 17:19:38,771 DEBUG SenderThread:235 [sender.py:send():382] send: config +2024-02-21 17:19:38,772 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:19:38,772 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:19:38,772 WARNING SenderThread:235 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) +2024-02-21 17:19:39,146 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: pause +2024-02-21 17:19:39,146 INFO HandlerThread:235 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-21 17:19:39,146 INFO HandlerThread:235 [system_monitor.py:finish():203] Stopping system monitor +2024-02-21 17:19:39,146 DEBUG SystemMonitor:235 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-21 17:19:39,146 DEBUG SystemMonitor:235 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-21 17:19:39,146 DEBUG SystemMonitor:235 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-21 17:19:39,148 INFO HandlerThread:235 [interfaces.py:finish():202] Joined cpu monitor +2024-02-21 17:19:39,148 INFO HandlerThread:235 [interfaces.py:finish():202] Joined disk monitor +2024-02-21 17:19:39,158 INFO HandlerThread:235 [interfaces.py:finish():202] Joined gpu monitor +2024-02-21 17:19:39,158 INFO HandlerThread:235 [interfaces.py:finish():202] Joined memory monitor +2024-02-21 17:19:39,158 INFO HandlerThread:235 [interfaces.py:finish():202] Joined network monitor +2024-02-21 17:19:39,159 DEBUG SenderThread:235 [sender.py:send():382] send: stats +2024-02-21 17:19:39,159 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:19:39,420 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:19:44,160 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:19:44,420 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:19:49,161 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:19:49,421 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:19:54,162 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:19:54,422 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:19:59,163 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:19:59,423 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:20:04,169 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:20:04,424 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:20:04,501 INFO Thread-12 :235 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240221_171730-9wbsk0df/files/config.yaml +2024-02-21 17:20:09,300 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:20:09,425 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:20:14,301 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:20:14,426 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:20:19,302 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:20:19,427 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:20:24,303 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:20:24,428 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:20:29,304 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:20:29,429 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:20:34,305 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:20:34,430 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:20:39,306 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:20:39,431 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:20:44,307 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:20:44,432 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:20:49,308 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:20:49,433 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:20:54,309 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:20:54,435 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:20:59,310 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:20:59,436 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:21:04,311 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:21:04,437 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:21:09,312 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:21:09,438 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:21:14,313 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:21:14,439 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:21:19,314 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:21:19,440 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:21:19,591 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: resume +2024-02-21 17:21:19,591 INFO HandlerThread:235 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-21 17:21:19,591 INFO HandlerThread:235 [system_monitor.py:start():194] Starting system monitor +2024-02-21 17:21:19,591 INFO SystemMonitor:235 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-21 17:21:19,592 INFO SystemMonitor:235 [interfaces.py:start():190] Started cpu monitoring +2024-02-21 17:21:19,593 INFO SystemMonitor:235 [interfaces.py:start():190] Started disk monitoring +2024-02-21 17:21:19,595 INFO SystemMonitor:235 [interfaces.py:start():190] Started gpu monitoring +2024-02-21 17:21:19,596 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: pause +2024-02-21 17:21:19,596 INFO HandlerThread:235 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-21 17:21:19,596 INFO HandlerThread:235 [system_monitor.py:finish():203] Stopping system monitor +2024-02-21 17:21:19,597 INFO HandlerThread:235 [interfaces.py:finish():202] Joined cpu monitor +2024-02-21 17:21:19,598 INFO SystemMonitor:235 [interfaces.py:start():190] Started memory monitoring +2024-02-21 17:21:19,598 DEBUG SystemMonitor:235 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-21 17:21:19,598 DEBUG SystemMonitor:235 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-21 17:21:19,598 DEBUG SystemMonitor:235 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-21 17:21:19,601 INFO HandlerThread:235 [interfaces.py:finish():202] Joined disk monitor +2024-02-21 17:21:19,619 INFO HandlerThread:235 [interfaces.py:finish():202] Joined gpu monitor +2024-02-21 17:21:19,620 INFO HandlerThread:235 [interfaces.py:finish():202] Joined memory monitor +2024-02-21 17:21:19,620 DEBUG SenderThread:235 [sender.py:send():382] send: stats +2024-02-21 17:21:24,441 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:21:24,622 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:21:29,442 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:21:29,622 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:21:34,443 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:21:34,623 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:21:35,971 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: resume +2024-02-21 17:21:35,971 INFO HandlerThread:235 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-21 17:21:35,971 INFO HandlerThread:235 [system_monitor.py:start():194] Starting system monitor +2024-02-21 17:21:35,971 INFO SystemMonitor:235 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-21 17:21:35,972 INFO SystemMonitor:235 [interfaces.py:start():190] Started cpu monitoring +2024-02-21 17:21:35,972 INFO SystemMonitor:235 [interfaces.py:start():190] Started disk monitoring +2024-02-21 17:21:35,973 INFO SystemMonitor:235 [interfaces.py:start():190] Started gpu monitoring +2024-02-21 17:21:35,974 INFO SystemMonitor:235 [interfaces.py:start():190] Started memory monitoring +2024-02-21 17:21:35,975 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: pause +2024-02-21 17:21:35,975 INFO HandlerThread:235 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-21 17:21:35,975 INFO HandlerThread:235 [system_monitor.py:finish():203] Stopping system monitor +2024-02-21 17:21:35,976 INFO SystemMonitor:235 [interfaces.py:start():190] Started network monitoring +2024-02-21 17:21:35,976 INFO HandlerThread:235 [interfaces.py:finish():202] Joined cpu monitor +2024-02-21 17:21:35,976 DEBUG SystemMonitor:235 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-21 17:21:35,977 DEBUG SystemMonitor:235 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-21 17:21:35,977 DEBUG SystemMonitor:235 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-21 17:21:35,980 INFO HandlerThread:235 [interfaces.py:finish():202] Joined disk monitor +2024-02-21 17:21:35,997 INFO HandlerThread:235 [interfaces.py:finish():202] Joined gpu monitor +2024-02-21 17:21:35,997 INFO HandlerThread:235 [interfaces.py:finish():202] Joined memory monitor +2024-02-21 17:21:35,997 INFO HandlerThread:235 [interfaces.py:finish():202] Joined network monitor +2024-02-21 17:21:35,997 DEBUG SenderThread:235 [sender.py:send():382] send: stats +2024-02-21 17:21:39,444 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:21:39,998 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:21:41,191 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: resume +2024-02-21 17:21:41,191 INFO HandlerThread:235 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-21 17:21:41,191 INFO HandlerThread:235 [system_monitor.py:start():194] Starting system monitor +2024-02-21 17:21:41,191 INFO SystemMonitor:235 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-21 17:21:41,192 INFO SystemMonitor:235 [interfaces.py:start():190] Started cpu monitoring +2024-02-21 17:21:41,192 INFO SystemMonitor:235 [interfaces.py:start():190] Started disk monitoring +2024-02-21 17:21:41,193 INFO SystemMonitor:235 [interfaces.py:start():190] Started gpu monitoring +2024-02-21 17:21:41,193 INFO SystemMonitor:235 [interfaces.py:start():190] Started memory monitoring +2024-02-21 17:21:41,195 INFO SystemMonitor:235 [interfaces.py:start():190] Started network monitoring +2024-02-21 17:21:42,153 DEBUG SenderThread:235 [sender.py:send():382] send: config +2024-02-21 17:21:42,154 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:21:42,155 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:21:42,155 WARNING SenderThread:235 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) +2024-02-21 17:21:42,563 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: pause +2024-02-21 17:21:42,563 INFO HandlerThread:235 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-21 17:21:42,563 INFO HandlerThread:235 [system_monitor.py:finish():203] Stopping system monitor +2024-02-21 17:21:42,564 INFO HandlerThread:235 [interfaces.py:finish():202] Joined cpu monitor +2024-02-21 17:21:42,564 DEBUG SystemMonitor:235 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-21 17:21:42,565 DEBUG SystemMonitor:235 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-21 17:21:42,565 DEBUG SystemMonitor:235 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-21 17:21:42,565 INFO HandlerThread:235 [interfaces.py:finish():202] Joined disk monitor +2024-02-21 17:21:42,577 INFO HandlerThread:235 [interfaces.py:finish():202] Joined gpu monitor +2024-02-21 17:21:42,577 INFO HandlerThread:235 [interfaces.py:finish():202] Joined memory monitor +2024-02-21 17:21:42,577 INFO HandlerThread:235 [interfaces.py:finish():202] Joined network monitor +2024-02-21 17:21:42,578 DEBUG SenderThread:235 [sender.py:send():382] send: stats +2024-02-21 17:21:43,655 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: resume +2024-02-21 17:21:43,655 INFO HandlerThread:235 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-21 17:21:43,655 INFO HandlerThread:235 [system_monitor.py:start():194] Starting system monitor +2024-02-21 17:21:43,655 INFO SystemMonitor:235 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-21 17:21:43,656 INFO SystemMonitor:235 [interfaces.py:start():190] Started cpu monitoring +2024-02-21 17:21:43,656 INFO SystemMonitor:235 [interfaces.py:start():190] Started disk monitoring +2024-02-21 17:21:43,658 INFO SystemMonitor:235 [interfaces.py:start():190] Started gpu monitoring +2024-02-21 17:21:43,658 INFO SystemMonitor:235 [interfaces.py:start():190] Started memory monitoring +2024-02-21 17:21:43,659 INFO SystemMonitor:235 [interfaces.py:start():190] Started network monitoring +2024-02-21 17:21:44,520 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:21:44,873 DEBUG SenderThread:235 [sender.py:send():382] send: config +2024-02-21 17:21:44,874 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:21:44,874 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:21:44,874 WARNING SenderThread:235 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) +2024-02-21 17:21:45,874 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:21:47,046 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: pause +2024-02-21 17:21:47,046 INFO HandlerThread:235 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-21 17:21:47,046 INFO HandlerThread:235 [system_monitor.py:finish():203] Stopping system monitor +2024-02-21 17:21:47,047 DEBUG SystemMonitor:235 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-21 17:21:47,047 DEBUG SystemMonitor:235 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-21 17:21:47,047 DEBUG SystemMonitor:235 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-21 17:21:47,048 INFO HandlerThread:235 [interfaces.py:finish():202] Joined cpu monitor +2024-02-21 17:21:47,048 INFO HandlerThread:235 [interfaces.py:finish():202] Joined disk monitor +2024-02-21 17:21:47,059 INFO HandlerThread:235 [interfaces.py:finish():202] Joined gpu monitor +2024-02-21 17:21:47,059 INFO HandlerThread:235 [interfaces.py:finish():202] Joined memory monitor +2024-02-21 17:21:47,059 INFO HandlerThread:235 [interfaces.py:finish():202] Joined network monitor +2024-02-21 17:21:47,060 DEBUG SenderThread:235 [sender.py:send():382] send: stats +2024-02-21 17:21:49,521 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:21:51,061 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:21:54,522 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:21:56,062 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:21:59,523 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:22:01,063 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:22:04,524 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:22:06,069 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:22:06,543 INFO Thread-12 :235 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240221_171730-9wbsk0df/files/config.yaml +2024-02-21 17:22:09,525 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:22:11,178 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:22:14,526 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:22:16,179 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:22:19,527 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:22:21,180 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:22:24,528 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:22:26,181 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:22:29,529 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:22:31,182 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:22:34,530 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:22:36,183 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:22:39,531 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:22:41,184 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:22:44,532 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:22:46,185 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:22:49,534 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:22:51,186 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:22:54,535 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:22:56,187 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:22:59,536 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:23:01,188 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:23:04,537 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:23:06,188 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:23:07,592 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: resume +2024-02-21 17:23:07,592 INFO HandlerThread:235 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-21 17:23:07,592 INFO HandlerThread:235 [system_monitor.py:start():194] Starting system monitor +2024-02-21 17:23:07,592 INFO SystemMonitor:235 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-21 17:23:07,593 INFO SystemMonitor:235 [interfaces.py:start():190] Started cpu monitoring +2024-02-21 17:23:07,593 INFO SystemMonitor:235 [interfaces.py:start():190] Started disk monitoring +2024-02-21 17:23:07,594 INFO SystemMonitor:235 [interfaces.py:start():190] Started gpu monitoring +2024-02-21 17:23:07,596 INFO SystemMonitor:235 [interfaces.py:start():190] Started memory monitoring +2024-02-21 17:23:07,596 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: pause +2024-02-21 17:23:07,596 INFO HandlerThread:235 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-21 17:23:07,596 INFO HandlerThread:235 [system_monitor.py:finish():203] Stopping system monitor +2024-02-21 17:23:07,597 INFO SystemMonitor:235 [interfaces.py:start():190] Started network monitoring +2024-02-21 17:23:07,598 DEBUG SystemMonitor:235 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-21 17:23:07,598 DEBUG SystemMonitor:235 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-21 17:23:07,598 DEBUG SystemMonitor:235 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-21 17:23:07,600 INFO HandlerThread:235 [interfaces.py:finish():202] Joined cpu monitor +2024-02-21 17:23:07,601 INFO HandlerThread:235 [interfaces.py:finish():202] Joined disk monitor +2024-02-21 17:23:07,619 INFO HandlerThread:235 [interfaces.py:finish():202] Joined gpu monitor +2024-02-21 17:23:07,619 INFO HandlerThread:235 [interfaces.py:finish():202] Joined memory monitor +2024-02-21 17:23:07,619 INFO HandlerThread:235 [interfaces.py:finish():202] Joined network monitor +2024-02-21 17:23:07,619 DEBUG SenderThread:235 [sender.py:send():382] send: stats +2024-02-21 17:23:09,538 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:23:09,896 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: resume +2024-02-21 17:23:09,896 INFO HandlerThread:235 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-21 17:23:09,896 INFO HandlerThread:235 [system_monitor.py:start():194] Starting system monitor +2024-02-21 17:23:09,896 INFO SystemMonitor:235 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-21 17:23:09,897 INFO SystemMonitor:235 [interfaces.py:start():190] Started cpu monitoring +2024-02-21 17:23:09,898 INFO SystemMonitor:235 [interfaces.py:start():190] Started disk monitoring +2024-02-21 17:23:09,898 INFO SystemMonitor:235 [interfaces.py:start():190] Started gpu monitoring +2024-02-21 17:23:09,899 INFO SystemMonitor:235 [interfaces.py:start():190] Started memory monitoring +2024-02-21 17:23:09,901 INFO SystemMonitor:235 [interfaces.py:start():190] Started network monitoring +2024-02-21 17:23:11,620 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:23:14,690 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:23:14,722 DEBUG SenderThread:235 [sender.py:send():382] send: config +2024-02-21 17:23:14,724 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:23:14,724 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:23:14,724 WARNING SenderThread:235 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) +2024-02-21 17:23:15,140 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: pause +2024-02-21 17:23:15,140 INFO HandlerThread:235 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-21 17:23:15,140 INFO HandlerThread:235 [system_monitor.py:finish():203] Stopping system monitor +2024-02-21 17:23:15,140 DEBUG SystemMonitor:235 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-21 17:23:15,141 DEBUG SystemMonitor:235 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-21 17:23:15,141 DEBUG SystemMonitor:235 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-21 17:23:15,142 INFO HandlerThread:235 [interfaces.py:finish():202] Joined cpu monitor +2024-02-21 17:23:15,142 INFO HandlerThread:235 [interfaces.py:finish():202] Joined disk monitor +2024-02-21 17:23:15,153 INFO HandlerThread:235 [interfaces.py:finish():202] Joined gpu monitor +2024-02-21 17:23:15,154 INFO HandlerThread:235 [interfaces.py:finish():202] Joined memory monitor +2024-02-21 17:23:15,154 INFO HandlerThread:235 [interfaces.py:finish():202] Joined network monitor +2024-02-21 17:23:15,154 DEBUG SenderThread:235 [sender.py:send():382] send: stats +2024-02-21 17:23:16,281 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: resume +2024-02-21 17:23:16,281 INFO HandlerThread:235 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-21 17:23:16,281 INFO HandlerThread:235 [system_monitor.py:start():194] Starting system monitor +2024-02-21 17:23:16,281 INFO SystemMonitor:235 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-21 17:23:16,281 INFO SystemMonitor:235 [interfaces.py:start():190] Started cpu monitoring +2024-02-21 17:23:16,282 INFO SystemMonitor:235 [interfaces.py:start():190] Started disk monitoring +2024-02-21 17:23:16,284 INFO SystemMonitor:235 [interfaces.py:start():190] Started gpu monitoring +2024-02-21 17:23:16,285 INFO SystemMonitor:235 [interfaces.py:start():190] Started memory monitoring +2024-02-21 17:23:16,285 INFO SystemMonitor:235 [interfaces.py:start():190] Started network monitoring +2024-02-21 17:23:17,155 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:23:19,972 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:23:21,590 DEBUG SenderThread:235 [sender.py:send():382] send: config +2024-02-21 17:23:21,591 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:23:21,591 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:23:21,591 WARNING SenderThread:235 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) +2024-02-21 17:23:22,592 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:23:25,241 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:23:27,592 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:23:30,242 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:23:32,593 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:23:35,243 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:23:37,599 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:23:38,576 INFO Thread-12 :235 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240221_171730-9wbsk0df/files/config.yaml +2024-02-21 17:23:40,246 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:23:42,743 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:23:45,247 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:23:47,744 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:23:50,248 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:23:52,745 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:23:55,249 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:23:57,746 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:24:00,250 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:24:02,747 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:24:05,256 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:24:07,747 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:24:10,257 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:24:12,748 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:24:15,261 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:24:16,286 DEBUG SystemMonitor:235 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-21 17:24:16,293 DEBUG SenderThread:235 [sender.py:send():382] send: stats +2024-02-21 17:24:18,293 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:24:20,262 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:24:23,294 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:24:25,263 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:24:28,295 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:24:30,264 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:24:33,296 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:24:35,265 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:24:38,297 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:24:40,266 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:24:43,297 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:24:45,267 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:24:46,288 DEBUG SenderThread:235 [sender.py:send():382] send: stats +2024-02-21 17:24:49,289 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:24:50,268 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:24:54,290 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:24:55,270 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:24:59,291 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:25:00,270 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:25:04,291 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:25:05,281 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:25:09,292 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:25:10,282 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:25:14,293 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:25:15,283 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:25:16,289 DEBUG SenderThread:235 [sender.py:send():382] send: stats +2024-02-21 17:25:20,284 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:25:20,291 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:25:25,285 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:25:25,292 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:25:30,293 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:25:30,293 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:25:35,293 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:25:35,304 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:25:40,294 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:25:40,305 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:25:45,295 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:25:45,306 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:25:46,289 DEBUG SenderThread:235 [sender.py:send():382] send: stats +2024-02-21 17:25:50,307 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:25:51,291 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:25:55,318 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:25:56,292 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:26:00,380 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:26:01,292 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:26:05,416 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:26:06,378 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:26:07,628 INFO Thread-12 :235 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240221_171730-9wbsk0df/files/output.log +2024-02-21 17:26:10,453 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:26:11,379 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:26:13,207 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: partial_history +2024-02-21 17:26:13,212 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:26:13,212 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:26:13,212 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:26:13,212 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:26:13,212 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:26:13,212 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:26:13,212 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:26:13,213 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:26:13,213 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:26:13,213 DEBUG SenderThread:235 [sender.py:send():382] send: metric +2024-02-21 17:26:13,213 DEBUG SenderThread:235 [sender.py:send():382] send: history +2024-02-21 17:26:13,213 DEBUG SenderThread:235 [sender.py:send_request():409] send_request: summary_record +2024-02-21 17:26:13,214 INFO SenderThread:235 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-02-21 17:26:13,630 INFO Thread-12 :235 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240221_171730-9wbsk0df/files/wandb-summary.json +2024-02-21 17:26:15,610 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:26:16,290 DEBUG SenderThread:235 [sender.py:send():382] send: stats +2024-02-21 17:26:17,291 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report +2024-02-21 17:26:20,611 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: keepalive +2024-02-21 17:26:22,292 DEBUG HandlerThread:235 [handler.py:handle_request():146] handle_request: status_report