diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" --- "a/wandb/debug-internal.log" +++ "b/wandb/debug-internal.log" @@ -1,1002 +1,179 @@ -2024-03-06 14:54:55,720 INFO StreamThr :137 [internal.py:wandb_internal():86] W&B internal server running at pid: 137, started at: 2024-03-06 14:54:55.719541 -2024-03-06 14:54:55,721 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status -2024-03-06 14:54:56,525 INFO WriterThread:137 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/run-h1uv5tyi.wandb -2024-03-06 14:54:56,525 DEBUG SenderThread:137 [sender.py:send():382] send: header -2024-03-06 14:54:56,526 DEBUG SenderThread:137 [sender.py:send():382] send: run -2024-03-06 14:54:56,743 INFO SenderThread:137 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files -2024-03-06 14:54:56,743 INFO SenderThread:137 [sender.py:_start_run_threads():1136] run started: h1uv5tyi with start time 1709736895.718757 -2024-03-06 14:54:56,750 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: check_version -2024-03-06 14:54:56,750 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: check_version -2024-03-06 14:55:00,206 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:55:00,811 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:55:01,751 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:55:05,207 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:55:05,812 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:55:06,752 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:55:10,208 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:55:10,813 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:55:11,753 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:55:15,209 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:55:15,814 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:55:16,754 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:55:20,210 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:55:20,815 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:55:21,755 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:55:25,210 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:55:25,816 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:55:26,762 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: run_start -2024-03-06 14:55:26,774 DEBUG HandlerThread:137 [system_info.py:__init__():27] System info init -2024-03-06 14:55:26,774 DEBUG HandlerThread:137 [system_info.py:__init__():42] System info init done -2024-03-06 14:55:26,774 INFO HandlerThread:137 [system_monitor.py:start():194] Starting system monitor -2024-03-06 14:55:26,775 INFO SystemMonitor:137 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-03-06 14:55:26,776 INFO HandlerThread:137 [system_monitor.py:probe():214] Collecting system info -2024-03-06 14:55:26,776 INFO SystemMonitor:137 [interfaces.py:start():190] Started cpu monitoring -2024-03-06 14:55:26,777 INFO SystemMonitor:137 [interfaces.py:start():190] Started disk monitoring -2024-03-06 14:55:26,778 INFO SystemMonitor:137 [interfaces.py:start():190] Started gpu monitoring -2024-03-06 14:55:26,778 INFO SystemMonitor:137 [interfaces.py:start():190] Started memory monitoring -2024-03-06 14:55:26,779 INFO SystemMonitor:137 [interfaces.py:start():190] Started network monitoring -2024-03-06 14:55:26,788 DEBUG HandlerThread:137 [system_info.py:probe():151] Probing system -2024-03-06 14:55:26,791 DEBUG HandlerThread:137 [gitlib.py:_init_repo():56] git repository is invalid -2024-03-06 14:55:26,791 DEBUG HandlerThread:137 [system_info.py:probe():199] Probing system done -2024-03-06 14:55:26,791 DEBUG HandlerThread:137 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-03-06T14:55:26.789044', 'startedAt': '2024-03-06T14:54:55.713807', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': '41759fa8e6ad', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.172, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.172, 'min': 0.0, 'max': 0.0}, {'current': 2000.172, 'min': 0.0, 'max': 0.0}, {'current': 2000.172, 'min': 0.0, 'max': 0.0}, {'current': 2000.172, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5519.315299987793}}, 'gpu': 'Tesla P100-PCIE-16GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'Tesla P100-PCIE-16GB', 'memory_total': 17179869184}], 'memory': {'total': 31.357559204101562}} -2024-03-06 14:55:26,791 INFO HandlerThread:137 [system_monitor.py:probe():224] Finished collecting system info -2024-03-06 14:55:26,791 INFO HandlerThread:137 [system_monitor.py:probe():227] Publishing system info -2024-03-06 14:55:26,791 DEBUG HandlerThread:137 [system_info.py:_save_conda():208] Saving list of conda packages installed into the current environment -2024-03-06 14:55:27,753 INFO Thread-18 :137 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/conda-environment.yaml -2024-03-06 14:55:30,211 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:55:35,212 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:55:40,213 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:55:45,213 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:55:50,215 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:55:55,215 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:00,216 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:05,217 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:10,218 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:15,218 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:20,219 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:21,535 DEBUG HandlerThread:137 [system_info.py:_save_conda():220] Saving conda packages done -2024-03-06 14:56:21,535 INFO HandlerThread:137 [system_monitor.py:probe():229] Finished publishing system info -2024-03-06 14:56:21,543 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:21,543 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:56:21,543 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:21,543 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:56:21,544 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:21,544 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:56:21,544 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:21,544 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:56:21,545 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:21,545 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:56:21,545 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:21,545 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: python_packages -2024-03-06 14:56:21,545 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: stop_status -2024-03-06 14:56:21,545 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-06 14:56:21,546 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:21,546 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:56:21,546 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: python_packages -2024-03-06 14:56:21,547 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:21,550 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: stop_status -2024-03-06 14:56:21,550 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:56:21,554 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:21,554 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:56:21,555 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:21,555 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:56:21,555 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:21,620 DEBUG SenderThread:137 [sender.py:send():382] send: telemetry -2024-03-06 14:56:21,620 DEBUG SenderThread:137 [sender.py:send():382] send: config -2024-03-06 14:56:21,622 DEBUG SenderThread:137 [sender.py:send():382] send: metric -2024-03-06 14:56:21,622 DEBUG SenderThread:137 [sender.py:send():382] send: telemetry -2024-03-06 14:56:21,622 DEBUG SenderThread:137 [sender.py:send():382] send: metric -2024-03-06 14:56:21,622 WARNING SenderThread:137 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) -2024-03-06 14:56:21,623 DEBUG SenderThread:137 [sender.py:send():382] send: telemetry -2024-03-06 14:56:21,623 DEBUG SenderThread:137 [sender.py:send():382] send: files -2024-03-06 14:56:21,623 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now -2024-03-06 14:56:21,768 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/conda-environment.yaml -2024-03-06 14:56:21,769 INFO Thread-18 :137 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-metadata.json -2024-03-06 14:56:21,769 INFO Thread-18 :137 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/requirements.txt -2024-03-06 14:56:21,911 INFO wandb-upload_0:137 [upload_job.py:push():131] Uploaded file /tmp/tmpall_1zgjwandb/0kp4rydi-wandb-metadata.json -2024-03-06 14:56:22,799 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:56:25,220 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:25,811 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 14:56:25,812 DEBUG SenderThread:137 [sender.py:send():382] send: metric -2024-03-06 14:56:25,813 DEBUG SenderThread:137 [sender.py:send():382] send: metric -2024-03-06 14:56:25,813 DEBUG SenderThread:137 [sender.py:send():382] send: metric -2024-03-06 14:56:25,813 DEBUG SenderThread:137 [sender.py:send():382] send: metric -2024-03-06 14:56:25,813 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 14:56:25,813 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 14:56:25,815 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 14:56:26,770 INFO Thread-18 :137 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 14:56:26,779 DEBUG SystemMonitor:137 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-03-06 14:56:26,781 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 14:56:26,786 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:26,929 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 14:56:26,932 DEBUG SenderThread:137 [sender.py:send():382] send: metric -2024-03-06 14:56:26,932 DEBUG SenderThread:137 [sender.py:send():382] send: metric -2024-03-06 14:56:26,932 DEBUG SenderThread:137 [sender.py:send():382] send: metric -2024-03-06 14:56:26,932 DEBUG SenderThread:137 [sender.py:send():382] send: metric -2024-03-06 14:56:26,932 DEBUG SenderThread:137 [sender.py:send():382] send: metric -2024-03-06 14:56:26,932 DEBUG SenderThread:137 [sender.py:send():382] send: metric -2024-03-06 14:56:26,933 DEBUG SenderThread:137 [sender.py:send():382] send: metric -2024-03-06 14:56:26,933 DEBUG SenderThread:137 [sender.py:send():382] send: metric -2024-03-06 14:56:26,933 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 14:56:26,933 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 14:56:26,933 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 14:56:27,771 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/config.yaml -2024-03-06 14:56:27,771 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 14:56:27,772 INFO Thread-18 :137 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 14:56:27,867 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:56:27,868 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:56:29,772 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 14:56:30,221 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:31,935 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:32,890 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:56:35,222 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:36,935 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:37,891 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:56:40,223 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:41,936 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:42,892 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:56:45,224 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:46,937 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:47,893 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:56:50,225 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:51,938 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:52,895 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:56:55,225 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:56,781 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 14:56:57,787 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:56:57,901 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:56:58,020 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 14:56:58,021 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 14:56:58,022 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 14:56:58,022 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 14:56:58,790 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/config.yaml -2024-03-06 14:56:58,791 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 14:56:59,136 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 14:56:59,137 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 14:56:59,138 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 14:56:59,138 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 14:56:59,791 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 14:57:00,226 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:57:01,792 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 14:57:03,059 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:57:03,139 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:57:05,227 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:57:08,060 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:57:08,140 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:57:10,228 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:57:13,061 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:57:13,141 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:57:15,229 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:57:18,063 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:57:18,142 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:57:20,230 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:57:23,064 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:57:23,143 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:57:25,231 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:57:26,782 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 14:57:28,065 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:57:28,783 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:57:30,231 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:57:32,504 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 14:57:32,505 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 14:57:32,506 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 14:57:32,506 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 14:57:32,805 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 14:57:33,066 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:57:33,616 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 14:57:33,618 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 14:57:33,618 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 14:57:33,619 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 14:57:33,805 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 14:57:34,620 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:57:35,232 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:57:35,806 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 14:57:38,084 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:57:39,621 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:57:40,233 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:57:43,085 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:57:44,622 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:57:45,233 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:57:48,093 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:57:49,623 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:57:50,234 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:57:53,094 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:57:54,624 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:57:55,235 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:57:56,783 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 14:57:58,095 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:57:59,784 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:58:00,236 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:58:03,097 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:58:04,785 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:58:05,236 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:58:07,778 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 14:58:07,779 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 14:58:07,779 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 14:58:07,781 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 14:58:07,818 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 14:58:08,098 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:58:08,889 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 14:58:08,890 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 14:58:08,890 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 14:58:08,891 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 14:58:09,819 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 14:58:09,891 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:58:10,238 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:58:11,820 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 14:58:13,166 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:58:14,892 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:58:15,239 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:58:18,167 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:58:19,893 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:58:20,240 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:58:23,168 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:58:24,894 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:58:25,240 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:58:26,784 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 14:58:28,169 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:58:30,242 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:58:30,785 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:58:33,170 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:58:35,243 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:58:35,786 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:58:38,171 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:58:40,244 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:58:40,787 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:58:42,233 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 14:58:42,234 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 14:58:42,234 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 14:58:42,235 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 14:58:42,831 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 14:58:43,172 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:58:43,344 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 14:58:43,345 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 14:58:43,345 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 14:58:43,346 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 14:58:43,832 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 14:58:45,245 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:58:45,832 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 14:58:46,347 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:58:48,293 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:58:48,460 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 14:58:48,462 DEBUG SenderThread:137 [sender.py:send():382] send: metric -2024-03-06 14:58:48,462 DEBUG SenderThread:137 [sender.py:send():382] send: metric -2024-03-06 14:58:48,462 DEBUG SenderThread:137 [sender.py:send():382] send: metric -2024-03-06 14:58:48,462 DEBUG SenderThread:137 [sender.py:send():382] send: metric -2024-03-06 14:58:48,462 DEBUG SenderThread:137 [sender.py:send():382] send: metric -2024-03-06 14:58:48,463 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 14:58:48,463 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 14:58:48,465 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 14:58:48,833 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 14:58:50,246 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:58:51,466 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:58:53,295 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:58:55,247 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:58:56,467 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:58:56,785 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 14:58:58,296 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:59:00,247 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:59:01,360 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 14:59:01,362 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 14:59:01,362 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 14:59:01,364 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 14:59:01,838 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 14:59:02,370 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:59:02,839 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/config.yaml -2024-03-06 14:59:03,297 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:59:03,839 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 14:59:05,248 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:59:07,562 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:59:08,298 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:59:10,249 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:59:12,563 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:59:13,299 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:59:15,250 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:59:17,564 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:59:18,300 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:59:20,251 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:59:22,565 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:59:23,301 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:59:25,252 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:59:26,785 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 14:59:27,786 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:59:28,302 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:59:30,253 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:59:32,787 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:59:33,304 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:59:35,254 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:59:37,788 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:59:38,305 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:59:40,255 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:59:42,790 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:59:43,306 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:59:45,256 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:59:47,791 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:59:48,307 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:59:50,257 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:59:52,792 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:59:53,308 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:59:55,258 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:59:56,786 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 14:59:58,310 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 14:59:58,661 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 14:59:59,469 DEBUG SenderThread:137 [sender.py:send():382] send: config -2024-03-06 14:59:59,470 DEBUG SenderThread:137 [sender.py:send():382] send: metric -2024-03-06 14:59:59,471 DEBUG SenderThread:137 [sender.py:send():382] send: metric -2024-03-06 14:59:59,471 WARNING SenderThread:137 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) -2024-03-06 14:59:59,858 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 15:00:00,259 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:00:03,430 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:00:04,477 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:00:04,860 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/config.yaml -2024-03-06 15:00:05,260 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:00:08,431 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:00:09,637 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:00:10,260 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:00:13,437 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:00:14,638 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:00:15,261 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:00:18,438 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:00:19,639 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:00:20,262 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:00:23,439 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:00:24,640 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:00:25,263 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:00:26,787 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:00:27,703 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:00:27,704 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:00:27,705 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:00:27,705 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:00:27,867 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:00:28,440 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:00:28,821 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:00:28,823 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:00:28,823 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:00:28,824 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:00:28,868 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:00:29,826 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:00:30,263 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:00:31,869 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 15:00:34,305 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:00:34,827 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:00:35,264 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:00:39,306 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:00:39,828 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:00:40,265 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:00:44,308 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:00:44,829 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:00:45,265 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:00:49,309 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:00:49,830 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:00:50,266 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:00:54,309 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:00:54,831 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:00:55,267 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:00:56,788 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:00:59,311 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:01:00,267 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:01:00,789 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:01:01,367 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:01:01,368 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:01:01,369 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:01:01,369 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:01:01,880 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:01:02,484 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:01:02,485 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:01:02,486 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:01:02,486 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:01:02,880 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:01:03,881 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 15:01:04,991 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:01:05,268 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:01:06,488 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:01:10,269 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:01:10,633 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:01:11,489 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:01:15,270 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:01:15,634 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:01:16,490 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:01:20,271 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:01:20,636 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:01:21,491 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:01:25,272 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:01:25,637 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:01:26,492 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:01:26,789 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:01:30,272 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:01:30,638 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:01:31,790 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:01:35,273 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:01:35,559 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:01:35,561 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:01:35,561 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:01:35,561 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:01:35,638 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:01:35,893 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:01:36,671 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:01:36,672 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:01:36,672 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:01:36,673 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:01:36,893 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:01:37,674 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:01:37,893 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 15:01:40,274 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:01:40,910 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:01:42,675 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:01:45,275 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:01:45,911 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:01:47,676 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:01:50,276 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:01:50,912 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:01:52,677 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:01:55,277 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:01:55,915 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:01:56,789 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:01:57,790 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:02:00,278 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:02:00,916 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:02:02,791 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:02:05,279 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:02:05,917 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:02:07,792 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:02:08,680 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:02:08,681 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:02:08,681 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:02:08,682 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:02:08,905 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:02:09,794 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:02:09,801 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:02:09,801 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:02:09,802 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:02:09,905 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:02:10,280 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:02:10,961 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:02:11,906 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 15:02:12,803 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:02:15,281 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:02:16,026 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:02:17,804 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:02:20,282 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:02:21,111 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:02:22,804 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:02:25,283 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:02:26,118 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:02:26,790 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:02:28,791 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:02:30,283 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:02:31,123 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:02:33,792 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:02:35,284 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:02:36,124 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:02:38,793 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:02:40,285 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:02:41,126 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:02:41,912 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:02:41,913 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:02:41,914 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:02:41,914 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:02:41,917 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:02:43,028 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:02:43,029 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:02:43,029 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:02:43,030 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:02:43,918 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:02:44,031 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:02:45,286 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:02:45,919 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 15:02:46,341 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:02:46,342 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:02:49,032 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:02:50,286 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:02:51,344 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:02:54,033 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:02:55,287 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:02:56,345 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:02:56,791 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:02:59,792 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:03:00,288 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:03:01,347 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:03:04,793 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:03:05,289 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:03:06,347 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:03:09,794 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:03:10,290 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:03:11,349 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:03:14,795 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:03:15,152 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:03:15,153 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:03:15,154 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:03:15,154 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:03:15,291 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:03:15,931 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:03:16,264 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:03:16,265 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:03:16,266 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:03:16,267 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:03:16,350 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:03:16,931 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:03:17,931 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 15:03:20,268 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:03:20,291 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:03:21,486 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:03:25,269 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:03:25,292 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:03:26,487 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:03:26,792 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:03:30,293 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:03:30,793 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:03:31,488 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:03:35,294 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:03:35,794 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:03:36,489 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:03:40,294 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:03:40,795 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:03:41,490 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:03:45,295 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:03:45,796 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:03:46,492 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:03:50,296 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:03:50,797 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:03:51,157 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:03:51,158 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:03:51,158 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:03:51,159 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:03:51,500 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:03:51,944 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:03:52,269 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:03:52,270 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:03:52,271 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:03:52,271 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:03:52,945 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:03:53,945 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 15:03:55,297 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:03:56,272 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:03:56,545 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:03:56,793 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:04:00,298 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:04:01,546 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:04:01,794 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:04:05,299 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:04:06,547 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:04:06,795 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:04:10,300 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:04:11,548 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:04:11,796 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:04:15,301 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:04:16,549 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:04:16,797 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:04:20,302 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:04:21,550 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:04:21,798 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:04:25,302 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:04:26,552 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:04:26,793 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:04:27,794 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:04:30,303 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:04:31,553 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:04:32,795 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:04:35,304 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:04:36,554 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:04:37,796 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:04:40,305 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:04:41,556 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:04:41,963 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 15:04:43,556 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:04:45,306 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:04:46,557 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:04:48,557 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:04:50,307 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:04:51,558 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:04:53,558 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:04:55,307 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:04:56,559 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:04:56,794 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:04:58,795 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:05:00,308 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:05:01,560 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:05:03,796 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:05:05,309 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:05:06,561 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:05:08,797 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:05:10,310 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:05:11,562 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:05:13,798 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:05:15,311 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:05:16,563 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:05:18,799 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:05:20,311 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:05:21,564 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:05:23,800 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:05:25,312 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:05:26,565 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:05:26,795 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:05:29,796 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:05:30,313 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:05:31,567 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:05:34,797 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:05:35,314 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:05:36,568 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:05:39,798 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:05:40,315 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:05:41,569 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:05:44,799 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:05:45,316 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:05:46,570 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:05:49,800 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:05:50,317 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:05:51,571 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:05:54,801 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:05:55,318 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:05:56,572 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:05:56,795 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:06:00,320 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:06:00,797 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:06:01,573 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:06:05,320 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:06:05,798 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:06:06,574 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:06:10,322 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:06:10,799 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:06:11,575 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:06:15,323 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:06:15,800 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:06:16,576 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:06:20,324 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:06:20,801 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:06:21,577 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:06:25,325 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:06:25,802 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:06:26,578 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:06:26,796 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:06:30,326 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:06:31,579 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:06:31,798 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:06:35,327 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:06:36,580 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:06:36,799 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:06:40,328 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:06:41,581 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:06:41,800 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:06:45,328 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:06:46,582 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:06:46,801 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:06:49,869 DEBUG SenderThread:137 [sender.py:send():382] send: config -2024-03-06 15:06:49,871 DEBUG SenderThread:137 [sender.py:send():382] send: metric -2024-03-06 15:06:49,871 DEBUG SenderThread:137 [sender.py:send():382] send: metric -2024-03-06 15:06:49,871 WARNING SenderThread:137 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) -2024-03-06 15:06:50,021 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 15:06:50,329 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:06:51,649 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:06:51,872 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:06:55,330 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:06:56,650 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:06:56,797 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:06:57,798 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:07:00,331 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:07:01,651 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:07:02,799 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:07:05,331 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:07:06,652 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:07:07,800 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:07:10,332 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:07:11,654 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:07:12,801 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:07:15,333 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:07:16,655 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:07:17,025 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:07:17,026 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:07:17,026 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:07:17,027 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:07:17,033 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:07:18,033 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:07:18,154 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:07:18,357 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:07:18,357 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:07:18,358 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:07:19,038 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/config.yaml -2024-03-06 15:07:19,038 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:07:20,038 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 15:07:20,338 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:07:21,908 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:07:23,359 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:07:25,338 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:07:26,798 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:07:26,909 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:07:28,799 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:07:30,339 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:07:31,922 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:07:33,800 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:07:35,340 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:07:36,923 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:07:38,801 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:07:40,341 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:07:41,924 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:07:43,802 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:07:45,343 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:07:46,925 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:07:48,803 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:07:50,190 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:07:50,191 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:07:50,191 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:07:50,192 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:07:50,343 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:07:51,049 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:07:51,303 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:07:51,305 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:07:51,305 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:07:51,306 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:07:52,049 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:07:52,385 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:07:54,050 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 15:07:54,308 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:07:55,344 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:07:56,799 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:07:57,427 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:07:59,800 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:08:00,345 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:08:02,430 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:08:04,801 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:08:05,345 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:08:07,431 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:08:09,802 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:08:10,347 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:08:12,438 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:08:14,803 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:08:15,348 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:08:17,439 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:08:19,804 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:08:20,348 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:08:22,392 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:08:22,393 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:08:22,393 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:08:22,393 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:08:22,443 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:08:23,065 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:08:23,507 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:08:23,508 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:08:23,509 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:08:23,509 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:08:24,066 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:08:25,349 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:08:25,510 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:08:26,066 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 15:08:26,800 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:08:27,494 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:08:30,350 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:08:30,801 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:08:32,495 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:08:35,351 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:08:35,802 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:08:37,496 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:08:40,352 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:08:40,803 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:08:42,497 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:08:45,353 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:08:45,804 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:08:47,498 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:08:50,354 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:08:50,805 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:08:52,500 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:08:55,355 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:08:55,488 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:08:55,490 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:08:55,490 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:08:55,490 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:08:56,077 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:08:56,491 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:08:56,604 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:08:56,605 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:08:56,605 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:08:56,606 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:08:56,801 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:08:57,077 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:08:58,077 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 15:08:58,542 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:08:58,543 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:09:00,356 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:09:01,802 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:09:03,722 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:09:05,357 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:09:06,803 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:09:08,723 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:09:10,358 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:09:11,804 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:09:13,724 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:09:15,358 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:09:16,805 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:09:18,734 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:09:20,359 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:09:21,806 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:09:23,735 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:09:25,360 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:09:26,801 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:09:27,802 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:09:28,675 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:09:28,676 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:09:28,676 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:09:28,677 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:09:28,736 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:09:29,091 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:09:29,792 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:09:29,793 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:09:29,793 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:09:29,794 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:09:30,092 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:09:30,361 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:09:32,093 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 15:09:33,754 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:09:33,795 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:09:35,362 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:09:38,755 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:09:38,796 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:09:40,363 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:09:43,756 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:09:43,797 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:09:45,364 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:09:48,757 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:09:48,798 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:09:50,365 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:09:53,758 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:09:53,799 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:09:55,366 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:09:56,802 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:09:58,769 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:09:58,803 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:10:00,367 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:10:02,535 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:10:02,537 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:10:02,537 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:10:02,537 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:10:03,108 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:10:03,653 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:10:03,655 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:10:03,655 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:10:03,656 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:10:03,826 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:10:04,108 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:10:04,657 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:10:05,368 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:10:06,109 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 15:10:08,851 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:10:09,658 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:10:10,368 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:10:13,851 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:10:14,659 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:10:15,369 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:10:18,852 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:10:19,660 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:10:20,370 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:10:23,855 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:10:24,661 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:10:25,371 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:10:26,803 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:10:28,865 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:10:29,804 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:10:30,372 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:10:33,873 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:10:34,750 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:10:34,752 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:10:34,752 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:10:34,753 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:10:35,123 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:10:35,373 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:10:35,753 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:10:35,868 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:10:35,869 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:10:35,869 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:10:35,870 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:10:36,123 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 15:10:36,124 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:10:38,897 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:10:40,374 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:10:40,871 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:10:43,898 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:10:45,375 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:10:45,872 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:10:48,908 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:10:50,376 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:10:50,877 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:10:53,909 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:10:55,377 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:10:55,878 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:10:56,804 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:10:58,910 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:11:00,378 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:11:01,806 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:11:03,911 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:11:05,379 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:11:06,806 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:11:07,830 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:11:07,831 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:11:07,832 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:11:07,832 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:11:08,137 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:11:08,912 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:11:08,947 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:11:08,949 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:11:08,949 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:11:08,950 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:11:09,137 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:11:10,138 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 15:11:10,380 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:11:11,951 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:11:13,975 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:11:15,380 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:11:16,952 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:11:19,031 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:11:20,381 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:11:21,953 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:11:24,032 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:11:25,382 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:11:26,805 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:11:27,806 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:11:29,033 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:11:30,382 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:11:32,807 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:11:34,034 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:11:35,383 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:11:37,807 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:11:39,035 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:11:39,899 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:11:39,906 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:11:39,906 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:11:39,907 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:11:40,150 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:11:40,384 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:11:41,013 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:11:41,015 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:11:41,015 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:11:41,016 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:11:41,151 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:11:42,151 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 15:11:43,017 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:11:44,048 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:11:45,385 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:11:48,018 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:11:49,049 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:11:50,386 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:11:53,019 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:11:54,051 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:11:55,387 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:11:56,805 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:11:58,806 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:11:59,052 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:12:00,388 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:12:03,807 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:12:04,053 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:12:05,388 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:12:08,808 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:12:09,054 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:12:10,389 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:12:13,029 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:12:13,030 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:12:13,030 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:12:13,031 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:12:13,165 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:12:14,031 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:12:14,057 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:12:14,145 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:12:14,146 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:12:14,146 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:12:14,147 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:12:14,165 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:12:15,390 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:12:16,166 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log -2024-03-06 15:12:18,591 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history -2024-03-06 15:12:18,593 DEBUG SenderThread:137 [sender.py:send():382] send: history -2024-03-06 15:12:18,593 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record -2024-03-06 15:12:18,594 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-06 15:12:19,169 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json -2024-03-06 15:12:19,595 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:12:19,859 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:12:20,391 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:12:24,596 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:12:24,860 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:12:25,392 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:12:26,806 DEBUG SenderThread:137 [sender.py:send():382] send: stats -2024-03-06 15:12:29,807 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:12:29,861 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:12:30,393 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:12:34,808 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report -2024-03-06 15:12:34,862 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive -2024-03-06 15:12:35,394 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:34:50,913 INFO StreamThr :135 [internal.py:wandb_internal():86] W&B internal server running at pid: 135, started at: 2024-03-06 15:34:50.913034 +2024-03-06 15:34:50,915 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status +2024-03-06 15:34:51,720 INFO WriterThread:135 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240306_153450-l1uc7yan/run-l1uc7yan.wandb +2024-03-06 15:34:51,721 DEBUG SenderThread:135 [sender.py:send():382] send: header +2024-03-06 15:34:51,723 DEBUG SenderThread:135 [sender.py:send():382] send: run +2024-03-06 15:34:56,499 INFO SenderThread:135 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240306_153450-l1uc7yan/files +2024-03-06 15:34:56,499 INFO SenderThread:135 [sender.py:_start_run_threads():1136] run started: l1uc7yan with start time 1709739290.914286 +2024-03-06 15:34:56,499 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:34:56,508 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: check_version +2024-03-06 15:34:56,508 DEBUG SenderThread:135 [sender.py:send_request():409] send_request: check_version +2024-03-06 15:34:56,584 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: run_start +2024-03-06 15:34:56,595 DEBUG HandlerThread:135 [system_info.py:__init__():27] System info init +2024-03-06 15:34:56,595 DEBUG HandlerThread:135 [system_info.py:__init__():42] System info init done +2024-03-06 15:34:56,595 INFO HandlerThread:135 [system_monitor.py:start():194] Starting system monitor +2024-03-06 15:34:56,596 INFO SystemMonitor:135 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-06 15:34:56,596 INFO HandlerThread:135 [system_monitor.py:probe():214] Collecting system info +2024-03-06 15:34:56,597 INFO SystemMonitor:135 [interfaces.py:start():190] Started cpu monitoring +2024-03-06 15:34:56,597 INFO SystemMonitor:135 [interfaces.py:start():190] Started disk monitoring +2024-03-06 15:34:56,599 INFO SystemMonitor:135 [interfaces.py:start():190] Started gpu monitoring +2024-03-06 15:34:56,600 INFO SystemMonitor:135 [interfaces.py:start():190] Started memory monitoring +2024-03-06 15:34:56,600 INFO SystemMonitor:135 [interfaces.py:start():190] Started network monitoring +2024-03-06 15:34:56,605 DEBUG HandlerThread:135 [system_info.py:probe():151] Probing system +2024-03-06 15:34:56,607 DEBUG HandlerThread:135 [gitlib.py:_init_repo():56] git repository is invalid +2024-03-06 15:34:56,607 DEBUG HandlerThread:135 [system_info.py:probe():199] Probing system done +2024-03-06 15:34:56,607 DEBUG HandlerThread:135 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-03-06T15:34:56.605558', 'startedAt': '2024-03-06T15:34:50.906766', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': '64d026e37c1e', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.172, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.172, 'min': 0.0, 'max': 0.0}, {'current': 2000.172, 'min': 0.0, 'max': 0.0}, {'current': 2000.172, 'min': 0.0, 'max': 0.0}, {'current': 2000.172, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5519.631824493408}}, 'gpu': 'Tesla P100-PCIE-16GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'Tesla P100-PCIE-16GB', 'memory_total': 17179869184}], 'memory': {'total': 31.357559204101562}} +2024-03-06 15:34:56,608 INFO HandlerThread:135 [system_monitor.py:probe():224] Finished collecting system info +2024-03-06 15:34:56,608 INFO HandlerThread:135 [system_monitor.py:probe():227] Publishing system info +2024-03-06 15:34:56,608 DEBUG HandlerThread:135 [system_info.py:_save_conda():208] Saving list of conda packages installed into the current environment +2024-03-06 15:34:57,501 INFO Thread-12 :135 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240306_153450-l1uc7yan/files/conda-environment.yaml +2024-03-06 15:35:51,347 DEBUG HandlerThread:135 [system_info.py:_save_conda():220] Saving conda packages done +2024-03-06 15:35:51,347 INFO HandlerThread:135 [system_monitor.py:probe():229] Finished publishing system info +2024-03-06 15:35:51,356 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:35:51,356 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:35:51,356 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:35:51,356 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:35:51,356 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:35:51,356 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:35:51,356 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:35:51,356 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:35:51,357 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:35:51,357 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:35:51,357 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:35:51,357 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: python_packages +2024-03-06 15:35:51,357 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: stop_status +2024-03-06 15:35:51,357 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-06 15:35:51,357 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:35:51,359 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:35:51,359 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:35:51,359 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:35:51,359 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:35:51,359 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:35:51,359 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:35:51,359 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:35:51,360 DEBUG SenderThread:135 [sender.py:send_request():409] send_request: python_packages +2024-03-06 15:35:51,362 DEBUG SenderThread:135 [sender.py:send():382] send: telemetry +2024-03-06 15:35:51,362 DEBUG SenderThread:135 [sender.py:send():382] send: config +2024-03-06 15:35:51,363 DEBUG SenderThread:135 [sender.py:send_request():409] send_request: stop_status +2024-03-06 15:35:51,455 DEBUG SenderThread:135 [sender.py:send():382] send: metric +2024-03-06 15:35:51,456 DEBUG SenderThread:135 [sender.py:send():382] send: telemetry +2024-03-06 15:35:51,456 DEBUG SenderThread:135 [sender.py:send():382] send: metric +2024-03-06 15:35:51,456 WARNING SenderThread:135 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) +2024-03-06 15:35:51,456 DEBUG SenderThread:135 [sender.py:send():382] send: telemetry +2024-03-06 15:35:51,456 DEBUG SenderThread:135 [sender.py:send():382] send: files +2024-03-06 15:35:51,456 INFO SenderThread:135 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-03-06 15:35:51,518 INFO Thread-12 :135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_153450-l1uc7yan/files/conda-environment.yaml +2024-03-06 15:35:51,518 INFO Thread-12 :135 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240306_153450-l1uc7yan/files/wandb-metadata.json +2024-03-06 15:35:51,519 INFO Thread-12 :135 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240306_153450-l1uc7yan/files/requirements.txt +2024-03-06 15:35:51,636 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:35:51,723 INFO wandb-upload_0:135 [upload_job.py:push():131] Uploaded file /tmp/tmpuitk7rj1wandb/c0nbxcmk-wandb-metadata.json +2024-03-06 15:35:52,462 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:35:53,519 INFO Thread-12 :135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_153450-l1uc7yan/files/config.yaml +2024-03-06 15:35:54,624 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: partial_history +2024-03-06 15:35:54,626 DEBUG SenderThread:135 [sender.py:send():382] send: metric +2024-03-06 15:35:54,626 DEBUG SenderThread:135 [sender.py:send():382] send: metric +2024-03-06 15:35:54,626 DEBUG SenderThread:135 [sender.py:send():382] send: metric +2024-03-06 15:35:54,626 DEBUG SenderThread:135 [sender.py:send():382] send: metric +2024-03-06 15:35:54,626 DEBUG SenderThread:135 [sender.py:send():382] send: history +2024-03-06 15:35:54,627 DEBUG SenderThread:135 [sender.py:send_request():409] send_request: summary_record +2024-03-06 15:35:54,628 INFO SenderThread:135 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-06 15:35:55,520 INFO Thread-12 :135 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240306_153450-l1uc7yan/files/wandb-summary.json +2024-03-06 15:35:56,601 DEBUG SystemMonitor:135 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-06 15:35:56,602 DEBUG SenderThread:135 [sender.py:send():382] send: stats +2024-03-06 15:35:56,644 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:35:58,253 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: pause +2024-03-06 15:35:58,253 INFO HandlerThread:135 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-06 15:35:58,254 INFO HandlerThread:135 [system_monitor.py:finish():203] Stopping system monitor +2024-03-06 15:35:58,254 INFO HandlerThread:135 [interfaces.py:finish():202] Joined cpu monitor +2024-03-06 15:35:58,254 DEBUG SystemMonitor:135 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-06 15:35:58,254 DEBUG SystemMonitor:135 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-06 15:35:58,255 INFO HandlerThread:135 [interfaces.py:finish():202] Joined disk monitor +2024-03-06 15:35:58,260 INFO HandlerThread:135 [interfaces.py:finish():202] Joined gpu monitor +2024-03-06 15:35:58,260 INFO HandlerThread:135 [interfaces.py:finish():202] Joined memory monitor +2024-03-06 15:35:58,260 INFO HandlerThread:135 [interfaces.py:finish():202] Joined network monitor +2024-03-06 15:35:58,261 DEBUG SenderThread:135 [sender.py:send():382] send: stats +2024-03-06 15:35:58,261 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:36:01,662 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:36:03,262 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:36:06,664 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:36:08,263 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:36:11,664 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:36:13,264 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:36:16,665 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:36:18,265 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:36:21,666 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:36:22,919 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: resume +2024-03-06 15:36:22,919 INFO HandlerThread:135 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-06 15:36:22,919 INFO HandlerThread:135 [system_monitor.py:start():194] Starting system monitor +2024-03-06 15:36:22,919 INFO SystemMonitor:135 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-06 15:36:22,920 INFO SystemMonitor:135 [interfaces.py:start():190] Started cpu monitoring +2024-03-06 15:36:22,921 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: pause +2024-03-06 15:36:22,921 INFO HandlerThread:135 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-06 15:36:22,921 INFO HandlerThread:135 [system_monitor.py:finish():203] Stopping system monitor +2024-03-06 15:36:22,921 INFO SystemMonitor:135 [interfaces.py:start():190] Started disk monitoring +2024-03-06 15:36:22,922 INFO HandlerThread:135 [interfaces.py:finish():202] Joined cpu monitor +2024-03-06 15:36:22,923 DEBUG SystemMonitor:135 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-06 15:36:22,923 INFO HandlerThread:135 [interfaces.py:finish():202] Joined disk monitor +2024-03-06 15:36:22,924 DEBUG SystemMonitor:135 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-06 15:36:22,924 DEBUG SystemMonitor:135 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-06 15:36:22,925 DEBUG SenderThread:135 [sender.py:send():382] send: stats +2024-03-06 15:36:23,931 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:36:24,529 INFO Thread-12 :135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_153450-l1uc7yan/files/config.yaml +2024-03-06 15:36:26,667 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:36:26,944 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: resume +2024-03-06 15:36:26,944 INFO HandlerThread:135 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-06 15:36:26,944 INFO HandlerThread:135 [system_monitor.py:start():194] Starting system monitor +2024-03-06 15:36:26,944 INFO SystemMonitor:135 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-06 15:36:26,945 INFO SystemMonitor:135 [interfaces.py:start():190] Started cpu monitoring +2024-03-06 15:36:26,948 INFO SystemMonitor:135 [interfaces.py:start():190] Started disk monitoring +2024-03-06 15:36:26,948 INFO SystemMonitor:135 [interfaces.py:start():190] Started gpu monitoring +2024-03-06 15:36:26,949 INFO SystemMonitor:135 [interfaces.py:start():190] Started memory monitoring +2024-03-06 15:36:26,949 INFO SystemMonitor:135 [interfaces.py:start():190] Started network monitoring +2024-03-06 15:36:27,530 INFO Thread-12 :135 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240306_153450-l1uc7yan/files/output.log +2024-03-06 15:36:28,075 DEBUG SenderThread:135 [sender.py:send():382] send: config +2024-03-06 15:36:28,076 DEBUG SenderThread:135 [sender.py:send():382] send: metric +2024-03-06 15:36:28,076 DEBUG SenderThread:135 [sender.py:send():382] send: metric +2024-03-06 15:36:28,076 WARNING SenderThread:135 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) +2024-03-06 15:36:29,531 INFO Thread-12 :135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_153450-l1uc7yan/files/output.log +2024-03-06 15:36:30,077 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:36:31,941 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:36:35,078 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:36:36,942 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:36:40,079 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:36:41,943 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:36:45,080 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:36:46,944 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:36:50,080 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:36:51,945 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:36:55,086 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:36:55,320 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: partial_history +2024-03-06 15:36:55,321 DEBUG SenderThread:135 [sender.py:send():382] send: history +2024-03-06 15:36:55,321 DEBUG SenderThread:135 [sender.py:send_request():409] send_request: summary_record +2024-03-06 15:36:55,323 INFO SenderThread:135 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-06 15:36:55,540 INFO Thread-12 :135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_153450-l1uc7yan/files/config.yaml +2024-03-06 15:36:55,541 INFO Thread-12 :135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_153450-l1uc7yan/files/wandb-summary.json +2024-03-06 15:36:56,959 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:36:57,345 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: partial_history +2024-03-06 15:36:57,348 DEBUG SenderThread:135 [sender.py:send():382] send: metric +2024-03-06 15:36:57,348 DEBUG SenderThread:135 [sender.py:send():382] send: metric +2024-03-06 15:36:57,348 DEBUG SenderThread:135 [sender.py:send():382] send: metric +2024-03-06 15:36:57,348 DEBUG SenderThread:135 [sender.py:send():382] send: metric +2024-03-06 15:36:57,349 DEBUG SenderThread:135 [sender.py:send():382] send: metric +2024-03-06 15:36:57,349 DEBUG SenderThread:135 [sender.py:send():382] send: metric +2024-03-06 15:36:57,349 DEBUG SenderThread:135 [sender.py:send():382] send: metric +2024-03-06 15:36:57,349 DEBUG SenderThread:135 [sender.py:send():382] send: metric +2024-03-06 15:36:57,349 DEBUG SenderThread:135 [sender.py:send():382] send: history +2024-03-06 15:36:57,349 DEBUG SenderThread:135 [sender.py:send_request():409] send_request: summary_record +2024-03-06 15:36:57,349 INFO SenderThread:135 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-06 15:36:57,541 INFO Thread-12 :135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_153450-l1uc7yan/files/wandb-summary.json +2024-03-06 15:36:59,542 INFO Thread-12 :135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_153450-l1uc7yan/files/output.log +2024-03-06 15:37:00,351 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:37:02,046 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:37:05,351 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:37:07,048 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:37:10,352 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:37:12,053 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:37:15,353 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:37:17,054 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive +2024-03-06 15:37:20,354 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: status_report +2024-03-06 15:37:22,055 DEBUG HandlerThread:135 [handler.py:handle_request():146] handle_request: keepalive