Training in progress, epoch 4

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0d1f56ea1e05ad74bc53455acacb664910c8f783d5c5824ca135c98f200c5841
+oid sha256:db18a2fa5759bcda6556cf606b69ba1426d08b1c4826aa94b4d8f7928cde32bf
 size 502675828

runs/Apr10_14-58-13_9088970cc926/events.out.tfevents.1712761095.9088970cc926.34.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a6dbe48f5d296eff27d4ea13c24bffcf6c2b9390dc939c92828ea74298795d61
-size 7209
+oid sha256:9de86818b9e941cf73554f120d6dbecbcf09dcb08ff2562f60614b69097e80e6
+size 7892

wandb/debug-internal.log

@@ -194,3 +194,39 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-10 15:04:56,841 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages
 2024-04-10 15:04:56,842 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: stop_status
 2024-04-10 15:04:59,133 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:05:04,133 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:05:09,135 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:05:11,842 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-10 15:05:11,842 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-10 15:05:11,843 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: stop_status
+2024-04-10 15:05:15,129 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:05:20,131 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:05:25,131 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:05:25,779 DEBUG   SenderThread:100 [sender.py:send():379] send: stats
+2024-04-10 15:05:26,841 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-10 15:05:26,842 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-10 15:05:26,842 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: stop_status
+2024-04-10 15:05:31,121 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:05:36,122 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:05:37,954 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-10 15:05:37,955 DEBUG   SenderThread:100 [sender.py:send():379] send: history
+2024-04-10 15:05:37,955 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: summary_record
+2024-04-10 15:05:37,957 INFO    SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-10 15:05:38,745 INFO    Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json
+2024-04-10 15:05:40,795 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-10 15:05:40,796 DEBUG   SenderThread:100 [sender.py:send():379] send: history
+2024-04-10 15:05:40,797 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: summary_record
+2024-04-10 15:05:40,797 INFO    SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-10 15:05:41,746 INFO    Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json
+2024-04-10 15:05:41,798 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:05:42,212 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-10 15:05:42,213 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: stop_status
+2024-04-10 15:05:42,253 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-10 15:05:42,747 INFO    Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/output.log
+2024-04-10 15:05:47,489 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:05:52,490 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:05:55,780 DEBUG   SenderThread:100 [sender.py:send():379] send: stats
+2024-04-10 15:05:56,977 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-10 15:05:56,996 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-10 15:05:56,997 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: stop_status
+2024-04-10 15:05:58,215 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report

wandb/run-20240410_150123-cuua991r/files/output.log

@@ -2,4 +2,6 @@
   _warn_prf(average, modifier, msg_start, len(result))
 /opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
   _warn_prf(average, modifier, msg_start, len(result))
+/opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
+  _warn_prf(average, modifier, msg_start, len(result))
 /opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.

wandb/run-20240410_150123-cuua991r/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 1.764, "train/grad_norm": 6.489156246185303, "train/learning_rate": 8.033898305084746e-06, "train/epoch": 3.0, "train/global_step": 354, "_timestamp": 1712761480.346977, "_runtime": 196.796245098114, "_step": 5, "eval/loss": 1.8461236953735352, "eval/accuracy": 0.24647887323943662, "eval/precision": 0.23441761681198298, "eval/recall": 0.24647887323943662, "eval/f1": 0.1976516887500408, "eval/runtime": 2.8417, "eval/samples_per_second": 49.97, "eval/steps_per_second": 6.334}
+{"train/loss": 1.6949, "train/grad_norm": 8.816020011901855, "train/learning_rate": 4.067796610169492e-06, "train/epoch": 4.0, "train/global_step": 472, "_timestamp": 1712761540.7952058, "_runtime": 257.2444739341736, "_step": 7, "eval/loss": 1.831845998764038, "eval/accuracy": 0.24647887323943662, "eval/precision": 0.2143956274963212, "eval/recall": 0.24647887323943662, "eval/f1": 0.19645709307390735, "eval/runtime": 2.837, "eval/samples_per_second": 50.052, "eval/steps_per_second": 6.345}

wandb/run-20240410_150123-cuua991r/logs/debug-internal.log

@@ -194,3 +194,39 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-10 15:04:56,841 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages
 2024-04-10 15:04:56,842 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: stop_status
 2024-04-10 15:04:59,133 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:05:04,133 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:05:09,135 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:05:11,842 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-10 15:05:11,842 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-10 15:05:11,843 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: stop_status
+2024-04-10 15:05:15,129 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:05:20,131 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:05:25,131 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:05:25,779 DEBUG   SenderThread:100 [sender.py:send():379] send: stats
+2024-04-10 15:05:26,841 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-10 15:05:26,842 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-10 15:05:26,842 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: stop_status
+2024-04-10 15:05:31,121 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:05:36,122 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:05:37,954 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-10 15:05:37,955 DEBUG   SenderThread:100 [sender.py:send():379] send: history
+2024-04-10 15:05:37,955 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: summary_record
+2024-04-10 15:05:37,957 INFO    SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-10 15:05:38,745 INFO    Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json
+2024-04-10 15:05:40,795 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-10 15:05:40,796 DEBUG   SenderThread:100 [sender.py:send():379] send: history
+2024-04-10 15:05:40,797 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: summary_record
+2024-04-10 15:05:40,797 INFO    SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-10 15:05:41,746 INFO    Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json
+2024-04-10 15:05:41,798 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:05:42,212 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-10 15:05:42,213 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: stop_status
+2024-04-10 15:05:42,253 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-10 15:05:42,747 INFO    Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/output.log
+2024-04-10 15:05:47,489 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:05:52,490 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:05:55,780 DEBUG   SenderThread:100 [sender.py:send():379] send: stats
+2024-04-10 15:05:56,977 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-10 15:05:56,996 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-10 15:05:56,997 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: stop_status
+2024-04-10 15:05:58,215 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report