Training in progress, epoch 4

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:595387e04ed4c89ad2261cabc84ada41dc85f9c625a8fe84e799366424331531
+oid sha256:a22a4c92bbf3913b0c34c960c4459c97dcd8c04f67f6b1ce1e321329097633e7
 size 502675828

runs/Mar06_14-59-58_41759fa8e6ad/events.out.tfevents.1709737199.41759fa8e6ad.34.3

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f1a7c761ef214f4dc02a43fc9c266f7bdb50e1f21385d23682b2bdd1f69d5db9
-size 7191
+oid sha256:c13270d753577a6b526afe9d18f1a90d706f21e6e8131ba9f2e77ded1722f291
+size 7874

wandb/debug-internal.log

@@ -429,3 +429,39 @@
 2024-03-06 15:01:50,276 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
 2024-03-06 15:01:50,912 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
 2024-03-06 15:01:52,677 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:01:55,277 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:01:55,915 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:01:56,789 DEBUG   SenderThread:137 [sender.py:send():382] send: stats
+2024-03-06 15:01:57,790 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:00,278 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:00,916 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:02:02,791 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:05,279 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:05,917 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:02:07,792 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:08,680 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:02:08,681 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:02:08,681 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:02:08,682 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:02:08,905 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:02:09,794 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:02:09,801 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:02:09,801 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:02:09,802 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:02:09,905 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:02:10,280 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:10,961 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:02:11,906 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log
+2024-03-06 15:02:12,803 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:15,281 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:16,026 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:02:17,804 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:20,282 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:21,111 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:02:22,804 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:25,283 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:26,118 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:02:26,790 DEBUG   SenderThread:137 [sender.py:send():382] send: stats
+2024-03-06 15:02:28,791 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:30,283 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:31,123 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive

wandb/run-20240306_145424-trm7fvg4/logs/debug-internal.log

@@ -480,3 +480,39 @@ wandb.errors.AuthenticationError: The API key you provided is either invalid or
 2024-03-06 15:01:50,276 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
 2024-03-06 15:01:50,912 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
 2024-03-06 15:01:52,677 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:01:55,277 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:01:55,915 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:01:56,789 DEBUG   SenderThread:137 [sender.py:send():382] send: stats
+2024-03-06 15:01:57,790 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:00,278 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:00,916 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:02:02,791 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:05,279 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:05,917 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:02:07,792 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:08,680 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:02:08,681 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:02:08,681 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:02:08,682 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:02:08,905 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:02:09,794 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:02:09,801 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:02:09,801 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:02:09,802 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:02:09,905 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:02:10,280 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:10,961 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:02:11,906 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log
+2024-03-06 15:02:12,803 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:15,281 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:16,026 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:02:17,804 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:20,282 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:21,111 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:02:22,804 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:25,283 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:26,118 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:02:26,790 DEBUG   SenderThread:137 [sender.py:send():382] send: stats
+2024-03-06 15:02:28,791 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:30,283 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:31,123 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive

wandb/run-20240306_145455-h1uv5tyi/files/output.log

@@ -20,3 +20,6 @@ Checkpoint destination directory /kaggle/working/checkpoint-62 already exists an
 Checkpoint destination directory /kaggle/working/checkpoint-124 already exists and is non-empty. Saving will proceed but saved results may be invalid.
 /opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
   _warn_prf(average, modifier, msg_start, len(result))
+Checkpoint destination directory /kaggle/working/checkpoint-186 already exists and is non-empty. Saving will proceed but saved results may be invalid.
+/opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
+  _warn_prf(average, modifier, msg_start, len(result))

wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 1.7235, "train/grad_norm": 8.239474296569824, "train/learning_rate": 7.0161290322580654e-06, "train/epoch": 3.0, "train/global_step": 186, "_timestamp": 1709737296.670676, "_runtime": 400.9519190788269, "_step": 17, "eval/loss": 1.7306572198867798, "eval/accuracy": 0.36363636363636365, "eval/precision": 0.26503496503496504, "eval/recall": 0.36363636363636365, "eval/f1": 0.23334186939820742, "eval/runtime": 1.1077, "eval/samples_per_second": 49.653, "eval/steps_per_second": 6.319, "train/train_runtime": 237.4526, "train/train_samples_per_second": 10.339, "train/train_steps_per_second": 1.306, "train/total_flos": 645966638976000.0, "train/train_loss": 1.7031736066264491}
+{"train/loss": 1.6957, "train/grad_norm": 11.199089050292969, "train/learning_rate": 6.016129032258065e-06, "train/epoch": 4.0, "train/global_step": 248, "_timestamp": 1709737329.7935672, "_runtime": 434.07481026649475, "_step": 19, "eval/loss": 1.7372241020202637, "eval/accuracy": 0.36363636363636365, "eval/precision": 0.2310160427807487, "eval/recall": 0.36363636363636365, "eval/f1": 0.23220779220779222, "eval/runtime": 1.1099, "eval/samples_per_second": 49.555, "eval/steps_per_second": 6.307, "train/train_runtime": 237.4526, "train/train_samples_per_second": 10.339, "train/train_steps_per_second": 1.306, "train/total_flos": 645966638976000.0, "train/train_loss": 1.7031736066264491}

wandb/run-20240306_145455-h1uv5tyi/logs/debug-internal.log

@@ -429,3 +429,39 @@
 2024-03-06 15:01:50,276 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
 2024-03-06 15:01:50,912 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
 2024-03-06 15:01:52,677 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:01:55,277 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:01:55,915 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:01:56,789 DEBUG   SenderThread:137 [sender.py:send():382] send: stats
+2024-03-06 15:01:57,790 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:00,278 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:00,916 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:02:02,791 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:05,279 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:05,917 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:02:07,792 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:08,680 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:02:08,681 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:02:08,681 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:02:08,682 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:02:08,905 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:02:09,794 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:02:09,801 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:02:09,801 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:02:09,802 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:02:09,905 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:02:10,280 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:10,961 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:02:11,906 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log
+2024-03-06 15:02:12,803 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:15,281 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:16,026 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:02:17,804 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:20,282 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:21,111 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:02:22,804 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:25,283 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:26,118 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:02:26,790 DEBUG   SenderThread:137 [sender.py:send():382] send: stats
+2024-03-06 15:02:28,791 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:30,283 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:02:31,123 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive