Training in progress, epoch 4

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a57e47fb384faf33138a4416f9e5a2be79f2b4be384456bdcb3d266154cfa06b
+oid sha256:184f2bb730d40284f45451db2d209e9f5cc74e6b4657c771fb9eb210675c8cc0
 size 502675828

runs/Mar06_15-06-49_41759fa8e6ad/events.out.tfevents.1709737609.41759fa8e6ad.34.4

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d161e64b9478e067452d03d91fc62fce3591d774f8a60eaddbffb678e0cc370c
-size 7180
+oid sha256:987af32ac547c8583921bb99d3d3a0ce9deb1ac55f15962995271473124e96d4
+size 7850

wandb/debug-internal.log

@@ -769,3 +769,37 @@
 2024-03-06 15:08:35,802 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
 2024-03-06 15:08:37,496 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
 2024-03-06 15:08:40,352 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:40,803 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:42,497 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:08:45,353 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:45,804 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:47,498 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:08:50,354 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:50,805 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:52,500 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:08:55,355 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:55,488 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:08:55,490 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:08:55,490 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:08:55,490 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:08:56,077 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:08:56,491 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:56,604 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:08:56,605 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:08:56,605 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:08:56,606 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:08:56,801 DEBUG   SenderThread:137 [sender.py:send():382] send: stats
+2024-03-06 15:08:57,077 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:08:58,077 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log
+2024-03-06 15:08:58,542 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:08:58,543 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:09:00,356 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:01,802 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:03,722 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:09:05,357 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:06,803 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:08,723 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:09:10,358 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:11,804 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:13,724 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:09:15,358 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report

wandb/run-20240306_145424-trm7fvg4/logs/debug-internal.log

@@ -820,3 +820,37 @@ wandb.errors.AuthenticationError: The API key you provided is either invalid or
 2024-03-06 15:08:35,802 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
 2024-03-06 15:08:37,496 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
 2024-03-06 15:08:40,352 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:40,803 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:42,497 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:08:45,353 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:45,804 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:47,498 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:08:50,354 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:50,805 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:52,500 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:08:55,355 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:55,488 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:08:55,490 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:08:55,490 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:08:55,490 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:08:56,077 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:08:56,491 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:56,604 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:08:56,605 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:08:56,605 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:08:56,606 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:08:56,801 DEBUG   SenderThread:137 [sender.py:send():382] send: stats
+2024-03-06 15:08:57,077 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:08:58,077 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log
+2024-03-06 15:08:58,542 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:08:58,543 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:09:00,356 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:01,802 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:03,722 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:09:05,357 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:06,803 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:08,723 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:09:10,358 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:11,804 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:13,724 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:09:15,358 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report

wandb/run-20240306_145455-h1uv5tyi/files/output.log

@@ -41,3 +41,6 @@ You should probably TRAIN this model on a down-stream task to be able to use it
   _warn_prf(average, modifier, msg_start, len(result))
 Checkpoint destination directory /kaggle/working/checkpoint-62 already exists and is non-empty. Saving will proceed but saved results may be invalid.
 /opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
+  _warn_prf(average, modifier, msg_start, len(result))
+/opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
+  _warn_prf(average, modifier, msg_start, len(result))

wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 1.7018, "train/grad_norm": 5.185774326324463, "train/learning_rate": 2.1e-05, "train/epoch": 3.0, "train/global_step": 93, "_timestamp": 1709737703.506952, "_runtime": 807.7881951332092, "_step": 31, "eval/loss": 1.739591121673584, "eval/accuracy": 0.34545454545454546, "eval/precision": 0.18308668076109938, "eval/recall": 0.34545454545454546, "eval/f1": 0.23284457478005866, "eval/runtime": 1.1118, "eval/samples_per_second": 49.471, "eval/steps_per_second": 3.598, "train/train_runtime": 237.4526, "train/train_samples_per_second": 10.339, "train/train_steps_per_second": 1.306, "train/total_flos": 645966638976000.0, "train/train_loss": 1.7031736066264491}
+{"train/loss": 1.652, "train/grad_norm": 5.863879680633545, "train/learning_rate": 1.8e-05, "train/epoch": 4.0, "train/global_step": 124, "_timestamp": 1709737736.6036682, "_runtime": 840.8849112987518, "_step": 33, "eval/loss": 1.7384552955627441, "eval/accuracy": 0.34545454545454546, "eval/precision": 0.19107438016528924, "eval/recall": 0.34545454545454546, "eval/f1": 0.2349731077003804, "eval/runtime": 1.1115, "eval/samples_per_second": 49.484, "eval/steps_per_second": 3.599, "train/train_runtime": 237.4526, "train/train_samples_per_second": 10.339, "train/train_steps_per_second": 1.306, "train/total_flos": 645966638976000.0, "train/train_loss": 1.7031736066264491}

wandb/run-20240306_145455-h1uv5tyi/logs/debug-internal.log

@@ -769,3 +769,37 @@
 2024-03-06 15:08:35,802 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
 2024-03-06 15:08:37,496 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
 2024-03-06 15:08:40,352 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:40,803 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:42,497 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:08:45,353 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:45,804 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:47,498 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:08:50,354 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:50,805 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:52,500 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:08:55,355 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:55,488 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:08:55,490 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:08:55,490 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:08:55,490 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:08:56,077 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:08:56,491 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:56,604 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:08:56,605 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:08:56,605 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:08:56,606 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:08:56,801 DEBUG   SenderThread:137 [sender.py:send():382] send: stats
+2024-03-06 15:08:57,077 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:08:58,077 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log
+2024-03-06 15:08:58,542 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:08:58,543 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:09:00,356 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:01,802 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:03,722 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:09:05,357 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:06,803 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:08,723 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:09:10,358 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:11,804 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:13,724 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:09:15,358 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report