Training in progress, epoch 7

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa4e636578a89d52fa8a79b790d5fbae475d52c5002e01cc9ca1bad925a5a52a
+oid sha256:0655282b3c26d39cb6f722edab8cf5fb3f46b98806cf214937baa673bee90259
 size 502675828

runs/Mar06_14-59-58_41759fa8e6ad/events.out.tfevents.1709737199.41759fa8e6ad.34.3

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b50afec718284985d69f44e0a7adf4cd6aebf6a9d3c4d4b7f24370571c666aa7
-size 9240
+oid sha256:13df7ac5ba1e025d4d9c8a5ee993d45348bebba0863cba62d3de869e421b982c
+size 9923

wandb/debug-internal.log

@@ -526,3 +526,42 @@
 2024-03-06 15:03:30,293 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
 2024-03-06 15:03:30,793 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
 2024-03-06 15:03:31,488 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:03:35,294 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:35,794 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:36,489 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:03:40,294 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:40,795 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:41,490 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:03:45,295 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:45,796 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:46,492 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:03:50,296 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:50,797 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:51,157 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:03:51,158 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:03:51,158 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:03:51,159 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:03:51,500 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:03:51,944 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:03:52,269 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:03:52,270 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:03:52,271 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:03:52,271 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:03:52,945 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:03:53,945 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log
+2024-03-06 15:03:55,297 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:56,272 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:56,545 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:03:56,793 DEBUG   SenderThread:137 [sender.py:send():382] send: stats
+2024-03-06 15:04:00,298 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:04:01,546 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:04:01,794 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:04:05,299 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:04:06,547 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:04:06,795 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:04:10,300 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:04:11,548 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:04:11,796 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:04:15,301 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:04:16,549 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:04:16,797 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report

wandb/run-20240306_145424-trm7fvg4/logs/debug-internal.log

@@ -577,3 +577,42 @@ wandb.errors.AuthenticationError: The API key you provided is either invalid or
 2024-03-06 15:03:30,293 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
 2024-03-06 15:03:30,793 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
 2024-03-06 15:03:31,488 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:03:35,294 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:35,794 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:36,489 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:03:40,294 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:40,795 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:41,490 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:03:45,295 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:45,796 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:46,492 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:03:50,296 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:50,797 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:51,157 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:03:51,158 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:03:51,158 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:03:51,159 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:03:51,500 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:03:51,944 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:03:52,269 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:03:52,270 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:03:52,271 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:03:52,271 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:03:52,945 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:03:53,945 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log
+2024-03-06 15:03:55,297 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:56,272 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:56,545 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:03:56,793 DEBUG   SenderThread:137 [sender.py:send():382] send: stats
+2024-03-06 15:04:00,298 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:04:01,546 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:04:01,794 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:04:05,299 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:04:06,547 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:04:06,795 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:04:10,300 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:04:11,548 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:04:11,796 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:04:15,301 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:04:16,549 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:04:16,797 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report

wandb/run-20240306_145455-h1uv5tyi/files/output.log

@@ -27,4 +27,6 @@ Checkpoint destination directory /kaggle/working/checkpoint-248 already exists a
 /opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
   _warn_prf(average, modifier, msg_start, len(result))
 Checkpoint destination directory /kaggle/working/checkpoint-310 already exists and is non-empty. Saving will proceed but saved results may be invalid.
+/opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
+  _warn_prf(average, modifier, msg_start, len(result))
 /opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.

wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 1.6727, "train/grad_norm": 14.989230155944824, "train/learning_rate": 4.032258064516129e-06, "train/epoch": 6.0, "train/global_step": 372, "_timestamp": 1709737396.2640631, "_runtime": 500.5453062057495, "_step": 23, "eval/loss": 1.7222378253936768, "eval/accuracy": 0.38181818181818183, "eval/precision": 0.2246376811594203, "eval/recall": 0.38181818181818183, "eval/f1": 0.2638161838161838, "eval/runtime": 1.1081, "eval/samples_per_second": 49.633, "eval/steps_per_second": 6.317, "train/train_runtime": 237.4526, "train/train_samples_per_second": 10.339, "train/train_steps_per_second": 1.306, "train/total_flos": 645966638976000.0, "train/train_loss": 1.7031736066264491}
+{"train/loss": 1.63, "train/grad_norm": 15.767184257507324, "train/learning_rate": 3.0322580645161295e-06, "train/epoch": 7.0, "train/global_step": 434, "_timestamp": 1709737432.2685022, "_runtime": 536.5497453212738, "_step": 25, "eval/loss": 1.7277967929840088, "eval/accuracy": 0.38181818181818183, "eval/precision": 0.2177777777777778, "eval/recall": 0.38181818181818183, "eval/f1": 0.2628615702479339, "eval/runtime": 1.108, "eval/samples_per_second": 49.638, "eval/steps_per_second": 6.318, "train/train_runtime": 237.4526, "train/train_samples_per_second": 10.339, "train/train_steps_per_second": 1.306, "train/total_flos": 645966638976000.0, "train/train_loss": 1.7031736066264491}

wandb/run-20240306_145455-h1uv5tyi/logs/debug-internal.log

@@ -526,3 +526,42 @@
 2024-03-06 15:03:30,293 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
 2024-03-06 15:03:30,793 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
 2024-03-06 15:03:31,488 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:03:35,294 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:35,794 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:36,489 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:03:40,294 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:40,795 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:41,490 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:03:45,295 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:45,796 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:46,492 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:03:50,296 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:50,797 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:51,157 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:03:51,158 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:03:51,158 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:03:51,159 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:03:51,500 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:03:51,944 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:03:52,269 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:03:52,270 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:03:52,271 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:03:52,271 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:03:52,945 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:03:53,945 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log
+2024-03-06 15:03:55,297 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:56,272 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:03:56,545 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:03:56,793 DEBUG   SenderThread:137 [sender.py:send():382] send: stats
+2024-03-06 15:04:00,298 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:04:01,546 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:04:01,794 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:04:05,299 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:04:06,547 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:04:06,795 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:04:10,300 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:04:11,548 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:04:11,796 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:04:15,301 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:04:16,549 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:04:16,797 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report