Training in progress, epoch 2

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:24d66d5c08239982b291b18547bf7a3b413e1f5f8f379ea71084959d0e365310
+oid sha256:6c3dd47b4c910468e268d03db7d5f3bef3a00253c6ade9204cfd15cf73968149
 size 502675828

runs/Mar06_15-06-49_41759fa8e6ad/events.out.tfevents.1709737609.41759fa8e6ad.34.4

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eda21a91222defd1d9c04a5595afc81f0a4f282670fd283512e28c8aabc0465c
-size 5840
+oid sha256:997076216899c04f517674f013430d160c7582ddfa4cc8c986d74d9741ca7ec2
+size 6510

wandb/debug-internal.log

@@ -707,3 +707,35 @@
 2024-03-06 15:07:31,922 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
 2024-03-06 15:07:33,800 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
 2024-03-06 15:07:35,340 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:36,923 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:07:38,801 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:40,341 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:41,924 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:07:43,802 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:45,343 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:46,925 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:07:48,803 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:50,190 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:07:50,191 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:07:50,191 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:07:50,192 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:07:50,343 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:51,049 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:07:51,303 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:07:51,305 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:07:51,305 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:07:51,306 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:07:52,049 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:07:52,385 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:07:54,050 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log
+2024-03-06 15:07:54,308 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:55,344 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:56,799 DEBUG   SenderThread:137 [sender.py:send():382] send: stats
+2024-03-06 15:07:57,427 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:07:59,800 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:00,345 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:02,430 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:08:04,801 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:05,345 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:07,431 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:08:09,802 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report

wandb/run-20240306_145424-trm7fvg4/logs/debug-internal.log

@@ -758,3 +758,35 @@ wandb.errors.AuthenticationError: The API key you provided is either invalid or
 2024-03-06 15:07:31,922 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
 2024-03-06 15:07:33,800 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
 2024-03-06 15:07:35,340 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:36,923 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:07:38,801 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:40,341 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:41,924 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:07:43,802 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:45,343 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:46,925 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:07:48,803 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:50,190 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:07:50,191 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:07:50,191 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:07:50,192 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:07:50,343 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:51,049 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:07:51,303 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:07:51,305 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:07:51,305 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:07:51,306 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:07:52,049 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:07:52,385 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:07:54,050 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log
+2024-03-06 15:07:54,308 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:55,344 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:56,799 DEBUG   SenderThread:137 [sender.py:send():382] send: stats
+2024-03-06 15:07:57,427 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:07:59,800 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:00,345 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:02,430 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:08:04,801 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:05,345 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:07,431 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:08:09,802 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report

wandb/run-20240306_145455-h1uv5tyi/files/output.log

@@ -36,3 +36,6 @@ You should probably TRAIN this model on a down-stream task to be able to use it
 Some weights of MegatronBertForSequenceClassification were not initialized from the model checkpoint at mmukh/SOBertBase and are newly initialized: ['bert.embeddings.token_type_embeddings.weight', 'bert.pooler.dense.bias', 'bert.pooler.dense.weight', 'classifier.bias', 'classifier.weight']
 You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
 /opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
+  _warn_prf(average, modifier, msg_start, len(result))
+/opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
+  _warn_prf(average, modifier, msg_start, len(result))

wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 1.8204, "train/grad_norm": 5.6789045333862305, "train/learning_rate": 2.7000000000000002e-05, "train/epoch": 1.0, "train/global_step": 31, "_timestamp": 1709737638.1537485, "_runtime": 742.4349915981293, "_step": 27, "eval/loss": 1.7834787368774414, "eval/accuracy": 0.32727272727272727, "eval/precision": 0.11515151515151514, "eval/recall": 0.32727272727272727, "eval/f1": 0.17036114570361147, "eval/runtime": 1.1251, "eval/samples_per_second": 48.886, "eval/steps_per_second": 3.555, "train/train_runtime": 237.4526, "train/train_samples_per_second": 10.339, "train/train_steps_per_second": 1.306, "train/total_flos": 645966638976000.0, "train/train_loss": 1.7031736066264491}
+{"train/loss": 1.7527, "train/grad_norm": 5.8667988777160645, "train/learning_rate": 2.4e-05, "train/epoch": 2.0, "train/global_step": 62, "_timestamp": 1709737671.3031292, "_runtime": 775.5843722820282, "_step": 29, "eval/loss": 1.773923397064209, "eval/accuracy": 0.34545454545454546, "eval/precision": 0.11933884297520661, "eval/recall": 0.34545454545454546, "eval/f1": 0.17739557739557738, "eval/runtime": 1.1095, "eval/samples_per_second": 49.57, "eval/steps_per_second": 3.605, "train/train_runtime": 237.4526, "train/train_samples_per_second": 10.339, "train/train_steps_per_second": 1.306, "train/total_flos": 645966638976000.0, "train/train_loss": 1.7031736066264491}

wandb/run-20240306_145455-h1uv5tyi/logs/debug-internal.log

@@ -707,3 +707,35 @@
 2024-03-06 15:07:31,922 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
 2024-03-06 15:07:33,800 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
 2024-03-06 15:07:35,340 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:36,923 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:07:38,801 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:40,341 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:41,924 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:07:43,802 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:45,343 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:46,925 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:07:48,803 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:50,190 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:07:50,191 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:07:50,191 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:07:50,192 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:07:50,343 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:51,049 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:07:51,303 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:07:51,305 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:07:51,305 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:07:51,306 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:07:52,049 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:07:52,385 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:07:54,050 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log
+2024-03-06 15:07:54,308 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:55,344 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:07:56,799 DEBUG   SenderThread:137 [sender.py:send():382] send: stats
+2024-03-06 15:07:57,427 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:07:59,800 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:00,345 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:02,430 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:08:04,801 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:05,345 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:08:07,431 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:08:09,802 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report