Training in progress, step 3500
Browse files- model.safetensors +1 -1
- runs/Jan12_02-28-34_ea19ce76d6cc/events.out.tfevents.1705026515.ea19ce76d6cc.10822.0 +2 -2
- wandb/debug-internal.log +44 -0
- wandb/run-20240112_022837-twhkj0lf/files/output.log +3 -0
- wandb/run-20240112_022837-twhkj0lf/files/wandb-summary.json +1 -1
- wandb/run-20240112_022837-twhkj0lf/logs/debug-internal.log +44 -0
- wandb/run-20240112_022837-twhkj0lf/run-twhkj0lf.wandb +0 -0
model.safetensors
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 328693404
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:91f02f501f1865950c5b71eac1056d9f4e69a7df89f1a18e95f167d7784c95e7
|
| 3 |
size 328693404
|
runs/Jan12_02-28-34_ea19ce76d6cc/events.out.tfevents.1705026515.ea19ce76d6cc.10822.0
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c2664f8416b6b127799ad8e98c3a05c524a983e995d4ee8b74443adcd36ba85e
|
| 3 |
+
size 5464
|
wandb/debug-internal.log
CHANGED
|
@@ -324,3 +324,47 @@
|
|
| 324 |
2024-01-12 02:37:35,283 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 325 |
2024-01-12 02:37:36,536 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 326 |
2024-01-12 02:37:38,302 DEBUG SenderThread:10874 [sender.py:send():382] send: stats
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 324 |
2024-01-12 02:37:35,283 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 325 |
2024-01-12 02:37:36,536 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 326 |
2024-01-12 02:37:38,302 DEBUG SenderThread:10874 [sender.py:send():382] send: stats
|
| 327 |
+
2024-01-12 02:37:40,303 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 328 |
+
2024-01-12 02:37:41,819 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 329 |
+
2024-01-12 02:37:45,304 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 330 |
+
2024-01-12 02:37:46,822 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 331 |
+
2024-01-12 02:37:50,305 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 332 |
+
2024-01-12 02:37:51,824 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 333 |
+
2024-01-12 02:37:55,306 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 334 |
+
2024-01-12 02:37:56,824 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 335 |
+
2024-01-12 02:38:00,306 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 336 |
+
2024-01-12 02:38:01,827 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 337 |
+
2024-01-12 02:38:05,307 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 338 |
+
2024-01-12 02:38:06,832 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 339 |
+
2024-01-12 02:38:08,303 DEBUG SenderThread:10874 [sender.py:send():382] send: stats
|
| 340 |
+
2024-01-12 02:38:11,304 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 341 |
+
2024-01-12 02:38:11,840 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 342 |
+
2024-01-12 02:38:16,305 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 343 |
+
2024-01-12 02:38:16,841 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 344 |
+
2024-01-12 02:38:21,306 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 345 |
+
2024-01-12 02:38:21,843 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 346 |
+
2024-01-12 02:38:26,306 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 347 |
+
2024-01-12 02:38:26,849 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 348 |
+
2024-01-12 02:38:31,307 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 349 |
+
2024-01-12 02:38:31,850 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 350 |
+
2024-01-12 02:38:36,308 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 351 |
+
2024-01-12 02:38:36,851 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 352 |
+
2024-01-12 02:38:38,304 DEBUG SenderThread:10874 [sender.py:send():382] send: stats
|
| 353 |
+
2024-01-12 02:38:41,857 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 354 |
+
2024-01-12 02:38:42,305 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 355 |
+
2024-01-12 02:38:46,859 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 356 |
+
2024-01-12 02:38:47,306 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 357 |
+
2024-01-12 02:38:51,505 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: partial_history
|
| 358 |
+
2024-01-12 02:38:51,506 DEBUG SenderThread:10874 [sender.py:send():382] send: history
|
| 359 |
+
2024-01-12 02:38:51,507 DEBUG SenderThread:10874 [sender.py:send_request():409] send_request: summary_record
|
| 360 |
+
2024-01-12 02:38:51,507 INFO SenderThread:10874 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
|
| 361 |
+
2024-01-12 02:38:51,860 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 362 |
+
2024-01-12 02:38:52,383 INFO Thread-12 :10874 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240112_022837-twhkj0lf/files/wandb-summary.json
|
| 363 |
+
2024-01-12 02:38:52,508 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 364 |
+
2024-01-12 02:38:54,384 INFO Thread-12 :10874 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240112_022837-twhkj0lf/files/output.log
|
| 365 |
+
2024-01-12 02:38:56,385 INFO Thread-12 :10874 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240112_022837-twhkj0lf/files/output.log
|
| 366 |
+
2024-01-12 02:38:56,869 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 367 |
+
2024-01-12 02:38:58,310 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 368 |
+
2024-01-12 02:39:01,870 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 369 |
+
2024-01-12 02:39:03,311 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 370 |
+
2024-01-12 02:39:06,871 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
wandb/run-20240112_022837-twhkj0lf/files/output.log
CHANGED
|
@@ -17,4 +17,7 @@ Checkpoint destination directory /kaggle/working/checkpoint-2500 already exists
|
|
| 17 |
/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.
|
| 18 |
warnings.warn('Was asked to gather along dimension 0, but all '
|
| 19 |
Checkpoint destination directory /kaggle/working/checkpoint-3000 already exists and is non-empty.Saving will proceed but saved results may be invalid.
|
|
|
|
|
|
|
|
|
|
| 20 |
/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.
|
|
|
|
| 17 |
/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.
|
| 18 |
warnings.warn('Was asked to gather along dimension 0, but all '
|
| 19 |
Checkpoint destination directory /kaggle/working/checkpoint-3000 already exists and is non-empty.Saving will proceed but saved results may be invalid.
|
| 20 |
+
/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.
|
| 21 |
+
warnings.warn('Was asked to gather along dimension 0, but all '
|
| 22 |
+
Checkpoint destination directory /kaggle/working/checkpoint-3500 already exists and is non-empty.Saving will proceed but saved results may be invalid.
|
| 23 |
/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.
|
wandb/run-20240112_022837-twhkj0lf/files/wandb-summary.json
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
{"train/loss": 0.
|
|
|
|
| 1 |
+
{"train/loss": 0.6047, "train/learning_rate": 6.538461538461539e-06, "train/epoch": 269.23, "train/global_step": 3500, "_timestamp": 1705027131.5049787, "_runtime": 613.8460266590118, "_step": 6}
|
wandb/run-20240112_022837-twhkj0lf/logs/debug-internal.log
CHANGED
|
@@ -324,3 +324,47 @@
|
|
| 324 |
2024-01-12 02:37:35,283 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 325 |
2024-01-12 02:37:36,536 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 326 |
2024-01-12 02:37:38,302 DEBUG SenderThread:10874 [sender.py:send():382] send: stats
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 324 |
2024-01-12 02:37:35,283 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 325 |
2024-01-12 02:37:36,536 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 326 |
2024-01-12 02:37:38,302 DEBUG SenderThread:10874 [sender.py:send():382] send: stats
|
| 327 |
+
2024-01-12 02:37:40,303 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 328 |
+
2024-01-12 02:37:41,819 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 329 |
+
2024-01-12 02:37:45,304 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 330 |
+
2024-01-12 02:37:46,822 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 331 |
+
2024-01-12 02:37:50,305 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 332 |
+
2024-01-12 02:37:51,824 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 333 |
+
2024-01-12 02:37:55,306 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 334 |
+
2024-01-12 02:37:56,824 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 335 |
+
2024-01-12 02:38:00,306 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 336 |
+
2024-01-12 02:38:01,827 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 337 |
+
2024-01-12 02:38:05,307 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 338 |
+
2024-01-12 02:38:06,832 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 339 |
+
2024-01-12 02:38:08,303 DEBUG SenderThread:10874 [sender.py:send():382] send: stats
|
| 340 |
+
2024-01-12 02:38:11,304 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 341 |
+
2024-01-12 02:38:11,840 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 342 |
+
2024-01-12 02:38:16,305 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 343 |
+
2024-01-12 02:38:16,841 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 344 |
+
2024-01-12 02:38:21,306 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 345 |
+
2024-01-12 02:38:21,843 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 346 |
+
2024-01-12 02:38:26,306 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 347 |
+
2024-01-12 02:38:26,849 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 348 |
+
2024-01-12 02:38:31,307 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 349 |
+
2024-01-12 02:38:31,850 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 350 |
+
2024-01-12 02:38:36,308 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 351 |
+
2024-01-12 02:38:36,851 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 352 |
+
2024-01-12 02:38:38,304 DEBUG SenderThread:10874 [sender.py:send():382] send: stats
|
| 353 |
+
2024-01-12 02:38:41,857 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 354 |
+
2024-01-12 02:38:42,305 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 355 |
+
2024-01-12 02:38:46,859 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 356 |
+
2024-01-12 02:38:47,306 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 357 |
+
2024-01-12 02:38:51,505 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: partial_history
|
| 358 |
+
2024-01-12 02:38:51,506 DEBUG SenderThread:10874 [sender.py:send():382] send: history
|
| 359 |
+
2024-01-12 02:38:51,507 DEBUG SenderThread:10874 [sender.py:send_request():409] send_request: summary_record
|
| 360 |
+
2024-01-12 02:38:51,507 INFO SenderThread:10874 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
|
| 361 |
+
2024-01-12 02:38:51,860 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 362 |
+
2024-01-12 02:38:52,383 INFO Thread-12 :10874 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240112_022837-twhkj0lf/files/wandb-summary.json
|
| 363 |
+
2024-01-12 02:38:52,508 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 364 |
+
2024-01-12 02:38:54,384 INFO Thread-12 :10874 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240112_022837-twhkj0lf/files/output.log
|
| 365 |
+
2024-01-12 02:38:56,385 INFO Thread-12 :10874 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240112_022837-twhkj0lf/files/output.log
|
| 366 |
+
2024-01-12 02:38:56,869 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 367 |
+
2024-01-12 02:38:58,310 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 368 |
+
2024-01-12 02:39:01,870 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
| 369 |
+
2024-01-12 02:39:03,311 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: status_report
|
| 370 |
+
2024-01-12 02:39:06,871 DEBUG HandlerThread:10874 [handler.py:handle_request():146] handle_request: keepalive
|
wandb/run-20240112_022837-twhkj0lf/run-twhkj0lf.wandb
CHANGED
|
Binary files a/wandb/run-20240112_022837-twhkj0lf/run-twhkj0lf.wandb and b/wandb/run-20240112_022837-twhkj0lf/run-twhkj0lf.wandb differ
|
|
|