Training in progress, step 160000, checkpoint
Browse files
last-checkpoint/model.safetensors
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 373077376
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3a7d4b7ae0b53507db5cf14ad9c09e970d9eb6381f657ee2363dccf8af18e53a
|
| 3 |
size 373077376
|
last-checkpoint/optimizer.pt
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 422378059
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b86fa25a29e4e16491ad6a1565427137b331bac6d6fb7d37e738443140ffd866
|
| 3 |
size 422378059
|
last-checkpoint/rng_state_0.pth
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 14917
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:05104d5a3ff46ec15aca2c8ab4d432991c08c6a2122e550de6fafb4b4a7565ac
|
| 3 |
size 14917
|
last-checkpoint/rng_state_1.pth
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 14917
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9741fd6cf63217d6f8cbdb641df62a9fb93f6b75fabd55cbc34ce053af823e49
|
| 3 |
size 14917
|
last-checkpoint/scheduler.pt
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 1401
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5bfc5fa70fa1087a61c07ea9f4f006e1c0224d2746393af9436852b1663faa92
|
| 3 |
size 1401
|
last-checkpoint/trainer_state.json
CHANGED
|
@@ -2,9 +2,9 @@
|
|
| 2 |
"best_global_step": 159000,
|
| 3 |
"best_metric": 2.6867077350616455,
|
| 4 |
"best_model_checkpoint": "./artifacts/models/base-250725/checkpoint-159000",
|
| 5 |
-
"epoch": 0.
|
| 6 |
"eval_steps": 1000,
|
| 7 |
-
"global_step":
|
| 8 |
"is_hyper_param_search": false,
|
| 9 |
"is_local_process_zero": true,
|
| 10 |
"is_world_process_zero": true,
|
|
@@ -45800,6 +45800,294 @@
|
|
| 45800 |
"eval_samples_per_second": 50.932,
|
| 45801 |
"eval_steps_per_second": 3.191,
|
| 45802 |
"step": 159000
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45803 |
}
|
| 45804 |
],
|
| 45805 |
"logging_steps": 25,
|
|
@@ -45819,7 +46107,7 @@
|
|
| 45819 |
"attributes": {}
|
| 45820 |
}
|
| 45821 |
},
|
| 45822 |
-
"total_flos": 4.
|
| 45823 |
"train_batch_size": 16,
|
| 45824 |
"trial_name": null,
|
| 45825 |
"trial_params": null
|
|
|
|
| 2 |
"best_global_step": 159000,
|
| 3 |
"best_metric": 2.6867077350616455,
|
| 4 |
"best_model_checkpoint": "./artifacts/models/base-250725/checkpoint-159000",
|
| 5 |
+
"epoch": 0.999394117316377,
|
| 6 |
"eval_steps": 1000,
|
| 7 |
+
"global_step": 160000,
|
| 8 |
"is_hyper_param_search": false,
|
| 9 |
"is_local_process_zero": true,
|
| 10 |
"is_world_process_zero": true,
|
|
|
|
| 45800 |
"eval_samples_per_second": 50.932,
|
| 45801 |
"eval_steps_per_second": 3.191,
|
| 45802 |
"step": 159000
|
| 45803 |
+
},
|
| 45804 |
+
{
|
| 45805 |
+
"epoch": 0.9933040594139803,
|
| 45806 |
+
"grad_norm": 0.30859375,
|
| 45807 |
+
"learning_rate": 1.2280291557259515e-07,
|
| 45808 |
+
"loss": 2.7153,
|
| 45809 |
+
"step": 159025
|
| 45810 |
+
},
|
| 45811 |
+
{
|
| 45812 |
+
"epoch": 0.9934602147448109,
|
| 45813 |
+
"grad_norm": 0.302734375,
|
| 45814 |
+
"learning_rate": 1.1714739025120436e-07,
|
| 45815 |
+
"loss": 2.7109,
|
| 45816 |
+
"step": 159050
|
| 45817 |
+
},
|
| 45818 |
+
{
|
| 45819 |
+
"epoch": 0.9936163700756416,
|
| 45820 |
+
"grad_norm": 0.3125,
|
| 45821 |
+
"learning_rate": 1.1162516649565469e-07,
|
| 45822 |
+
"loss": 2.7189,
|
| 45823 |
+
"step": 159075
|
| 45824 |
+
},
|
| 45825 |
+
{
|
| 45826 |
+
"epoch": 0.9937725254064723,
|
| 45827 |
+
"grad_norm": 0.310546875,
|
| 45828 |
+
"learning_rate": 1.0623624577843494e-07,
|
| 45829 |
+
"loss": 2.6997,
|
| 45830 |
+
"step": 159100
|
| 45831 |
+
},
|
| 45832 |
+
{
|
| 45833 |
+
"epoch": 0.993928680737303,
|
| 45834 |
+
"grad_norm": 0.314453125,
|
| 45835 |
+
"learning_rate": 1.0098062953667331e-07,
|
| 45836 |
+
"loss": 2.7167,
|
| 45837 |
+
"step": 159125
|
| 45838 |
+
},
|
| 45839 |
+
{
|
| 45840 |
+
"epoch": 0.9940848360681337,
|
| 45841 |
+
"grad_norm": 0.302734375,
|
| 45842 |
+
"learning_rate": 9.585831917180432e-08,
|
| 45843 |
+
"loss": 2.7093,
|
| 45844 |
+
"step": 159150
|
| 45845 |
+
},
|
| 45846 |
+
{
|
| 45847 |
+
"epoch": 0.9942409913989644,
|
| 45848 |
+
"grad_norm": 0.3046875,
|
| 45849 |
+
"learning_rate": 9.086931604984638e-08,
|
| 45850 |
+
"loss": 2.7237,
|
| 45851 |
+
"step": 159175
|
| 45852 |
+
},
|
| 45853 |
+
{
|
| 45854 |
+
"epoch": 0.9943971467297951,
|
| 45855 |
+
"grad_norm": 0.3046875,
|
| 45856 |
+
"learning_rate": 8.601362150112423e-08,
|
| 45857 |
+
"loss": 2.7144,
|
| 45858 |
+
"step": 159200
|
| 45859 |
+
},
|
| 45860 |
+
{
|
| 45861 |
+
"epoch": 0.9945533020606258,
|
| 45862 |
+
"grad_norm": 0.30078125,
|
| 45863 |
+
"learning_rate": 8.129123682054651e-08,
|
| 45864 |
+
"loss": 2.7051,
|
| 45865 |
+
"step": 159225
|
| 45866 |
+
},
|
| 45867 |
+
{
|
| 45868 |
+
"epoch": 0.9947094573914564,
|
| 45869 |
+
"grad_norm": 0.310546875,
|
| 45870 |
+
"learning_rate": 7.670216326732815e-08,
|
| 45871 |
+
"loss": 2.7206,
|
| 45872 |
+
"step": 159250
|
| 45873 |
+
},
|
| 45874 |
+
{
|
| 45875 |
+
"epoch": 0.9948656127222871,
|
| 45876 |
+
"grad_norm": 0.298828125,
|
| 45877 |
+
"learning_rate": 7.224640206532351e-08,
|
| 45878 |
+
"loss": 2.6995,
|
| 45879 |
+
"step": 159275
|
| 45880 |
+
},
|
| 45881 |
+
{
|
| 45882 |
+
"epoch": 0.9950217680531178,
|
| 45883 |
+
"grad_norm": 0.298828125,
|
| 45884 |
+
"learning_rate": 6.792395440263777e-08,
|
| 45885 |
+
"loss": 2.6854,
|
| 45886 |
+
"step": 159300
|
| 45887 |
+
},
|
| 45888 |
+
{
|
| 45889 |
+
"epoch": 0.9951779233839485,
|
| 45890 |
+
"grad_norm": 0.35546875,
|
| 45891 |
+
"learning_rate": 6.373482143195997e-08,
|
| 45892 |
+
"loss": 2.7094,
|
| 45893 |
+
"step": 159325
|
| 45894 |
+
},
|
| 45895 |
+
{
|
| 45896 |
+
"epoch": 0.9953340787147792,
|
| 45897 |
+
"grad_norm": 0.296875,
|
| 45898 |
+
"learning_rate": 5.967900427034101e-08,
|
| 45899 |
+
"loss": 2.7157,
|
| 45900 |
+
"step": 159350
|
| 45901 |
+
},
|
| 45902 |
+
{
|
| 45903 |
+
"epoch": 0.9954902340456099,
|
| 45904 |
+
"grad_norm": 0.306640625,
|
| 45905 |
+
"learning_rate": 5.5756503999415675e-08,
|
| 45906 |
+
"loss": 2.7079,
|
| 45907 |
+
"step": 159375
|
| 45908 |
+
},
|
| 45909 |
+
{
|
| 45910 |
+
"epoch": 0.9956463893764406,
|
| 45911 |
+
"grad_norm": 0.310546875,
|
| 45912 |
+
"learning_rate": 5.1967321665125076e-08,
|
| 45913 |
+
"loss": 2.7186,
|
| 45914 |
+
"step": 159400
|
| 45915 |
+
},
|
| 45916 |
+
{
|
| 45917 |
+
"epoch": 0.9958025447072713,
|
| 45918 |
+
"grad_norm": 0.294921875,
|
| 45919 |
+
"learning_rate": 4.83114582779387e-08,
|
| 45920 |
+
"loss": 2.6919,
|
| 45921 |
+
"step": 159425
|
| 45922 |
+
},
|
| 45923 |
+
{
|
| 45924 |
+
"epoch": 0.9959587000381019,
|
| 45925 |
+
"grad_norm": 0.318359375,
|
| 45926 |
+
"learning_rate": 4.478891481274338e-08,
|
| 45927 |
+
"loss": 2.6852,
|
| 45928 |
+
"step": 159450
|
| 45929 |
+
},
|
| 45930 |
+
{
|
| 45931 |
+
"epoch": 0.9961148553689326,
|
| 45932 |
+
"grad_norm": 0.330078125,
|
| 45933 |
+
"learning_rate": 4.139969220884332e-08,
|
| 45934 |
+
"loss": 2.7079,
|
| 45935 |
+
"step": 159475
|
| 45936 |
+
},
|
| 45937 |
+
{
|
| 45938 |
+
"epoch": 0.9962710106997633,
|
| 45939 |
+
"grad_norm": 0.30078125,
|
| 45940 |
+
"learning_rate": 3.814379137007107e-08,
|
| 45941 |
+
"loss": 2.7028,
|
| 45942 |
+
"step": 159500
|
| 45943 |
+
},
|
| 45944 |
+
{
|
| 45945 |
+
"epoch": 0.996427166030594,
|
| 45946 |
+
"grad_norm": 0.3046875,
|
| 45947 |
+
"learning_rate": 3.502121316462104e-08,
|
| 45948 |
+
"loss": 2.7195,
|
| 45949 |
+
"step": 159525
|
| 45950 |
+
},
|
| 45951 |
+
{
|
| 45952 |
+
"epoch": 0.9965833213614247,
|
| 45953 |
+
"grad_norm": 0.3046875,
|
| 45954 |
+
"learning_rate": 3.2031958425216004e-08,
|
| 45955 |
+
"loss": 2.7048,
|
| 45956 |
+
"step": 159550
|
| 45957 |
+
},
|
| 45958 |
+
{
|
| 45959 |
+
"epoch": 0.9967394766922553,
|
| 45960 |
+
"grad_norm": 0.298828125,
|
| 45961 |
+
"learning_rate": 2.9176027948940588e-08,
|
| 45962 |
+
"loss": 2.7019,
|
| 45963 |
+
"step": 159575
|
| 45964 |
+
},
|
| 45965 |
+
{
|
| 45966 |
+
"epoch": 0.996895632023086,
|
| 45967 |
+
"grad_norm": 0.30078125,
|
| 45968 |
+
"learning_rate": 2.6453422497463298e-08,
|
| 45969 |
+
"loss": 2.7062,
|
| 45970 |
+
"step": 159600
|
| 45971 |
+
},
|
| 45972 |
+
{
|
| 45973 |
+
"epoch": 0.9970517873539166,
|
| 45974 |
+
"grad_norm": 0.29296875,
|
| 45975 |
+
"learning_rate": 2.3864142796703457e-08,
|
| 45976 |
+
"loss": 2.7034,
|
| 45977 |
+
"step": 159625
|
| 45978 |
+
},
|
| 45979 |
+
{
|
| 45980 |
+
"epoch": 0.9972079426847473,
|
| 45981 |
+
"grad_norm": 0.3046875,
|
| 45982 |
+
"learning_rate": 2.140818953721979e-08,
|
| 45983 |
+
"loss": 2.6915,
|
| 45984 |
+
"step": 159650
|
| 45985 |
+
},
|
| 45986 |
+
{
|
| 45987 |
+
"epoch": 0.997364098015578,
|
| 45988 |
+
"grad_norm": 0.33984375,
|
| 45989 |
+
"learning_rate": 1.9085563373877345e-08,
|
| 45990 |
+
"loss": 2.7074,
|
| 45991 |
+
"step": 159675
|
| 45992 |
+
},
|
| 45993 |
+
{
|
| 45994 |
+
"epoch": 0.9975202533464087,
|
| 45995 |
+
"grad_norm": 0.314453125,
|
| 45996 |
+
"learning_rate": 1.6896264926069548e-08,
|
| 45997 |
+
"loss": 2.7079,
|
| 45998 |
+
"step": 159700
|
| 45999 |
+
},
|
| 46000 |
+
{
|
| 46001 |
+
"epoch": 0.9976764086772394,
|
| 46002 |
+
"grad_norm": 0.3046875,
|
| 46003 |
+
"learning_rate": 1.4840294777607177e-08,
|
| 46004 |
+
"loss": 2.6876,
|
| 46005 |
+
"step": 159725
|
| 46006 |
+
},
|
| 46007 |
+
{
|
| 46008 |
+
"epoch": 0.9978325640080701,
|
| 46009 |
+
"grad_norm": 0.310546875,
|
| 46010 |
+
"learning_rate": 1.291765347671836e-08,
|
| 46011 |
+
"loss": 2.6994,
|
| 46012 |
+
"step": 159750
|
| 46013 |
+
},
|
| 46014 |
+
{
|
| 46015 |
+
"epoch": 0.9979887193389008,
|
| 46016 |
+
"grad_norm": 0.30078125,
|
| 46017 |
+
"learning_rate": 1.1128341536104091e-08,
|
| 46018 |
+
"loss": 2.7104,
|
| 46019 |
+
"step": 159775
|
| 46020 |
+
},
|
| 46021 |
+
{
|
| 46022 |
+
"epoch": 0.9981448746697315,
|
| 46023 |
+
"grad_norm": 0.2890625,
|
| 46024 |
+
"learning_rate": 9.472359432938227e-09,
|
| 46025 |
+
"loss": 2.6878,
|
| 46026 |
+
"step": 159800
|
| 46027 |
+
},
|
| 46028 |
+
{
|
| 46029 |
+
"epoch": 0.9983010300005621,
|
| 46030 |
+
"grad_norm": 0.3125,
|
| 46031 |
+
"learning_rate": 7.949707608811973e-09,
|
| 46032 |
+
"loss": 2.7114,
|
| 46033 |
+
"step": 159825
|
| 46034 |
+
},
|
| 46035 |
+
{
|
| 46036 |
+
"epoch": 0.9984571853313928,
|
| 46037 |
+
"grad_norm": 0.40625,
|
| 46038 |
+
"learning_rate": 6.560386469789403e-09,
|
| 46039 |
+
"loss": 2.7105,
|
| 46040 |
+
"step": 159850
|
| 46041 |
+
},
|
| 46042 |
+
{
|
| 46043 |
+
"epoch": 0.9986133406622235,
|
| 46044 |
+
"grad_norm": 0.306640625,
|
| 46045 |
+
"learning_rate": 5.3043963862964285e-09,
|
| 46046 |
+
"loss": 2.7138,
|
| 46047 |
+
"step": 159875
|
| 46048 |
+
},
|
| 46049 |
+
{
|
| 46050 |
+
"epoch": 0.9987694959930542,
|
| 46051 |
+
"grad_norm": 0.3046875,
|
| 46052 |
+
"learning_rate": 4.181737693231824e-09,
|
| 46053 |
+
"loss": 2.7074,
|
| 46054 |
+
"step": 159900
|
| 46055 |
+
},
|
| 46056 |
+
{
|
| 46057 |
+
"epoch": 0.9989256513238849,
|
| 46058 |
+
"grad_norm": 0.310546875,
|
| 46059 |
+
"learning_rate": 3.1924106900782514e-09,
|
| 46060 |
+
"loss": 2.7035,
|
| 46061 |
+
"step": 159925
|
| 46062 |
+
},
|
| 46063 |
+
{
|
| 46064 |
+
"epoch": 0.9990818066547156,
|
| 46065 |
+
"grad_norm": 0.30078125,
|
| 46066 |
+
"learning_rate": 2.3364156406247005e-09,
|
| 46067 |
+
"loss": 2.7013,
|
| 46068 |
+
"step": 159950
|
| 46069 |
+
},
|
| 46070 |
+
{
|
| 46071 |
+
"epoch": 0.9992379619855463,
|
| 46072 |
+
"grad_norm": 0.302734375,
|
| 46073 |
+
"learning_rate": 1.6137527730775148e-09,
|
| 46074 |
+
"loss": 2.7102,
|
| 46075 |
+
"step": 159975
|
| 46076 |
+
},
|
| 46077 |
+
{
|
| 46078 |
+
"epoch": 0.999394117316377,
|
| 46079 |
+
"grad_norm": 0.32421875,
|
| 46080 |
+
"learning_rate": 1.024422280226922e-09,
|
| 46081 |
+
"loss": 2.6958,
|
| 46082 |
+
"step": 160000
|
| 46083 |
+
},
|
| 46084 |
+
{
|
| 46085 |
+
"epoch": 0.999394117316377,
|
| 46086 |
+
"eval_loss": 2.686711072921753,
|
| 46087 |
+
"eval_runtime": 102.1585,
|
| 46088 |
+
"eval_samples_per_second": 50.931,
|
| 46089 |
+
"eval_steps_per_second": 3.191,
|
| 46090 |
+
"step": 160000
|
| 46091 |
}
|
| 46092 |
],
|
| 46093 |
"logging_steps": 25,
|
|
|
|
| 46107 |
"attributes": {}
|
| 46108 |
}
|
| 46109 |
},
|
| 46110 |
+
"total_flos": 4.058098408986195e+19,
|
| 46111 |
"train_batch_size": 16,
|
| 46112 |
"trial_name": null,
|
| 46113 |
"trial_params": null
|