ngwgsang commited on
Commit
51afcc8
·
verified ·
1 Parent(s): 08a3a8d

Training in progress, epoch 10, checkpoint

Browse files
last-checkpoint/model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4ac0a34bebeefbbb11135799066142a50e5c723e5979e6e70c5c4e04d10e97b1
3
  size 540026052
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:40e63c540803c66ab93614b2ff5d15de410719ac1be826b88bfe243577997d1e
3
  size 540026052
last-checkpoint/optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:5d10254a30bd95c953e46d02a9f53a8bb47dfae6ef3410f580a1bcc875d7657b
3
  size 1075446330
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1c62a36aee495e818da31750c0172ed44160dadea553535b4a8ae082c985e0a0
3
  size 1075446330
last-checkpoint/rng_state.pth CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0a64a36e7ef0e3bc6b8c152fc0252bfa67634414f868b55756b55ec69afcd692
3
  size 14244
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c55c0792d1b7dccb405f01a97a5e512cae549864cac8915f93574637399af64a
3
  size 14244
last-checkpoint/scheduler.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ebf5d2d6cf23b3caa8314a5b87205e2b2427fc1bd1b2618e1c648902b74ca7cc
3
  size 1064
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:46e3123fb72e92b10df5ef82d396a7b1f35005c16f55ce1c7706ce527f3809d4
3
  size 1064
last-checkpoint/trainer_state.json CHANGED
@@ -1,9 +1,9 @@
1
  {
2
- "best_metric": 10.502753462428936,
3
- "best_model_checkpoint": "./results/checkpoint-5130",
4
- "epoch": 9.0,
5
  "eval_steps": 500,
6
- "global_step": 5130,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
@@ -3706,6 +3706,417 @@
3706
  "eval_samples_per_second": 265.057,
3707
  "eval_steps_per_second": 16.624,
3708
  "step": 5130
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3709
  }
3710
  ],
3711
  "logging_steps": 10,
@@ -3720,7 +4131,7 @@
3720
  "should_evaluate": false,
3721
  "should_log": false,
3722
  "should_save": true,
3723
- "should_training_stop": false
3724
  },
3725
  "attributes": {}
3726
  }
 
1
  {
2
+ "best_metric": 10.266098292947513,
3
+ "best_model_checkpoint": "./results/checkpoint-5700",
4
+ "epoch": 10.0,
5
  "eval_steps": 500,
6
+ "global_step": 5700,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
 
3706
  "eval_samples_per_second": 265.057,
3707
  "eval_steps_per_second": 16.624,
3708
  "step": 5130
3709
+ },
3710
+ {
3711
+ "epoch": 9.017543859649123,
3712
+ "grad_norm": 281.239013671875,
3713
+ "learning_rate": 4.912280701754386e-06,
3714
+ "loss": 166.2685,
3715
+ "step": 5140
3716
+ },
3717
+ {
3718
+ "epoch": 9.035087719298245,
3719
+ "grad_norm": 297.7884216308594,
3720
+ "learning_rate": 4.824561403508772e-06,
3721
+ "loss": 182.959,
3722
+ "step": 5150
3723
+ },
3724
+ {
3725
+ "epoch": 9.052631578947368,
3726
+ "grad_norm": 200.22116088867188,
3727
+ "learning_rate": 4.736842105263159e-06,
3728
+ "loss": 160.0898,
3729
+ "step": 5160
3730
+ },
3731
+ {
3732
+ "epoch": 9.070175438596491,
3733
+ "grad_norm": 729.8984375,
3734
+ "learning_rate": 4.649122807017544e-06,
3735
+ "loss": 162.7628,
3736
+ "step": 5170
3737
+ },
3738
+ {
3739
+ "epoch": 9.087719298245615,
3740
+ "grad_norm": 476.8837585449219,
3741
+ "learning_rate": 4.5614035087719304e-06,
3742
+ "loss": 186.9401,
3743
+ "step": 5180
3744
+ },
3745
+ {
3746
+ "epoch": 9.105263157894736,
3747
+ "grad_norm": 233.36331176757812,
3748
+ "learning_rate": 4.473684210526316e-06,
3749
+ "loss": 181.0498,
3750
+ "step": 5190
3751
+ },
3752
+ {
3753
+ "epoch": 9.12280701754386,
3754
+ "grad_norm": 313.01953125,
3755
+ "learning_rate": 4.3859649122807014e-06,
3756
+ "loss": 157.1372,
3757
+ "step": 5200
3758
+ },
3759
+ {
3760
+ "epoch": 9.140350877192983,
3761
+ "grad_norm": 337.495361328125,
3762
+ "learning_rate": 4.298245614035088e-06,
3763
+ "loss": 181.6845,
3764
+ "step": 5210
3765
+ },
3766
+ {
3767
+ "epoch": 9.157894736842104,
3768
+ "grad_norm": 218.62841796875,
3769
+ "learning_rate": 4.210526315789474e-06,
3770
+ "loss": 164.5618,
3771
+ "step": 5220
3772
+ },
3773
+ {
3774
+ "epoch": 9.175438596491228,
3775
+ "grad_norm": 239.92413330078125,
3776
+ "learning_rate": 4.1228070175438605e-06,
3777
+ "loss": 174.9292,
3778
+ "step": 5230
3779
+ },
3780
+ {
3781
+ "epoch": 9.192982456140351,
3782
+ "grad_norm": 390.8914794921875,
3783
+ "learning_rate": 4.035087719298246e-06,
3784
+ "loss": 213.891,
3785
+ "step": 5240
3786
+ },
3787
+ {
3788
+ "epoch": 9.210526315789474,
3789
+ "grad_norm": 416.03936767578125,
3790
+ "learning_rate": 3.9473684210526315e-06,
3791
+ "loss": 169.2472,
3792
+ "step": 5250
3793
+ },
3794
+ {
3795
+ "epoch": 9.228070175438596,
3796
+ "grad_norm": 304.3415832519531,
3797
+ "learning_rate": 3.859649122807018e-06,
3798
+ "loss": 150.4329,
3799
+ "step": 5260
3800
+ },
3801
+ {
3802
+ "epoch": 9.24561403508772,
3803
+ "grad_norm": 319.07684326171875,
3804
+ "learning_rate": 3.7719298245614037e-06,
3805
+ "loss": 194.0962,
3806
+ "step": 5270
3807
+ },
3808
+ {
3809
+ "epoch": 9.263157894736842,
3810
+ "grad_norm": 230.61453247070312,
3811
+ "learning_rate": 3.6842105263157892e-06,
3812
+ "loss": 154.6355,
3813
+ "step": 5280
3814
+ },
3815
+ {
3816
+ "epoch": 9.280701754385966,
3817
+ "grad_norm": 217.14149475097656,
3818
+ "learning_rate": 3.5964912280701756e-06,
3819
+ "loss": 162.511,
3820
+ "step": 5290
3821
+ },
3822
+ {
3823
+ "epoch": 9.298245614035087,
3824
+ "grad_norm": 256.52978515625,
3825
+ "learning_rate": 3.5087719298245615e-06,
3826
+ "loss": 168.9292,
3827
+ "step": 5300
3828
+ },
3829
+ {
3830
+ "epoch": 9.31578947368421,
3831
+ "grad_norm": 230.6572265625,
3832
+ "learning_rate": 3.421052631578948e-06,
3833
+ "loss": 170.788,
3834
+ "step": 5310
3835
+ },
3836
+ {
3837
+ "epoch": 9.333333333333334,
3838
+ "grad_norm": 226.62417602539062,
3839
+ "learning_rate": 3.3333333333333333e-06,
3840
+ "loss": 150.7565,
3841
+ "step": 5320
3842
+ },
3843
+ {
3844
+ "epoch": 9.350877192982455,
3845
+ "grad_norm": 335.15692138671875,
3846
+ "learning_rate": 3.2456140350877192e-06,
3847
+ "loss": 185.207,
3848
+ "step": 5330
3849
+ },
3850
+ {
3851
+ "epoch": 9.368421052631579,
3852
+ "grad_norm": 241.26544189453125,
3853
+ "learning_rate": 3.1578947368421056e-06,
3854
+ "loss": 169.805,
3855
+ "step": 5340
3856
+ },
3857
+ {
3858
+ "epoch": 9.385964912280702,
3859
+ "grad_norm": 196.6084747314453,
3860
+ "learning_rate": 3.070175438596491e-06,
3861
+ "loss": 148.7297,
3862
+ "step": 5350
3863
+ },
3864
+ {
3865
+ "epoch": 9.403508771929825,
3866
+ "grad_norm": 231.8640899658203,
3867
+ "learning_rate": 2.9824561403508774e-06,
3868
+ "loss": 149.0492,
3869
+ "step": 5360
3870
+ },
3871
+ {
3872
+ "epoch": 9.421052631578947,
3873
+ "grad_norm": 322.551025390625,
3874
+ "learning_rate": 2.8947368421052634e-06,
3875
+ "loss": 171.7541,
3876
+ "step": 5370
3877
+ },
3878
+ {
3879
+ "epoch": 9.43859649122807,
3880
+ "grad_norm": 359.8404541015625,
3881
+ "learning_rate": 2.8070175438596493e-06,
3882
+ "loss": 134.9022,
3883
+ "step": 5380
3884
+ },
3885
+ {
3886
+ "epoch": 9.456140350877194,
3887
+ "grad_norm": 393.9560241699219,
3888
+ "learning_rate": 2.719298245614035e-06,
3889
+ "loss": 160.3259,
3890
+ "step": 5390
3891
+ },
3892
+ {
3893
+ "epoch": 9.473684210526315,
3894
+ "grad_norm": 307.36712646484375,
3895
+ "learning_rate": 2.631578947368421e-06,
3896
+ "loss": 169.45,
3897
+ "step": 5400
3898
+ },
3899
+ {
3900
+ "epoch": 9.491228070175438,
3901
+ "grad_norm": 429.7928161621094,
3902
+ "learning_rate": 2.543859649122807e-06,
3903
+ "loss": 161.9689,
3904
+ "step": 5410
3905
+ },
3906
+ {
3907
+ "epoch": 9.508771929824562,
3908
+ "grad_norm": 260.0240173339844,
3909
+ "learning_rate": 2.456140350877193e-06,
3910
+ "loss": 182.8279,
3911
+ "step": 5420
3912
+ },
3913
+ {
3914
+ "epoch": 9.526315789473685,
3915
+ "grad_norm": 260.8311767578125,
3916
+ "learning_rate": 2.3684210526315793e-06,
3917
+ "loss": 140.941,
3918
+ "step": 5430
3919
+ },
3920
+ {
3921
+ "epoch": 9.543859649122806,
3922
+ "grad_norm": 189.70962524414062,
3923
+ "learning_rate": 2.2807017543859652e-06,
3924
+ "loss": 187.3211,
3925
+ "step": 5440
3926
+ },
3927
+ {
3928
+ "epoch": 9.56140350877193,
3929
+ "grad_norm": 364.1365966796875,
3930
+ "learning_rate": 2.1929824561403507e-06,
3931
+ "loss": 155.0701,
3932
+ "step": 5450
3933
+ },
3934
+ {
3935
+ "epoch": 9.578947368421053,
3936
+ "grad_norm": 244.2755126953125,
3937
+ "learning_rate": 2.105263157894737e-06,
3938
+ "loss": 182.5883,
3939
+ "step": 5460
3940
+ },
3941
+ {
3942
+ "epoch": 9.596491228070175,
3943
+ "grad_norm": 417.791015625,
3944
+ "learning_rate": 2.017543859649123e-06,
3945
+ "loss": 173.9106,
3946
+ "step": 5470
3947
+ },
3948
+ {
3949
+ "epoch": 9.614035087719298,
3950
+ "grad_norm": 373.1618347167969,
3951
+ "learning_rate": 1.929824561403509e-06,
3952
+ "loss": 178.0014,
3953
+ "step": 5480
3954
+ },
3955
+ {
3956
+ "epoch": 9.631578947368421,
3957
+ "grad_norm": 221.83309936523438,
3958
+ "learning_rate": 1.8421052631578946e-06,
3959
+ "loss": 182.6862,
3960
+ "step": 5490
3961
+ },
3962
+ {
3963
+ "epoch": 9.649122807017545,
3964
+ "grad_norm": 301.48382568359375,
3965
+ "learning_rate": 1.7543859649122807e-06,
3966
+ "loss": 159.5674,
3967
+ "step": 5500
3968
+ },
3969
+ {
3970
+ "epoch": 9.666666666666666,
3971
+ "grad_norm": 487.5570983886719,
3972
+ "learning_rate": 1.6666666666666667e-06,
3973
+ "loss": 164.2859,
3974
+ "step": 5510
3975
+ },
3976
+ {
3977
+ "epoch": 9.68421052631579,
3978
+ "grad_norm": 326.3913269042969,
3979
+ "learning_rate": 1.5789473684210528e-06,
3980
+ "loss": 188.0145,
3981
+ "step": 5520
3982
+ },
3983
+ {
3984
+ "epoch": 9.701754385964913,
3985
+ "grad_norm": 314.8993835449219,
3986
+ "learning_rate": 1.4912280701754387e-06,
3987
+ "loss": 156.2349,
3988
+ "step": 5530
3989
+ },
3990
+ {
3991
+ "epoch": 9.719298245614034,
3992
+ "grad_norm": 1071.3399658203125,
3993
+ "learning_rate": 1.4035087719298246e-06,
3994
+ "loss": 187.2437,
3995
+ "step": 5540
3996
+ },
3997
+ {
3998
+ "epoch": 9.736842105263158,
3999
+ "grad_norm": 228.32090759277344,
4000
+ "learning_rate": 1.3157894736842106e-06,
4001
+ "loss": 156.9955,
4002
+ "step": 5550
4003
+ },
4004
+ {
4005
+ "epoch": 9.75438596491228,
4006
+ "grad_norm": 291.3077697753906,
4007
+ "learning_rate": 1.2280701754385965e-06,
4008
+ "loss": 160.9737,
4009
+ "step": 5560
4010
+ },
4011
+ {
4012
+ "epoch": 9.771929824561404,
4013
+ "grad_norm": 530.8067626953125,
4014
+ "learning_rate": 1.1403508771929826e-06,
4015
+ "loss": 151.8838,
4016
+ "step": 5570
4017
+ },
4018
+ {
4019
+ "epoch": 9.789473684210526,
4020
+ "grad_norm": 337.1194763183594,
4021
+ "learning_rate": 1.0526315789473685e-06,
4022
+ "loss": 158.343,
4023
+ "step": 5580
4024
+ },
4025
+ {
4026
+ "epoch": 9.807017543859649,
4027
+ "grad_norm": 304.9252624511719,
4028
+ "learning_rate": 9.649122807017545e-07,
4029
+ "loss": 172.4637,
4030
+ "step": 5590
4031
+ },
4032
+ {
4033
+ "epoch": 9.824561403508772,
4034
+ "grad_norm": 236.6869354248047,
4035
+ "learning_rate": 8.771929824561404e-07,
4036
+ "loss": 156.1226,
4037
+ "step": 5600
4038
+ },
4039
+ {
4040
+ "epoch": 9.842105263157894,
4041
+ "grad_norm": 249.76043701171875,
4042
+ "learning_rate": 7.894736842105264e-07,
4043
+ "loss": 167.4769,
4044
+ "step": 5610
4045
+ },
4046
+ {
4047
+ "epoch": 9.859649122807017,
4048
+ "grad_norm": 296.2056884765625,
4049
+ "learning_rate": 7.017543859649123e-07,
4050
+ "loss": 155.2643,
4051
+ "step": 5620
4052
+ },
4053
+ {
4054
+ "epoch": 9.87719298245614,
4055
+ "grad_norm": 198.53219604492188,
4056
+ "learning_rate": 6.140350877192982e-07,
4057
+ "loss": 168.2789,
4058
+ "step": 5630
4059
+ },
4060
+ {
4061
+ "epoch": 9.894736842105264,
4062
+ "grad_norm": 714.234130859375,
4063
+ "learning_rate": 5.263157894736843e-07,
4064
+ "loss": 183.7942,
4065
+ "step": 5640
4066
+ },
4067
+ {
4068
+ "epoch": 9.912280701754385,
4069
+ "grad_norm": 229.8941192626953,
4070
+ "learning_rate": 4.385964912280702e-07,
4071
+ "loss": 147.4333,
4072
+ "step": 5650
4073
+ },
4074
+ {
4075
+ "epoch": 9.929824561403509,
4076
+ "grad_norm": 462.6064147949219,
4077
+ "learning_rate": 3.5087719298245616e-07,
4078
+ "loss": 154.1472,
4079
+ "step": 5660
4080
+ },
4081
+ {
4082
+ "epoch": 9.947368421052632,
4083
+ "grad_norm": 275.9501953125,
4084
+ "learning_rate": 2.6315789473684213e-07,
4085
+ "loss": 161.2116,
4086
+ "step": 5670
4087
+ },
4088
+ {
4089
+ "epoch": 9.964912280701755,
4090
+ "grad_norm": 696.220703125,
4091
+ "learning_rate": 1.7543859649122808e-07,
4092
+ "loss": 164.8297,
4093
+ "step": 5680
4094
+ },
4095
+ {
4096
+ "epoch": 9.982456140350877,
4097
+ "grad_norm": 498.91448974609375,
4098
+ "learning_rate": 8.771929824561404e-08,
4099
+ "loss": 153.6297,
4100
+ "step": 5690
4101
+ },
4102
+ {
4103
+ "epoch": 10.0,
4104
+ "grad_norm": 873.9326782226562,
4105
+ "learning_rate": 0.0,
4106
+ "loss": 155.5076,
4107
+ "step": 5700
4108
+ },
4109
+ {
4110
+ "epoch": 10.0,
4111
+ "eval_loss": 171.82171630859375,
4112
+ "eval_mae_lex": 11.493141912159167,
4113
+ "eval_mae_overall": 10.266098292947513,
4114
+ "eval_mae_sem": 7.863255273250111,
4115
+ "eval_mae_syn": 11.44189769343326,
4116
+ "eval_runtime": 8.6814,
4117
+ "eval_samples_per_second": 262.631,
4118
+ "eval_steps_per_second": 16.472,
4119
+ "step": 5700
4120
  }
4121
  ],
4122
  "logging_steps": 10,
 
4131
  "should_evaluate": false,
4132
  "should_log": false,
4133
  "should_save": true,
4134
+ "should_training_stop": true
4135
  },
4136
  "attributes": {}
4137
  }