sparsetrace commited on
Commit
a558571
·
verified ·
1 Parent(s): 1d30aea

checkpoint: promote transactional candidate: step 340000

Browse files
Files changed (4) hide show
  1. ema.safetensors +1 -1
  2. model.safetensors +1 -1
  3. optim.safetensors +1 -1
  4. state.json +254 -225
ema.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:2b971f03112893de5ae83a2b11a79b266c10abfd7705ccd69b07a96c5dcd1d92
3
  size 1945760448
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d0b1ae4dac1b4fffd161e39d798162c8a799251d40d537e3f456cafaf02c5456
3
  size 1945760448
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:1bc9f9420b10d5f309c4a2a68f1d3ed1683b4db8fd4bcd12cf8aced74f1ded8a
3
  size 2046527448
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:af0ab1302bd854546a15e8ee6121571dce9bfd7041744c9baee2b70376b5faea
3
  size 2046527448
optim.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d111f2c043de49c9c7d8193d93a8f0143b32103a329ed066a31a29650e5bcdde
3
  size 3891530432
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0d752e1f945e9df784124f5a1339da572ed391fb5772635d48654ed4c91cfe03
3
  size 3891530432
state.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "step": 330000,
3
  "run_history": [
4
  {
5
  "start_step": 0,
@@ -935,216 +935,245 @@
935
  "seed_offset": 0,
936
  "target_steps": 400000
937
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
938
  }
939
  ],
940
  "guard_state": {
941
  "version": 3,
942
- "live_grad_ema": 0.11989347962593028,
943
- "grad_ema": 0.11989347962593028,
944
- "grad_reference": 0.1277403510408516,
945
- "grad_p90_reference": 0.1423632360760495,
946
  "spikes_total": 0,
947
  "recent_losses": [
948
- 0.7896416187286377,
949
- 0.7830684185028076,
950
- 0.7710197567939758,
951
- 0.7866576910018921,
952
- 0.7913663387298584,
953
- 0.7742933034896851,
954
- 0.7921265363693237,
955
- 0.7693139314651489,
956
- 0.799022912979126,
957
- 0.7780013084411621,
958
- 0.7917797565460205,
959
- 0.7859218716621399,
960
- 0.7802191972732544,
961
- 0.8009483814239502,
962
- 0.7884707450866699,
963
- 0.7741677761077881,
964
- 0.7919224500656128,
965
- 0.7796087265014648,
966
- 0.773560643196106,
967
- 0.7869712114334106,
968
- 0.7982282638549805,
969
- 0.7729727625846863,
970
- 0.8036818504333496,
971
- 0.7494423389434814,
972
- 0.8013620376586914,
973
- 0.7948620319366455,
974
- 0.7637408971786499,
975
- 0.7909289598464966,
976
- 0.770110011100769,
977
- 0.7980424761772156,
978
- 0.7875586152076721,
979
- 0.7790180444717407,
980
- 0.7909952402114868,
981
- 0.7824040651321411,
982
- 0.8026537299156189,
983
- 0.7915074825286865,
984
- 0.788095235824585,
985
- 0.7919423580169678,
986
- 0.7925325632095337,
987
- 0.7777642011642456,
988
- 0.7709310054779053,
989
- 0.7793556451797485,
990
- 0.7943840622901917,
991
- 0.7924278974533081,
992
- 0.8102889060974121,
993
- 0.7902472019195557,
994
- 0.7818676233291626,
995
- 0.7886601686477661,
996
- 0.79351407289505,
997
- 0.7754731774330139,
998
- 0.7955225706100464,
999
- 0.7862548828125,
1000
- 0.7936201691627502,
1001
- 0.783572256565094,
1002
- 0.7728341817855835,
1003
- 0.788084864616394,
1004
- 0.7876660823822021,
1005
- 0.7964785695075989,
1006
- 0.7931740283966064,
1007
- 0.7881606221199036,
1008
- 0.7883685231208801,
1009
- 0.7716016173362732,
1010
- 0.7924609780311584,
1011
- 0.7832834720611572,
1012
- 0.7994821071624756,
1013
- 0.7779035568237305,
1014
- 0.7978600263595581,
1015
- 0.8097856640815735,
1016
- 0.7866764068603516,
1017
- 0.8036105632781982,
1018
- 0.7699865102767944,
1019
- 0.7888925075531006,
1020
- 0.7895327806472778,
1021
- 0.7913026213645935,
1022
- 0.7800164818763733,
1023
- 0.8145246505737305,
1024
- 0.7796132564544678,
1025
- 0.7778063416481018,
1026
- 0.7715652585029602,
1027
- 0.7749142050743103,
1028
- 0.7954548597335815,
1029
- 0.7830942273139954,
1030
- 0.8039675354957581,
1031
- 0.7806425094604492,
1032
- 0.7676209211349487,
1033
- 0.7841391563415527,
1034
- 0.7951662540435791,
1035
- 0.7991046905517578,
1036
- 0.7963618636131287,
1037
- 0.7817696332931519,
1038
- 0.7930166721343994,
1039
- 0.7826007604598999,
1040
- 0.7685734629631042,
1041
- 0.7858765125274658,
1042
- 0.8090939521789551,
1043
- 0.8009799122810364,
1044
- 0.7795686721801758,
1045
- 0.7901196479797363,
1046
- 0.7911461591720581,
1047
- 0.7952975034713745,
1048
- 0.7883354425430298,
1049
- 0.7776168584823608,
1050
- 0.8164061903953552,
1051
- 0.7778499126434326,
1052
- 0.7849786281585693,
1053
- 0.7973150014877319,
1054
- 0.772922158241272,
1055
- 0.8022127747535706,
1056
- 0.8043490648269653,
1057
- 0.7662113308906555,
1058
- 0.8030599355697632,
1059
- 0.7810326814651489,
1060
- 0.7946909666061401,
1061
- 0.7973261475563049,
1062
- 0.7984815835952759,
1063
- 0.7814211845397949,
1064
- 0.7801750302314758,
1065
- 0.8036614656448364,
1066
- 0.7942310571670532,
1067
- 0.7816305160522461,
1068
- 0.8027132749557495,
1069
- 0.804101824760437,
1070
- 0.7808508276939392,
1071
- 0.782733142375946,
1072
- 0.7827922105789185,
1073
- 0.7943124771118164,
1074
- 0.8017924427986145,
1075
- 0.7825665473937988,
1076
- 0.7808253765106201,
1077
- 0.7813396453857422,
1078
- 0.7967122793197632,
1079
- 0.7627602815628052,
1080
- 0.7639561295509338,
1081
- 0.7797435522079468,
1082
- 0.7936455607414246,
1083
- 0.7791507244110107,
1084
- 0.7959131002426147,
1085
- 0.8002741932868958,
1086
- 0.8081020712852478,
1087
- 0.7892801761627197,
1088
- 0.79875648021698,
1089
- 0.7993462681770325,
1090
- 0.7802019119262695,
1091
- 0.8199928402900696,
1092
- 0.8017476797103882,
1093
- 0.7753216028213501,
1094
- 0.780362606048584,
1095
- 0.7781493067741394,
1096
- 0.778732180595398,
1097
- 0.7894307374954224,
1098
- 0.7903071641921997,
1099
- 0.8148328065872192,
1100
- 0.7744537591934204,
1101
- 0.8042833805084229,
1102
- 0.7727539539337158,
1103
- 0.7995544672012329,
1104
- 0.7929089069366455,
1105
- 0.7770389318466187,
1106
- 0.7632914185523987,
1107
- 0.8017297983169556,
1108
- 0.7955401539802551,
1109
- 0.7855898141860962,
1110
- 0.775995135307312,
1111
- 0.7973558902740479,
1112
- 0.800584614276886,
1113
- 0.8001367449760437,
1114
- 0.7788395285606384,
1115
- 0.8059795498847961,
1116
- 0.7909436225891113,
1117
- 0.7842425107955933,
1118
- 0.7752494215965271,
1119
- 0.7953119277954102,
1120
- 0.775480329990387,
1121
- 0.7822425365447998,
1122
- 0.7842751741409302,
1123
- 0.7888025045394897,
1124
- 0.7997403144836426,
1125
- 0.7784934043884277,
1126
- 0.8046996593475342,
1127
- 0.8008033037185669,
1128
- 0.7980520725250244,
1129
- 0.8104623556137085,
1130
- 0.7850831151008606,
1131
- 0.7809212803840637,
1132
- 0.7647312879562378,
1133
- 0.811896800994873,
1134
- 0.7907377481460571,
1135
- 0.8004019856452942,
1136
- 0.7688952684402466,
1137
- 0.7849960923194885,
1138
- 0.7880919575691223,
1139
- 0.7841589450836182,
1140
- 0.7888523936271667,
1141
- 0.7919774055480957,
1142
- 0.7947327494621277,
1143
- 0.7960951328277588,
1144
- 0.7793362140655518,
1145
- 0.784282922744751,
1146
- 0.7991900444030762,
1147
- 0.7833361029624939
1148
  ],
1149
  "loss_window": 200,
1150
  "grad_ema_decay": 0.99,
@@ -1177,30 +1206,30 @@
1177
  },
1178
  "committed_scale": 1.0,
1179
  "attempt_factor": 1.0,
1180
- "fast_loss": 0.7892285201560976,
1181
- "slow_loss": 0.788922989512829,
1182
  "best_loss": 0.7812624632790134,
1183
  "reference": {
1184
- "loss": 0.7842847464919089,
1185
- "grad_median": 0.1277403510408516,
1186
- "grad_p90": 0.1423632360760495,
1187
- "step": 330000,
1188
- "promotions": 3
1189
  },
1190
  "last_metrics": {
1191
- "loss": 0.7871961006522179,
1192
- "grad_median": 0.12022186443209648,
1193
- "grad_p90": 0.14998193234205245,
1194
  "skip_rate": 0.0,
1195
  "relative_spike_rate": 0.0
1196
  },
1197
- "last_loss_ratio": 1.0046444523217832,
1198
- "last_grad_ratio": 0.9274949063351521,
1199
- "last_grad_p90_ratio": 1.0678019922477653,
1200
  "healthy_windows": 0,
1201
  "warning_windows": 0,
1202
  "retry_windows": 0,
1203
- "windows_seen": 87,
1204
  "retry_count": 0
1205
  },
1206
  "best_window": 0.7812624632790134,
@@ -1208,14 +1237,14 @@
1208
  },
1209
  "transaction": {
1210
  "status": "committed",
1211
- "committed_at": "2026-07-22T17:35:06.684165+00:00",
1212
  "attempt": 0,
1213
  "health_reference": {
1214
- "loss": 0.7842847464919089,
1215
- "grad_median": 0.1277403510408516,
1216
- "grad_p90": 0.1423632360760495,
1217
- "step": 330000,
1218
- "promotions": 3
1219
  }
1220
  },
1221
  "config": {
 
1
  {
2
+ "step": 340000,
3
  "run_history": [
4
  {
5
  "start_step": 0,
 
935
  "seed_offset": 0,
936
  "target_steps": 400000
937
  }
938
+ },
939
+ {
940
+ "start_step": 330000,
941
+ "end_step": 340000,
942
+ "seconds": 4488.9,
943
+ "world": 1,
944
+ "objective": "flow",
945
+ "completed": false,
946
+ "finished_at": "2026-07-22T18:49:55.630085+00:00",
947
+ "promotion_reason": "periodic healthy candidate",
948
+ "effective": {
949
+ "attempt": 0,
950
+ "precision": "tf32",
951
+ "lr_policy": "constant",
952
+ "lr_base": 0.0001,
953
+ "lr_start": null,
954
+ "lr_end": 0.0001,
955
+ "lr_min": 1e-05,
956
+ "lr_hard_min": 1e-06,
957
+ "lr_scale": 1.0,
958
+ "weight_decay": 0.1,
959
+ "wd_ada": 0.0,
960
+ "clip": 1.0,
961
+ "spike_skip": 4.0,
962
+ "grad_ceiling": 25.0,
963
+ "steps_skipped": 0,
964
+ "seed_offset": 0,
965
+ "target_steps": 400000
966
+ }
967
  }
968
  ],
969
  "guard_state": {
970
  "version": 3,
971
+ "live_grad_ema": 0.12020335339737151,
972
+ "grad_ema": 0.12020335339737151,
973
+ "grad_reference": 0.1252626042999359,
974
+ "grad_p90_reference": 0.14129488615307212,
975
  "spikes_total": 0,
976
  "recent_losses": [
977
+ 0.7837042212486267,
978
+ 0.7959539890289307,
979
+ 0.7837053537368774,
980
+ 0.792834997177124,
981
+ 0.79377281665802,
982
+ 0.7859923839569092,
983
+ 0.792047381401062,
984
+ 0.782581627368927,
985
+ 0.7856507897377014,
986
+ 0.7816096544265747,
987
+ 0.7914053797721863,
988
+ 0.7798425555229187,
989
+ 0.7864272594451904,
990
+ 0.7811540365219116,
991
+ 0.7880648374557495,
992
+ 0.7859886288642883,
993
+ 0.7744649648666382,
994
+ 0.7812007665634155,
995
+ 0.7754606604576111,
996
+ 0.7733634114265442,
997
+ 0.8067691326141357,
998
+ 0.8012179732322693,
999
+ 0.7883100509643555,
1000
+ 0.788763165473938,
1001
+ 0.7792775630950928,
1002
+ 0.8012402653694153,
1003
+ 0.7777899503707886,
1004
+ 0.7971938252449036,
1005
+ 0.792236864566803,
1006
+ 0.7755023837089539,
1007
+ 0.7745698690414429,
1008
+ 0.7884120941162109,
1009
+ 0.791738748550415,
1010
+ 0.7844906449317932,
1011
+ 0.7762939929962158,
1012
+ 0.7786762714385986,
1013
+ 0.7933378219604492,
1014
+ 0.778876006603241,
1015
+ 0.7844874262809753,
1016
+ 0.801080584526062,
1017
+ 0.801337718963623,
1018
+ 0.7846294045448303,
1019
+ 0.7994412183761597,
1020
+ 0.803071141242981,
1021
+ 0.8036864995956421,
1022
+ 0.785882830619812,
1023
+ 0.7692743539810181,
1024
+ 0.822243332862854,
1025
+ 0.7934736013412476,
1026
+ 0.7783013582229614,
1027
+ 0.7774749994277954,
1028
+ 0.7859049439430237,
1029
+ 0.7793117761611938,
1030
+ 0.788796603679657,
1031
+ 0.7878576517105103,
1032
+ 0.7736337184906006,
1033
+ 0.7677261233329773,
1034
+ 0.7941679954528809,
1035
+ 0.8026267290115356,
1036
+ 0.779779851436615,
1037
+ 0.7970684766769409,
1038
+ 0.7781627178192139,
1039
+ 0.7830750346183777,
1040
+ 0.77712082862854,
1041
+ 0.8019779920578003,
1042
+ 0.7880858182907104,
1043
+ 0.7866196632385254,
1044
+ 0.7943106889724731,
1045
+ 0.7967958450317383,
1046
+ 0.795556366443634,
1047
+ 0.8007069826126099,
1048
+ 0.7869127988815308,
1049
+ 0.7826520204544067,
1050
+ 0.7849986553192139,
1051
+ 0.7721657156944275,
1052
+ 0.803903341293335,
1053
+ 0.7791146039962769,
1054
+ 0.7936654090881348,
1055
+ 0.767623245716095,
1056
+ 0.7675718069076538,
1057
+ 0.7931200861930847,
1058
+ 0.7867366671562195,
1059
+ 0.7877177000045776,
1060
+ 0.7896507978439331,
1061
+ 0.786452054977417,
1062
+ 0.7948963642120361,
1063
+ 0.7878752946853638,
1064
+ 0.7899014353752136,
1065
+ 0.7899609804153442,
1066
+ 0.782412052154541,
1067
+ 0.7935982942581177,
1068
+ 0.7736024856567383,
1069
+ 0.7785900831222534,
1070
+ 0.7784326076507568,
1071
+ 0.7804492115974426,
1072
+ 0.7623425722122192,
1073
+ 0.7742981910705566,
1074
+ 0.7792484760284424,
1075
+ 0.7916824817657471,
1076
+ 0.7861485481262207,
1077
+ 0.7760215997695923,
1078
+ 0.7681106925010681,
1079
+ 0.8021383285522461,
1080
+ 0.7817934155464172,
1081
+ 0.8022172451019287,
1082
+ 0.775996744632721,
1083
+ 0.7958743572235107,
1084
+ 0.7883899211883545,
1085
+ 0.7901175022125244,
1086
+ 0.7906835675239563,
1087
+ 0.7732946872711182,
1088
+ 0.8135435581207275,
1089
+ 0.7810014486312866,
1090
+ 0.7907371520996094,
1091
+ 0.8027175068855286,
1092
+ 0.7819023728370667,
1093
+ 0.7863631248474121,
1094
+ 0.7875015735626221,
1095
+ 0.7861364483833313,
1096
+ 0.788798451423645,
1097
+ 0.7756012082099915,
1098
+ 0.785270094871521,
1099
+ 0.7852610349655151,
1100
+ 0.779775857925415,
1101
+ 0.7805674076080322,
1102
+ 0.7850143909454346,
1103
+ 0.7723137140274048,
1104
+ 0.7699758410453796,
1105
+ 0.7957923412322998,
1106
+ 0.7892879247665405,
1107
+ 0.780127763748169,
1108
+ 0.8004384636878967,
1109
+ 0.7862828373908997,
1110
+ 0.7960135340690613,
1111
+ 0.788328230381012,
1112
+ 0.7829279899597168,
1113
+ 0.7782718539237976,
1114
+ 0.794108510017395,
1115
+ 0.784558892250061,
1116
+ 0.7971929907798767,
1117
+ 0.778393566608429,
1118
+ 0.7833342552185059,
1119
+ 0.7887024879455566,
1120
+ 0.7849143743515015,
1121
+ 0.7819156646728516,
1122
+ 0.7891097068786621,
1123
+ 0.8074806332588196,
1124
+ 0.7968727946281433,
1125
+ 0.809982180595398,
1126
+ 0.7997944355010986,
1127
+ 0.7953475117683411,
1128
+ 0.7785589098930359,
1129
+ 0.7856442928314209,
1130
+ 0.8010127544403076,
1131
+ 0.7681992053985596,
1132
+ 0.7907735705375671,
1133
+ 0.7933133840560913,
1134
+ 0.7969657182693481,
1135
+ 0.7840414047241211,
1136
+ 0.7930958271026611,
1137
+ 0.7833120226860046,
1138
+ 0.7698837518692017,
1139
+ 0.7933302521705627,
1140
+ 0.7943509221076965,
1141
+ 0.7820397615432739,
1142
+ 0.7846950888633728,
1143
+ 0.7892861366271973,
1144
+ 0.7819240689277649,
1145
+ 0.7748798131942749,
1146
+ 0.8055250644683838,
1147
+ 0.7734441757202148,
1148
+ 0.7720449566841125,
1149
+ 0.7771936655044556,
1150
+ 0.7850109338760376,
1151
+ 0.8016359210014343,
1152
+ 0.7904503345489502,
1153
+ 0.7859876155853271,
1154
+ 0.7981605529785156,
1155
+ 0.7825950384140015,
1156
+ 0.7893416285514832,
1157
+ 0.7868399620056152,
1158
+ 0.7819130420684814,
1159
+ 0.7937236428260803,
1160
+ 0.7989599704742432,
1161
+ 0.7960274815559387,
1162
+ 0.7877436876296997,
1163
+ 0.7913813591003418,
1164
+ 0.7801880836486816,
1165
+ 0.8012137413024902,
1166
+ 0.7700861692428589,
1167
+ 0.7817973494529724,
1168
+ 0.7865397334098816,
1169
+ 0.782233715057373,
1170
+ 0.7874511480331421,
1171
+ 0.8039686679840088,
1172
+ 0.7799859642982483,
1173
+ 0.7927159070968628,
1174
+ 0.7713406085968018,
1175
+ 0.7861450910568237,
1176
+ 0.8100098967552185
1177
  ],
1178
  "loss_window": 200,
1179
  "grad_ema_decay": 0.99,
 
1206
  },
1207
  "committed_scale": 1.0,
1208
  "attempt_factor": 1.0,
1209
+ "fast_loss": 0.7878259655905862,
1210
+ "slow_loss": 0.788495003310165,
1211
  "best_loss": 0.7812624632790134,
1212
  "reference": {
1213
+ "loss": 0.7849236974072455,
1214
+ "grad_median": 0.1252626042999359,
1215
+ "grad_p90": 0.14129488615307212,
1216
+ "step": 340000,
1217
+ "promotions": 4
1218
  },
1219
  "last_metrics": {
1220
+ "loss": 0.7874795010685921,
1221
+ "grad_median": 0.1153516173362732,
1222
+ "grad_p90": 0.13702148646116255,
1223
  "skip_rate": 0.0,
1224
  "relative_spike_rate": 0.0
1225
  },
1226
+ "last_loss_ratio": 1.0040734625924745,
1227
+ "last_grad_ratio": 0.903016285741876,
1228
+ "last_grad_p90_ratio": 0.9624780261946742,
1229
  "healthy_windows": 0,
1230
  "warning_windows": 0,
1231
  "retry_windows": 0,
1232
+ "windows_seen": 137,
1233
  "retry_count": 0
1234
  },
1235
  "best_window": 0.7812624632790134,
 
1237
  },
1238
  "transaction": {
1239
  "status": "committed",
1240
+ "committed_at": "2026-07-22T18:49:55.630160+00:00",
1241
  "attempt": 0,
1242
  "health_reference": {
1243
+ "loss": 0.7849236974072455,
1244
+ "grad_median": 0.1252626042999359,
1245
+ "grad_p90": 0.14129488615307212,
1246
+ "step": 340000,
1247
+ "promotions": 4
1248
  }
1249
  },
1250
  "config": {