Add forward self-model checkpoints (Llama 3.2 1B + toy GPT)

302675f verified about 1 month ago

12.9 kB

	{
	"source_layer": 7,
	"target_layer": 8,
	"d_model": 2048,
	"seq_len": 2048,
	"fwd_n_layer": 1,
	"fwd_n_head": 1,
	"fwd_d_head": 128,
	"fwd_mlp_mult": 2.0,
	"fwd_use_swiglu": true,
	"fwd_n_params": 26235264,
	"main_n_params": 1235814400,
	"capacity_ratio": 0.021229129552139868,
	"n_steps": 10000,
	"batch_size": 16,
	"fwd_lr": 0.0001,
	"n_train_shards": 99,
	"n_val_shards": 2,
	"n_val_sequences": 491,
	"final_val_mse": 0.001788996768394305,
	"final_val_cosine": 0.9372754942986273,
	"final_val_residual_norm": 1.8941153287887573,
	"residual_pca": {
	"effective_rank": 1908.2322998046875,
	"max_rank": 2048,
	"rank_50": 485,
	"rank_90": 1419,
	"rank_95": 1650,
	"top1_variance": 0.005640974268317223
	},
	"training_seconds": 12340.371667385101,
	"history": {
	"fwd_mse": [
	[
	0,
	0.05361234396696091
	],
	[
	200,
	0.005762346554547548
	],
	[
	400,
	0.004786794073879719
	],
	[
	600,
	0.003985430113971233
	],
	[
	800,
	0.0037297275848686695
	],
	[
	1000,
	0.0033700044732540846
	],
	[
	1200,
	0.003214194206520915
	],
	[
	1400,
	0.003064497606828809
	],
	[
	1600,
	0.0030410615727305412
	],
	[
	1800,
	0.002859783126041293
	],
	[
	2000,
	0.0029426398687064648
	],
	[
	2200,
	0.0027721929363906384
	],
	[
	2400,
	0.002723717363551259
	],
	[
	2600,
	0.0027984725311398506
	],
	[
	2800,
	0.002662141341716051
	],
	[
	3000,
	0.0026135139632970095
	],
	[
	3200,
	0.0025119641795754433
	],
	[
	3400,
	0.002608730923384428
	],
	[
	3600,
	0.002473900094628334
	],
	[
	3800,
	0.002471000887453556
	],
	[
	4000,
	0.002470198320224881
	],
	[
	4200,
	0.0024023596197366714
	],
	[
	4400,
	0.0023067130241543055
	],
	[
	4600,
	0.002266491763293743
	],
	[
	4800,
	0.0022853089030832052
	],
	[
	5000,
	0.0022200047969818115
	],
	[
	5200,
	0.002257999498397112
	],
	[
	5400,
	0.0022179423831403255
	],
	[
	5600,
	0.002188409212976694
	],
	[
	5800,
	0.0021320050582289696
	],
	[
	6000,
	0.0020800644997507334
	],
	[
	6200,
	0.002177655231207609
	],
	[
	6400,
	0.0020370068959891796
	],
	[
	6600,
	0.0020786449313163757
	],
	[
	6800,
	0.001991601660847664
	],
	[
	7000,
	0.00202348455786705
	],
	[
	7200,
	0.001995587255805731
	],
	[
	7400,
	0.001970375655218959
	],
	[
	7600,
	0.001978542422875762
	],
	[
	7800,
	0.001997516490519047
	],
	[
	8000,
	0.0018945611082017422
	],
	[
	8200,
	0.0019200685201212764
	],
	[
	8400,
	0.0019268350442871451
	],
	[
	8600,
	0.0018792669288814068
	],
	[
	8800,
	0.0018499521538615227
	],
	[
	9000,
	0.0019255225779488683
	],
	[
	9200,
	0.0017902057152241468
	],
	[
	9400,
	0.0018028962658718228
	],
	[
	9600,
	0.0018486585468053818
	],
	[
	9800,
	0.0018617857713252306
	],
	[
	9999,
	0.0017257111612707376
	]
	],
	"val_fwd_mse": [
	[
	0,
	0.03311384047071139
	],
	[
	200,
	0.005512383347377181
	],
	[
	400,
	0.0045879316671440995
	],
	[
	600,
	0.003962893248535692
	],
	[
	800,
	0.003686679759994149
	],
	[
	1000,
	0.003359384679545959
	],
	[
	1200,
	0.0032044988125562667
	],
	[
	1400,
	0.003068833821453154
	],
	[
	1600,
	0.002978292355934779
	],
	[
	1800,
	0.002922045214412113
	],
	[
	2000,
	0.0028643588225046794
	],
	[
	2200,
	0.0028142516734078527
	],
	[
	2400,
	0.0027492424395556252
	],
	[
	2600,
	0.002699357736855745
	],
	[
	2800,
	0.002640491467900574
	],
	[
	3000,
	0.0026000263014187416
	],
	[
	3200,
	0.0025552748236805202
	],
	[
	3400,
	0.0025211286110182604
	],
	[
	3600,
	0.002512407225246231
	],
	[
	3800,
	0.00245760358714809
	],
	[
	4000,
	0.0024378761183470486
	],
	[
	4200,
	0.002386905951425433
	],
	[
	4400,
	0.0023764715219537417
	],
	[
	4600,
	0.0023403250457098086
	],
	[
	4800,
	0.00231518279761076
	],
	[
	5000,
	0.0022747613489627837
	],
	[
	5200,
	0.0022313310919950405
	],
	[
	5400,
	0.0022013659744213026
	],
	[
	5600,
	0.0021793171996250748
	],
	[
	5800,
	0.0021587318119903405
	],
	[
	6000,
	0.002127136965282261
	],
	[
	6200,
	0.002119861962273717
	],
	[
	6400,
	0.002091070303382973
	],
	[
	6600,
	0.0020678113680332897
	],
	[
	6800,
	0.0020570720623557764
	],
	[
	7000,
	0.0020369751223673423
	],
	[
	7200,
	0.0020223997300490735
	],
	[
	7400,
	0.001986565021798015
	],
	[
	7600,
	0.0019637826131656765
	],
	[
	7800,
	0.0019431026380819579
	],
	[
	8000,
	0.0019247106043621897
	],
	[
	8200,
	0.0019088692807902893
	],
	[
	8400,
	0.001896107786645492
	],
	[
	8600,
	0.0018856525033091505
	],
	[
	8800,
	0.0018764173689608773
	],
	[
	9000,
	0.0018630559517381093
	],
	[
	9200,
	0.001854456685638676
	],
	[
	9400,
	0.0018435753881931305
	],
	[
	9600,
	0.0018305608769878744
	],
	[
	9800,
	0.0018034361147632203
	],
	[
	9999,
	0.001789538178127259
	]
	],
	"val_cosine_sim": [
	[
	0,
	0.4243091017007828
	],
	[
	200,
	0.8082238614559174
	],
	[
	400,
	0.8376828610897065
	],
	[
	600,
	0.8584890842437745
	],
	[
	800,
	0.8687650005022685
	],
	[
	1000,
	0.8799132625261943
	],
	[
	1200,
	0.8853773395220439
	],
	[
	1400,
	0.8903553942839305
	],
	[
	1600,
	0.8936428467432658
	],
	[
	1800,
	0.8957931141058604
	],
	[
	2000,
	0.8978019853432974
	],
	[
	2200,
	0.8997232178846996
	],
	[
	2400,
	0.9020942270755767
	],
	[
	2600,
	0.9039487282435099
	],
	[
	2800,
	0.9061655422051748
	],
	[
	3000,
	0.9076851467291515
	],
	[
	3200,
	0.9093430797259013
	],
	[
	3400,
	0.9105890274047852
	],
	[
	3600,
	0.9109355668226878
	],
	[
	3800,
	0.912908277908961
	],
	[
	4000,
	0.91365660627683
	],
	[
	4200,
	0.9155096729596456
	],
	[
	4400,
	0.9158911069234212
	],
	[
	4600,
	0.9172228117783864
	],
	[
	4800,
	0.9181304196516673
	],
	[
	5000,
	0.9196795304616292
	],
	[
	5200,
	0.9212888975938162
	],
	[
	5400,
	0.922396965821584
	],
	[
	5600,
	0.9231784681479136
	],
	[
	5800,
	0.9239337881406148
	],
	[
	6000,
	0.9250807682673137
	],
	[
	6200,
	0.9253247896830241
	],
	[
	6400,
	0.9263921101888021
	],
	[
	6600,
	0.9272292335828145
	],
	[
	6800,
	0.9276153564453125
	],
	[
	7000,
	0.9283371845881144
	],
	[
	7200,
	0.9288679758707682
	],
	[
	7400,
	0.9302072445551555
	],
	[
	7600,
	0.9310539762179056
	],
	[
	7800,
	0.9318127512931824
	],
	[
	8000,
	0.9324680864810944
	],
	[
	8200,
	0.9330347994963328
	],
	[
	8400,
	0.9334850251674652
	],
	[
	8600,
	0.9338701903820038
	],
	[
	8800,
	0.9342047671477
	],
	[
	9000,
	0.9346590379873911
	],
	[
	9200,
	0.9349598467350007
	],
	[
	9400,
	0.9353463292121887
	],
	[
	9600,
	0.9358185668786366
	],
	[
	9800,
	0.9368309001127879
	],
	[
	9999,
	0.9373501121997834
	]
	],
	"val_residual_norm": [
	[
	0,
	8.19957447052002
	],
	[
	200,
	3.3171073754628497
	],
	[
	400,
	3.038803553581238
	],
	[
	600,
	2.8268452167510985
	],
	[
	800,
	2.7267664591471354
	],
	[
	1000,
	2.601663025220235
	],
	[
	1200,
	2.540529958407084
	],
	[
	1400,
	2.485924998919169
	],
	[
	1600,
	2.4484671036402386
	],
	[
	1800,
	2.42530730565389
	],
	[
	2000,
	2.400950050354004
	],
	[
	2200,
	2.379831910133362
	],
	[
	2400,
	2.3519990046819053
	],
	[
	2600,
	2.3303396622339885
	],
	[
	2800,
	2.3045204798380534
	],
	[
	3000,
	2.2867403507232664
	],
	[
	3200,
	2.2670581499735514
	],
	[
	3400,
	2.251586365699768
	],
	[
	3600,
	2.2480901082356772
	],
	[
	3800,
	2.222928269704183
	],
	[
	4000,
	2.2138659000396728
	],
	[
	4200,
	2.190362556775411
	],
	[
	4400,
	2.1856037537256876
	],
	[
	4600,
	2.1689730246861774
	],
	[
	4800,
	2.156961464881897
	],
	[
	5000,
	2.1379854599634807
	],
	[
	5200,
	2.11747457186381
	],
	[
	5400,
	2.102999965349833
	],
	[
	5600,
	2.092522080739339
	],
	[
	5800,
	2.082704210281372
	],
	[
	6000,
	2.067051116625468
	],
	[
	6200,
	2.063450400034587
	],
	[
	6400,
	2.049210047721863
	],
	[
	6600,
	2.03757913907369
	],
	[
	6800,
	2.032305860519409
	],
	[
	7000,
	2.0222584287325542
	],
	[
	7200,
	2.015055950482686
	],
	[
	7400,
	1.996809720993042
	],
	[
	7600,
	1.9852852423985798
	],
	[
	7800,
	1.9748168031374613
	],
	[
	8000,
	1.9652122696240744
	],
	[
	8200,
	1.9570548454920451
	],
	[
	8400,
	1.9504539052645364
	],
	[
	8600,
	1.9450278560320535
	],
	[
	8800,
	1.940179415543874
	],
	[
	9000,
	1.9331526279449462
	],
	[
	9200,
	1.9286702315012614
	],
	[
	9400,
	1.922955584526062
	],
	[
	9600,
	1.9158894975980123
	],
	[
	9800,
	1.9016074856122336
	],
	[
	9999,
	1.8942763884862264
	]
	]
	}
	}