{
  "source_layer": 7,
  "target_layer": 8,
  "d_model": 2048,
  "seq_len": 2048,
  "fwd_n_layer": 1,
  "fwd_n_head": 1,
  "fwd_d_head": 128,
  "fwd_mlp_mult": 2.0,
  "fwd_use_swiglu": true,
  "fwd_n_params": 26235264,
  "main_n_params": 1235814400,
  "capacity_ratio": 0.021229129552139868,
  "n_steps": 10000,
  "batch_size": 16,
  "fwd_lr": 0.0001,
  "n_train_shards": 99,
  "n_val_shards": 2,
  "n_val_sequences": 491,
  "final_val_mse": 0.001788996768394305,
  "final_val_cosine": 0.9372754942986273,
  "final_val_residual_norm": 1.8941153287887573,
  "residual_pca": {
    "effective_rank": 1908.2322998046875,
    "max_rank": 2048,
    "rank_50": 485,
    "rank_90": 1419,
    "rank_95": 1650,
    "top1_variance": 0.005640974268317223
  },
  "training_seconds": 12340.371667385101,
  "history": {
    "fwd_mse": [
      [
        0,
        0.05361234396696091
      ],
      [
        200,
        0.005762346554547548
      ],
      [
        400,
        0.004786794073879719
      ],
      [
        600,
        0.003985430113971233
      ],
      [
        800,
        0.0037297275848686695
      ],
      [
        1000,
        0.0033700044732540846
      ],
      [
        1200,
        0.003214194206520915
      ],
      [
        1400,
        0.003064497606828809
      ],
      [
        1600,
        0.0030410615727305412
      ],
      [
        1800,
        0.002859783126041293
      ],
      [
        2000,
        0.0029426398687064648
      ],
      [
        2200,
        0.0027721929363906384
      ],
      [
        2400,
        0.002723717363551259
      ],
      [
        2600,
        0.0027984725311398506
      ],
      [
        2800,
        0.002662141341716051
      ],
      [
        3000,
        0.0026135139632970095
      ],
      [
        3200,
        0.0025119641795754433
      ],
      [
        3400,
        0.002608730923384428
      ],
      [
        3600,
        0.002473900094628334
      ],
      [
        3800,
        0.002471000887453556
      ],
      [
        4000,
        0.002470198320224881
      ],
      [
        4200,
        0.0024023596197366714
      ],
      [
        4400,
        0.0023067130241543055
      ],
      [
        4600,
        0.002266491763293743
      ],
      [
        4800,
        0.0022853089030832052
      ],
      [
        5000,
        0.0022200047969818115
      ],
      [
        5200,
        0.002257999498397112
      ],
      [
        5400,
        0.0022179423831403255
      ],
      [
        5600,
        0.002188409212976694
      ],
      [
        5800,
        0.0021320050582289696
      ],
      [
        6000,
        0.0020800644997507334
      ],
      [
        6200,
        0.002177655231207609
      ],
      [
        6400,
        0.0020370068959891796
      ],
      [
        6600,
        0.0020786449313163757
      ],
      [
        6800,
        0.001991601660847664
      ],
      [
        7000,
        0.00202348455786705
      ],
      [
        7200,
        0.001995587255805731
      ],
      [
        7400,
        0.001970375655218959
      ],
      [
        7600,
        0.001978542422875762
      ],
      [
        7800,
        0.001997516490519047
      ],
      [
        8000,
        0.0018945611082017422
      ],
      [
        8200,
        0.0019200685201212764
      ],
      [
        8400,
        0.0019268350442871451
      ],
      [
        8600,
        0.0018792669288814068
      ],
      [
        8800,
        0.0018499521538615227
      ],
      [
        9000,
        0.0019255225779488683
      ],
      [
        9200,
        0.0017902057152241468
      ],
      [
        9400,
        0.0018028962658718228
      ],
      [
        9600,
        0.0018486585468053818
      ],
      [
        9800,
        0.0018617857713252306
      ],
      [
        9999,
        0.0017257111612707376
      ]
    ],
    "val_fwd_mse": [
      [
        0,
        0.03311384047071139
      ],
      [
        200,
        0.005512383347377181
      ],
      [
        400,
        0.0045879316671440995
      ],
      [
        600,
        0.003962893248535692
      ],
      [
        800,
        0.003686679759994149
      ],
      [
        1000,
        0.003359384679545959
      ],
      [
        1200,
        0.0032044988125562667
      ],
      [
        1400,
        0.003068833821453154
      ],
      [
        1600,
        0.002978292355934779
      ],
      [
        1800,
        0.002922045214412113
      ],
      [
        2000,
        0.0028643588225046794
      ],
      [
        2200,
        0.0028142516734078527
      ],
      [
        2400,
        0.0027492424395556252
      ],
      [
        2600,
        0.002699357736855745
      ],
      [
        2800,
        0.002640491467900574
      ],
      [
        3000,
        0.0026000263014187416
      ],
      [
        3200,
        0.0025552748236805202
      ],
      [
        3400,
        0.0025211286110182604
      ],
      [
        3600,
        0.002512407225246231
      ],
      [
        3800,
        0.00245760358714809
      ],
      [
        4000,
        0.0024378761183470486
      ],
      [
        4200,
        0.002386905951425433
      ],
      [
        4400,
        0.0023764715219537417
      ],
      [
        4600,
        0.0023403250457098086
      ],
      [
        4800,
        0.00231518279761076
      ],
      [
        5000,
        0.0022747613489627837
      ],
      [
        5200,
        0.0022313310919950405
      ],
      [
        5400,
        0.0022013659744213026
      ],
      [
        5600,
        0.0021793171996250748
      ],
      [
        5800,
        0.0021587318119903405
      ],
      [
        6000,
        0.002127136965282261
      ],
      [
        6200,
        0.002119861962273717
      ],
      [
        6400,
        0.002091070303382973
      ],
      [
        6600,
        0.0020678113680332897
      ],
      [
        6800,
        0.0020570720623557764
      ],
      [
        7000,
        0.0020369751223673423
      ],
      [
        7200,
        0.0020223997300490735
      ],
      [
        7400,
        0.001986565021798015
      ],
      [
        7600,
        0.0019637826131656765
      ],
      [
        7800,
        0.0019431026380819579
      ],
      [
        8000,
        0.0019247106043621897
      ],
      [
        8200,
        0.0019088692807902893
      ],
      [
        8400,
        0.001896107786645492
      ],
      [
        8600,
        0.0018856525033091505
      ],
      [
        8800,
        0.0018764173689608773
      ],
      [
        9000,
        0.0018630559517381093
      ],
      [
        9200,
        0.001854456685638676
      ],
      [
        9400,
        0.0018435753881931305
      ],
      [
        9600,
        0.0018305608769878744
      ],
      [
        9800,
        0.0018034361147632203
      ],
      [
        9999,
        0.001789538178127259
      ]
    ],
    "val_cosine_sim": [
      [
        0,
        0.4243091017007828
      ],
      [
        200,
        0.8082238614559174
      ],
      [
        400,
        0.8376828610897065
      ],
      [
        600,
        0.8584890842437745
      ],
      [
        800,
        0.8687650005022685
      ],
      [
        1000,
        0.8799132625261943
      ],
      [
        1200,
        0.8853773395220439
      ],
      [
        1400,
        0.8903553942839305
      ],
      [
        1600,
        0.8936428467432658
      ],
      [
        1800,
        0.8957931141058604
      ],
      [
        2000,
        0.8978019853432974
      ],
      [
        2200,
        0.8997232178846996
      ],
      [
        2400,
        0.9020942270755767
      ],
      [
        2600,
        0.9039487282435099
      ],
      [
        2800,
        0.9061655422051748
      ],
      [
        3000,
        0.9076851467291515
      ],
      [
        3200,
        0.9093430797259013
      ],
      [
        3400,
        0.9105890274047852
      ],
      [
        3600,
        0.9109355668226878
      ],
      [
        3800,
        0.912908277908961
      ],
      [
        4000,
        0.91365660627683
      ],
      [
        4200,
        0.9155096729596456
      ],
      [
        4400,
        0.9158911069234212
      ],
      [
        4600,
        0.9172228117783864
      ],
      [
        4800,
        0.9181304196516673
      ],
      [
        5000,
        0.9196795304616292
      ],
      [
        5200,
        0.9212888975938162
      ],
      [
        5400,
        0.922396965821584
      ],
      [
        5600,
        0.9231784681479136
      ],
      [
        5800,
        0.9239337881406148
      ],
      [
        6000,
        0.9250807682673137
      ],
      [
        6200,
        0.9253247896830241
      ],
      [
        6400,
        0.9263921101888021
      ],
      [
        6600,
        0.9272292335828145
      ],
      [
        6800,
        0.9276153564453125
      ],
      [
        7000,
        0.9283371845881144
      ],
      [
        7200,
        0.9288679758707682
      ],
      [
        7400,
        0.9302072445551555
      ],
      [
        7600,
        0.9310539762179056
      ],
      [
        7800,
        0.9318127512931824
      ],
      [
        8000,
        0.9324680864810944
      ],
      [
        8200,
        0.9330347994963328
      ],
      [
        8400,
        0.9334850251674652
      ],
      [
        8600,
        0.9338701903820038
      ],
      [
        8800,
        0.9342047671477
      ],
      [
        9000,
        0.9346590379873911
      ],
      [
        9200,
        0.9349598467350007
      ],
      [
        9400,
        0.9353463292121887
      ],
      [
        9600,
        0.9358185668786366
      ],
      [
        9800,
        0.9368309001127879
      ],
      [
        9999,
        0.9373501121997834
      ]
    ],
    "val_residual_norm": [
      [
        0,
        8.19957447052002
      ],
      [
        200,
        3.3171073754628497
      ],
      [
        400,
        3.038803553581238
      ],
      [
        600,
        2.8268452167510985
      ],
      [
        800,
        2.7267664591471354
      ],
      [
        1000,
        2.601663025220235
      ],
      [
        1200,
        2.540529958407084
      ],
      [
        1400,
        2.485924998919169
      ],
      [
        1600,
        2.4484671036402386
      ],
      [
        1800,
        2.42530730565389
      ],
      [
        2000,
        2.400950050354004
      ],
      [
        2200,
        2.379831910133362
      ],
      [
        2400,
        2.3519990046819053
      ],
      [
        2600,
        2.3303396622339885
      ],
      [
        2800,
        2.3045204798380534
      ],
      [
        3000,
        2.2867403507232664
      ],
      [
        3200,
        2.2670581499735514
      ],
      [
        3400,
        2.251586365699768
      ],
      [
        3600,
        2.2480901082356772
      ],
      [
        3800,
        2.222928269704183
      ],
      [
        4000,
        2.2138659000396728
      ],
      [
        4200,
        2.190362556775411
      ],
      [
        4400,
        2.1856037537256876
      ],
      [
        4600,
        2.1689730246861774
      ],
      [
        4800,
        2.156961464881897
      ],
      [
        5000,
        2.1379854599634807
      ],
      [
        5200,
        2.11747457186381
      ],
      [
        5400,
        2.102999965349833
      ],
      [
        5600,
        2.092522080739339
      ],
      [
        5800,
        2.082704210281372
      ],
      [
        6000,
        2.067051116625468
      ],
      [
        6200,
        2.063450400034587
      ],
      [
        6400,
        2.049210047721863
      ],
      [
        6600,
        2.03757913907369
      ],
      [
        6800,
        2.032305860519409
      ],
      [
        7000,
        2.0222584287325542
      ],
      [
        7200,
        2.015055950482686
      ],
      [
        7400,
        1.996809720993042
      ],
      [
        7600,
        1.9852852423985798
      ],
      [
        7800,
        1.9748168031374613
      ],
      [
        8000,
        1.9652122696240744
      ],
      [
        8200,
        1.9570548454920451
      ],
      [
        8400,
        1.9504539052645364
      ],
      [
        8600,
        1.9450278560320535
      ],
      [
        8800,
        1.940179415543874
      ],
      [
        9000,
        1.9331526279449462
      ],
      [
        9200,
        1.9286702315012614
      ],
      [
        9400,
        1.922955584526062
      ],
      [
        9600,
        1.9158894975980123
      ],
      [
        9800,
        1.9016074856122336
      ],
      [
        9999,
        1.8942763884862264
      ]
    ]
  }
}