summary.json

{
  "scope": "3 Adam + 3 AdamW official 8L/8H epoch-1 checkpoints; 12 prompts; anchor seed 1",
  "hardware": "NVIDIA L4",
  "repo": "karanbali/attention_head_seed_stability",
  "variants": {
    "adam": {
      "layer_stability": [
        0.9814916116495928,
        0.9607114770139258,
        0.9224479893843334,
        0.8490621962894995,
        0.7527976653849086,
        0.8661786348869402,
        0.9469099780544639,
        0.950835146009922
      ],
      "least_stable_layer": 5,
      "least_relative_depth": 0.625,
      "mean_stability": 0.9038043373341982,
      "query_norm_stability_spearman": -0.9523809523809524,
      "mean_prompt_perplexity": 196.73812000800626,
      "residual_cka": [
        0.9741605245105477,
        0.9553503451525314,
        0.9473035149150335,
        0.9723556029345708,
        0.9068565583949886,
        0.9893650954442251,
        0.9880713017815188,
        0.9823976624006416
      ],
      "residual_gt_head_all_layers": false
    },
    "adamw": {
      "layer_stability": [
        0.9849251722916961,
        0.9867621883749962,
        0.9350648929054538,
        0.9133710606644552,
        0.9499357336511214,
        0.9504338751236597,
        0.865437988191843,
        0.9677081070840359
      ],
      "least_stable_layer": 7,
      "least_relative_depth": 0.875,
      "mean_stability": 0.9442048772859077,
      "query_norm_stability_spearman": -0.7142857142857144,
      "mean_prompt_perplexity": 189.4770007571722,
      "residual_cka": [
        0.9864301705054231,
        0.9741782934736878,
        0.9719583095497574,
        0.967257467402012,
        0.973911791734401,
        0.9694202497757964,
        0.9405004347462198,
        0.9748416641140125
      ],
      "residual_gt_head_all_layers": false
    }
  },
  "cross_layer_alignment": {
    "adam": [
      [
        0.14150532257505974,
        0.13257707296753585,
        0.1264640194315259,
        0.12460012731592282,
        0.11975631131871377,
        0.11955000484869299,
        0.11983626253019537,
        0.11571087901220943
      ],
      [
        0.11331789828980687,
        0.1325477572990859,
        0.12824143791632417,
        0.12270668466297545,
        0.12643124851612483,
        0.12622829516235518,
        0.12562927989355527,
        0.12489739825963433
      ],
      [
        0.11806368465795276,
        0.12908369017943783,
        0.1311142481787447,
        0.1270420690610492,
        0.12436842632242615,
        0.12387709085943456,
        0.12198847098221474,
        0.12446231975859802
      ],
      [
        0.12303129846187094,
        0.12278085566012642,
        0.13740740304216345,
        0.1327451301639814,
        0.12442973650325215,
        0.12158941229868822,
        0.116081286886729,
        0.12193487698303195
      ],
      [
        0.12545575570324105,
        0.12202411428088498,
        0.13457822671704825,
        0.14059105400592767,
        0.12538038420456923,
        0.11904714089156238,
        0.11619977459158498,
        0.1167235496050149
      ],
      [
        0.11610723266480559,
        0.12935080458203435,
        0.12717407187945634,
        0.12445540216650489,
        0.12603209551843778,
        0.12621012435275836,
        0.1264487875492856,
        0.12422148128657134
      ],
      [
        0.10396297002975668,
        0.1304112216898516,
        0.12830128607357838,
        0.11892964680784048,
        0.12935263816706669,
        0.13040077297379957,
        0.12884059992799413,
        0.12980086432997642
      ],
      [
        0.10958491479771044,
        0.1298976353867104,
        0.12740995619329776,
        0.12011859754860016,
        0.12666744099888405,
        0.12799860495484366,
        0.1292428024148647,
        0.129080047704953
      ]
    ],
    "adamw": [
      [
        0.13652576936094316,
        0.13303351241553374,
        0.12723095456503242,
        0.1228736554177027,
        0.12137225350042466,
        0.12172822874887991,
        0.11909426251934531,
        0.11814136347199942
      ],
      [
        0.12875334094934265,
        0.12904657623395047,
        0.12609825719129256,
        0.12293588256147779,
        0.12377566898577122,
        0.12327890732347842,
        0.12316548861382265,
        0.1229458781407335
      ],
      [
        0.13441392614651865,
        0.1351614142852763,
        0.14343564175683174,
        0.12756146029581236,
        0.12071375942654816,
        0.12392680472544113,
        0.10896477859291895,
        0.10582221477049925
      ],
      [
        0.12465617583417603,
        0.12621010753546108,
        0.13618111054707338,
        0.13586344851014093,
        0.12360065504784096,
        0.12775480718759302,
        0.11364051876026489,
        0.11209317657730099
      ],
      [
        0.12472651446918075,
        0.12629364171969062,
        0.1254534960448209,
        0.12558764188843494,
        0.12543124959962082,
        0.12453388946324168,
        0.12409159071718692,
        0.12388197609769137
      ],
      [
        0.1250689003343884,
        0.1269693933296726,
        0.12390273287015699,
        0.1224827909229065,
        0.12569836298988266,
        0.12659755163542952,
        0.1251126152838941,
        0.1241676526335361
      ],
      [
        0.12577084758632157,
        0.12759120889369893,
        0.1259477080723299,
        0.12480081875957387,
        0.1252159835073952,
        0.12812268218336198,
        0.12206511592714817,
        0.12048563507002935
      ],
      [
        0.12522071679089117,
        0.12577831203567466,
        0.12456403514030633,
        0.12497748343382718,
        0.12468405743639502,
        0.12438856837104408,
        0.12513592206760665,
        0.1252509047241255
      ]
    ]
  },
  "adamw_minus_adam_mean_stability": 0.04040053995170956
}