{
 "format": "holo-2bit/1",
 "mode": "bitnet",
 "model": "bitnet2b",
 "source": "https://huggingface.co/microsoft/BitNet-b1.58-2B-4T-gguf/resolve/main/ggml-model-i2_s.gguf",
 "bits": 3,
 "layout": "q3f",
 "sub_norm": true,
 "ffn_act": "relu2",
 "twoBit": false,
 "incoherent": false,
 "root": "sha256:19e16e6d51b4a68f60861b80d8f279c212449bf181fbb10c200bc9c56fa40de8",
 "d": 2560,
 "n_heads": 20,
 "n_kv_heads": 5,
 "ff": 6912,
 "vocab": 128256,
 "n_layers": 30,
 "hd": 128,
 "rope_base": 500000,
 "attn_bias": false,
 "qk_norm": false,
 "qk_norm_dim": 0,
 "tied": true,
 "tensors": {
  "embed": {
   "fmt": "q3",
   "N": 128256,
   "K": 2560,
   "kappa": "sha256:86685c8b39cc4197dc10f1b9a23d16eb3c931e3d5dd3a75c745b35a167bbd922",
   "stored": 145127032
  },
  "final_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:f3ed6ddffe28303f7f2612995807174954542cf7c98168884beb4dd750365a76",
   "stored": 8573
  },
  "lm_head": {
   "fmt": "q3",
   "N": 128256,
   "K": 2560,
   "kappa": "sha256:86685c8b39cc4197dc10f1b9a23d16eb3c931e3d5dd3a75c745b35a167bbd922",
   "stored": 145127032
  },
  "l0.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:d4284d126d8514ebc1225bb8961597206122b8365d83d3d11aa90ec6cfef8de8",
   "stored": 8653
  },
  "l0.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.2188547849655151,
   "kappa": "sha256:eedcfec0fab223567654c9e8cba0e1b4e5e74577cb92f83f1ff093d57672ec54",
   "stored": 1170936
  },
  "l0.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 1.7940564155578613,
   "kappa": "sha256:c64d7815926f964b99f7b8b8e5015194435eafcbb24212e8b5f7ed6e9331e67c",
   "stored": 298793
  },
  "l0.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 2.2931101322174072,
   "kappa": "sha256:520cc1864a0b2b176bfdb72e91a8dd7e7b4a52aa2de2172d0bb139243b7710e5",
   "stored": 305533
  },
  "l0.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:6bccb42bec3f0d4df231babb3fba94a7c4b9cb8452ea7782715ac1bc2ffc7815",
   "stored": 9416
  },
  "l0.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 0.965597927570343,
   "kappa": "sha256:4a4d9737cfa843aa90a8b764260197db9ff7a8e6c7d5d94a217085a1ec0e1a7a",
   "stored": 1187473
  },
  "l0.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:d6679d1308683791618105f311b33ad5f8712f9957fc31226b4a3632cbab4ff4",
   "stored": 8362
  },
  "l0.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:e456d56b800c8f17ce57985fc6baa581a60ff16393fb1e1b29c74ec065a9a5a1",
   "stored": 24416
  },
  "l0.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 1.5512433052062988,
   "kappa": "sha256:cc7d52c53fb68ba9201935a4167b5b6c0f8dff8bebf291d5e0c0d595f5e8bce0",
   "stored": 3373881
  },
  "l0.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 1.8309355974197388,
   "kappa": "sha256:571acf66c108028bde670583ff93853ef98754d4b5e353f137f5a9b862854168",
   "stored": 3381723
  },
  "l0.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.163161277770996,
   "kappa": "sha256:1dfbf91a0d9d4b86dc5b7e694268cd2c69651a4cb55ccb0736b07a824fa65a26",
   "stored": 3521138
  },
  "l1.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:692ffeaa496938f7d8fbb246f4727e1b9a8ba7eb1dfe0eb3d2822ac1395e20ec",
   "stored": 8610
  },
  "l1.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 0.8356457352638245,
   "kappa": "sha256:44caf920488c5af1ed0a5c79394e6090d0e35ad83cfbebdfc1112178d0f75cbb",
   "stored": 1256231
  },
  "l1.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 1.3455936908721924,
   "kappa": "sha256:4075ea3fc9ecec622a7ec4af242d843e66937985376ca12e0afc60f62dd70c7e",
   "stored": 320250
  },
  "l1.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 2.25688099861145,
   "kappa": "sha256:52e8bab774c8aaa60ef3dbd68838184ac53ee839e6dc198795ab1f60afb2e36d",
   "stored": 311276
  },
  "l1.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:d6680a2abb414b60968d2d69681934998fdd6bfda67ec11e94461bc3accdf9bb",
   "stored": 9243
  },
  "l1.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.6071064472198486,
   "kappa": "sha256:aa5e364a81bc37d3d376669ecce76b1dd551a252011c7fde896b973478409a72",
   "stored": 1267180
  },
  "l1.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:3e6862c455f450cfd04660812af02f505f64703abc933414f36270950cf238ef",
   "stored": 8438
  },
  "l1.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:57284f98e494193ce830bf6a92cbcb89be1b4b508f95e4fb93b2686acb4f2856",
   "stored": 24676
  },
  "l1.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 0.8077176213264465,
   "kappa": "sha256:ffc322b414176fc97e37259f471a9c7efc3951526166d7d29251f5da0a7cbdf9",
   "stored": 2006458
  },
  "l1.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 0.7442138195037842,
   "kappa": "sha256:768d6296e7d5335cdcc324c4e21281fc52111bae79ee17ce00293db846f3de3d",
   "stored": 2005495
  },
  "l1.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 1.265028715133667,
   "kappa": "sha256:aabce90f8d5caa03a7b8eae9b576e4bb73660d386dfd0b0c7dbedee35d3c0e04",
   "stored": 3132364
  },
  "l2.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:7794cb0da3c3aaab04d324091c45ede2819e6a9d2b2ee6890cb58ad6af608db0",
   "stored": 8575
  },
  "l2.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 0.9968166947364807,
   "kappa": "sha256:b819cf2aa22b25d06470640f50d25d5715dcb64e0f9b1473f78490cd0e243387",
   "stored": 1280269
  },
  "l2.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 1.4827167987823486,
   "kappa": "sha256:8b6b2a7670e2a04468ab68898467fc09bd34855d04558c756f7c7fcb8f1bd2fe",
   "stored": 324212
  },
  "l2.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 2.706890344619751,
   "kappa": "sha256:b70762c07fb0ab8912a43e60ba6a23e3a6e30821f8dc1ee89c3f7f59027477ca",
   "stored": 322905
  },
  "l2.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:9e618587af45c53d49d0e9bf8529ca34aecf89bcaf9d4c1c86925c5393305b16",
   "stored": 9098
  },
  "l2.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.9340922832489014,
   "kappa": "sha256:563abfd7381f960a3b2eb3fab6272fadc5a18b3a75065c096cbc16c16b0fd58a",
   "stored": 1273515
  },
  "l2.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:3e28b72d7dd7f19b005a4f93f684ae4bd3c48fa641c8ad13652a8f6f1e977e44",
   "stored": 8482
  },
  "l2.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:3ee5263b59bdfff077ba4e3a3d38d1e38816abc52469df27cc33e0fccac3c570",
   "stored": 24384
  },
  "l2.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 1.1282522678375244,
   "kappa": "sha256:cbce5d17b05f295b2589d1f80267189bd3f118daa69dad2f81373f17f822fcc3",
   "stored": 2586298
  },
  "l2.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 1.025633454322815,
   "kappa": "sha256:886d6c83a74ddb05700d1bb12bfaa7e4928889b9096e9edc9f6b9f3e1bb8f590",
   "stored": 2586385
  },
  "l2.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 1.7018623352050781,
   "kappa": "sha256:a3f5480e02ea08533f6accb02305304dbd76b691fa40e6dcd49ee7a48906f52d",
   "stored": 3312977
  },
  "l3.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:2f227ee74351331034d0d25d25a13727d9f7e8bc59f3a406f7ab52c52934a7e7",
   "stored": 8578
  },
  "l3.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.0245068073272705,
   "kappa": "sha256:a0745a319d3746c224f702efe23f24c1e20ce69ebbef4755f3a2e59a546b4e11",
   "stored": 1270277
  },
  "l3.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 1.5373108386993408,
   "kappa": "sha256:07569ce168db3d50cf391632f9ac2b3d4f44e6f3426a6ff7c78b5b8dd53f5a0b",
   "stored": 321002
  },
  "l3.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 2.6927521228790283,
   "kappa": "sha256:95bf010dd048b49028fabc4988da5d1e0e6e97be13b59d38f19dd2a955397c4c",
   "stored": 322052
  },
  "l3.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:448aa10e6e669381f080f03a3a1dab6a9e46af610add6b9454ba6393318023c0",
   "stored": 9100
  },
  "l3.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.9021772146224976,
   "kappa": "sha256:3e43ee1b3dac78d34f2f48af4dfff7b98070bbcf79ba4dc14f6c06264e0f3840",
   "stored": 1290528
  },
  "l3.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:4c91661088cdb8058b202eb4b8e0a4a2785fe77a7619977614bf3cf18a6e074d",
   "stored": 8469
  },
  "l3.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:6f66043d29aea907fc9ebba9c99eaed244aa8fe9d2a8d7ea75d7d341d4e861b1",
   "stored": 24504
  },
  "l3.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 1.4292961359024048,
   "kappa": "sha256:730f4cc1b2fa77335f21d4938206cb2bdee421487c52a6c31de5d8391f355f9f",
   "stored": 3027164
  },
  "l3.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 1.2692362070083618,
   "kappa": "sha256:a6aeabc47065987ebc6b2181ce74cc592ef1d6135cafdbb1a6454a5612d2b4d7",
   "stored": 3026512
  },
  "l3.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 1.9713046550750732,
   "kappa": "sha256:36ffa2d25e4c56cceabaee0ce4c0d6fe1736704027c63aa0a50984b19405bfc8",
   "stored": 3432754
  },
  "l4.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:aef70afa4e15c94014138edd713e11353f53c3d0104c6efcb06c2f18f329c838",
   "stored": 8554
  },
  "l4.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.1376540660858154,
   "kappa": "sha256:a7c2d195b7106780d6dc646a55adc8a37dfcb5a056b1cee43d200711e18fa6ba",
   "stored": 1261583
  },
  "l4.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 1.8145620822906494,
   "kappa": "sha256:8ae00c20de1bd5f6c13f478d1403877bdc7fdc43c0ad39dd549c7030a53679de",
   "stored": 320169
  },
  "l4.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 3.0370285511016846,
   "kappa": "sha256:ac1f28d22c420a602ba869fee175530d2f01642dfe428388eb94538d2c2222ba",
   "stored": 326579
  },
  "l4.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:eb367f75b22575c830f0bcab0d24fd335f6930a9346ea235a61de4c724f07e73",
   "stored": 9061
  },
  "l4.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 2.1330158710479736,
   "kappa": "sha256:61b5f702f4e3bfd444d2dfba5151b18a6a330c7818409d54dfa66dde93e18d83",
   "stored": 1293375
  },
  "l4.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:f479519120b05443a9e9f4faaf08bd9c4d4e5e3a98114fd0449d9e6d685cc727",
   "stored": 8491
  },
  "l4.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:c31f2ed8859fc62267d313b9e04301a62c401653dabc4a88691a733d47d8887e",
   "stored": 24558
  },
  "l4.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 1.6233707666397095,
   "kappa": "sha256:e129f9c76ac0f244dc93b20eb4e524f52e745ba44ba64b0d7e458aca610315de",
   "stored": 3221178
  },
  "l4.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 1.4957562685012817,
   "kappa": "sha256:5199eef196960030362fe17bab617fbb1cff581aafbf25e0553498f13e9bcfb9",
   "stored": 3222898
  },
  "l4.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.13490629196167,
   "kappa": "sha256:ff81c04bcc1ab1351024224a516d7d1c9626de3d70b7ed33aefc11b18a1ae187",
   "stored": 3472411
  },
  "l5.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:971a860e4b5bb450e32ecf49acc10b1ac7f5e99e5ff84d616315ddaea81fcca1",
   "stored": 8579
  },
  "l5.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.2880576848983765,
   "kappa": "sha256:705d9f3b71f29883191c8e7b551177ac69ced46308863c4cd0e5619198787996",
   "stored": 1249387
  },
  "l5.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 1.9532663822174072,
   "kappa": "sha256:6ed451a23d0b381885e5f276c7e2ffdac4c659cd1302b904e30440f130c9912c",
   "stored": 316211
  },
  "l5.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 3.1583521366119385,
   "kappa": "sha256:8155fb31d9b06cdf3864361ad3c68f77725534867dfed2a523ca51393524055e",
   "stored": 323643
  },
  "l5.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:2abdf791b1319830d07e81386c6c5106188b7ecf99f8063db90bbdcdeee6b32e",
   "stored": 8998
  },
  "l5.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 2.144779682159424,
   "kappa": "sha256:72e908f9d71630e38c46d04be24ac4721527219c440a7bab373200aa9f3308f6",
   "stored": 1288762
  },
  "l5.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:43dc31c0c267117271541de63cec7ce8bf592ac4a72ac6efa08b547ade082b08",
   "stored": 8523
  },
  "l5.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:e8ed5c6c5ea6afc2995d6a65e5f1e538b0ba04d37743792ebfe52449c0c74ba8",
   "stored": 24616
  },
  "l5.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 1.7476176023483276,
   "kappa": "sha256:4f9035197f2933807c73b72c8111ead321842a2fd6157164125b56e28d2f9414",
   "stored": 3367757
  },
  "l5.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 1.6368310451507568,
   "kappa": "sha256:a59899440487f801a0e07470a782fbab9c74477a58e917b3bdc915952a2d9b8b",
   "stored": 3369828
  },
  "l5.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.177177906036377,
   "kappa": "sha256:873a0a0e7b7c772ef5c7785a74323eb7283bb715a170798631c2a7d66d7d3f7c",
   "stored": 3488556
  },
  "l6.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:fd97f2f81d8fd792584b189bf7887855d09d5d8f6285ec58a3cab173c73b6cfc",
   "stored": 8571
  },
  "l6.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.3680181503295898,
   "kappa": "sha256:6e16bda0b604b7837899991199d8cc301442bda267e00b2b76e939f6f9e39de1",
   "stored": 1245136
  },
  "l6.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 2.1691858768463135,
   "kappa": "sha256:77e4d3562eee8f1bfd7accbb77f4afb7db530fb37baa08a5425381b3d7dea550",
   "stored": 311065
  },
  "l6.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 3.480070114135742,
   "kappa": "sha256:b5ba51de8180b296588c3627d3a0c38d2e4fce871718c22caf4889723b1ea071",
   "stored": 325142
  },
  "l6.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:dd25aa4a8a78833ab9f5adaa4ab660fa747914dc5dedcbc4dca0a02313083c29",
   "stored": 9006
  },
  "l6.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 2.3384294509887695,
   "kappa": "sha256:f3b0526e24d1403a67ee1b35be1ecdc453f4ecb8696b7e2b6a6ab6643b77d46c",
   "stored": 1295959
  },
  "l6.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:18009c00b2b0debcc1db961502d5e7dc5c630facaacd3c4483e8fc03e1b16f64",
   "stored": 8538
  },
  "l6.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:2dbdb1b93dd74d82f9857eae5e997cf4ec2b30483f2baeaf0e8c4af49cc99631",
   "stored": 24486
  },
  "l6.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 1.8770447969436646,
   "kappa": "sha256:8c4a233e39c3b1c717446669c7cd4f452d3b337bc5b158ce5198b418a7ba70de",
   "stored": 3421320
  },
  "l6.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 1.7816859483718872,
   "kappa": "sha256:a3b7de9949de7bc843ddc237c837d395fd1b1d99a5e30eb21cbc2fb75ecd450b",
   "stored": 3423850
  },
  "l6.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.3278253078460693,
   "kappa": "sha256:8a109645c73199c149003d29145ba15e8b038140feefcddc208bf7d700bc9cf3",
   "stored": 3495370
  },
  "l7.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:72aac3bcca02692efb9ad921d4349a891458c479fefbef1fafc44924fb8d0cbd",
   "stored": 8600
  },
  "l7.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.4199094772338867,
   "kappa": "sha256:8a0dc9f2c20a1c17efdb8e7c7c22174979ec497f4f1e0ae23c57a196d3256159",
   "stored": 1246258
  },
  "l7.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 2.2624335289001465,
   "kappa": "sha256:77bca20e1dc197fd480ee4ddcdbc7f5ba093ce9d1242a03608f5eceb79b4b653",
   "stored": 312316
  },
  "l7.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 3.229888916015625,
   "kappa": "sha256:1c10f530cae746ab5ea59d226388ea343146d82e3e8223c39e45f171eaf2a9fb",
   "stored": 321444
  },
  "l7.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:d8e7963a5dbdef62a2f16c75e0df7236d4df9ab5ed88e3cb0e8a8baed768000e",
   "stored": 8748
  },
  "l7.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 2.244603157043457,
   "kappa": "sha256:780d1b78022e6bd680d647450927e92948a5c51e832935b0fcc41a4d18e9ba4f",
   "stored": 1293210
  },
  "l7.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:65f742abac762cdad7942532137459bbd3810fa3471fa3bebc91682b930045ea",
   "stored": 8522
  },
  "l7.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:15f48d29df002e1653c3da8b9eab8e38014a6ea7eb045b4b666ffcc59929772a",
   "stored": 24415
  },
  "l7.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 1.921570897102356,
   "kappa": "sha256:fb023a0b278fb708e11827e1ca0f9007c0e82cdf616bf3a59c56d170ae34d710",
   "stored": 3406036
  },
  "l7.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 1.835010290145874,
   "kappa": "sha256:e13b3ae8baff331918812dafce136fe62ddbc4fac1a5cac8419d673f80275537",
   "stored": 3410662
  },
  "l7.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.358863592147827,
   "kappa": "sha256:2886364019c1c76049929a048dab94c82f611c0985b92383d64f5ce492bb6fc0",
   "stored": 3494492
  },
  "l8.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:bb4a8eff0b0a58bdf9ec2305b52da0c2b1fd8fef65bafe1b3f24ed1b3ab46a7b",
   "stored": 8588
  },
  "l8.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.4686142206192017,
   "kappa": "sha256:6526238a271557f08cc2bf6650526dec3e7fb5d299a9b04d650136208d520a61",
   "stored": 1208280
  },
  "l8.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 2.274625301361084,
   "kappa": "sha256:3076a90cd6b99295062744a7b10e39fe821e5dc42e89175b5d36fb78d0410a94",
   "stored": 308315
  },
  "l8.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 3.369652271270752,
   "kappa": "sha256:5bb0e7bf0d66130c22594c304c63c39d74ef3f0fe17f266b99eaae33ed275763",
   "stored": 326110
  },
  "l8.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:94d6650642a953cc71c95b35ab73d2624b00e9c71a11532cdf430cdb03d2ce89",
   "stored": 8809
  },
  "l8.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 2.2168290615081787,
   "kappa": "sha256:fd6c6ec36a7c2133cb1ff973efe9ab077b686e0294ba78c1052af1ea21e99a77",
   "stored": 1281107
  },
  "l8.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:634283223ba66c5961db00479366189c4bcde592448be8e4323648e23bc477d6",
   "stored": 8541
  },
  "l8.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:0243ef3ad9aad2f638c518c5e07ca3ac324a79cc80bc7ef27c889ac49e41d5f7",
   "stored": 24305
  },
  "l8.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 1.9750255346298218,
   "kappa": "sha256:a6c49a94b00ddb2987642e7bf0a9527c7a9aab4a038853fac2c73c3e487685fa",
   "stored": 3439969
  },
  "l8.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 1.8963160514831543,
   "kappa": "sha256:1cd8040709067e02598e5d67cedec21bd82b6a661f5a2026f8d7211ce5679225",
   "stored": 3445779
  },
  "l8.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.3891186714172363,
   "kappa": "sha256:f4b9d3a4092fe968222519f3c5706166afc9169460c44ef52e52262774711dcb",
   "stored": 3497860
  },
  "l9.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:90c44ce94849a4c3a1c7c93354ffacdf156651a5b1ef25f73effde32c39263eb",
   "stored": 8699
  },
  "l9.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.4829719066619873,
   "kappa": "sha256:465a37b73c3c262366cf432e1ce8ca08d9a6a4cf4880e830902053d460018c46",
   "stored": 1265908
  },
  "l9.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 2.3452494144439697,
   "kappa": "sha256:0886f776e68311b280df1cb667e369050a76ee5b3773651d2ca038ce485d8eb8",
   "stored": 319513
  },
  "l9.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 3.61929988861084,
   "kappa": "sha256:dea24f11b9d6b4e1e520f93e23651a8968801c22ecede45db21b097da495390b",
   "stored": 324106
  },
  "l9.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:11732da6626328631c33e0ffb6ca1cdd50856805609644353eeac53965c02cb1",
   "stored": 8704
  },
  "l9.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 2.453671455383301,
   "kappa": "sha256:589de8df982857b58896c60c7128a2ac4dbb14ed4f35d3d811b1b66e296b881c",
   "stored": 1297657
  },
  "l9.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:e25a9252436f05df8eb2304fdbf407233b0872ee5e33516103bf9b27f98da023",
   "stored": 8546
  },
  "l9.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:4c637cf120d150027ae9ae759783e7c89f179fcf58c22ca44c971a98fc1434a2",
   "stored": 24270
  },
  "l9.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 1.9943604469299316,
   "kappa": "sha256:ca9f9acd4753f15e611f65ca2de793d698462933859120ff621118f50162a021",
   "stored": 3469879
  },
  "l9.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 1.9441899061203003,
   "kappa": "sha256:2ead90fe4d39cf276ac9e8fa5436c6639400ebc4b1d31db8202ccfa6d53ac7c5",
   "stored": 3479023
  },
  "l9.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.404606580734253,
   "kappa": "sha256:58a9f563ea631f405ce7114cdf6be883869956eb46e3087fccfebf2256d9ef33",
   "stored": 3499823
  },
  "l10.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:ae2fe1bd96c37a2ee0edec1d508f48bca20a6f1ddbe65d8e7318d76f36b55f12",
   "stored": 8640
  },
  "l10.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.6026175022125244,
   "kappa": "sha256:ca364c90a7e08dbdd4b8868b95b4b61f593b9c047500ad1984f492720cde6c58",
   "stored": 1232583
  },
  "l10.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 2.5188024044036865,
   "kappa": "sha256:3b3453607402be049591ac77dc5a59069febc2b28ad7cba10c2487347cd313e7",
   "stored": 307549
  },
  "l10.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 3.8300766944885254,
   "kappa": "sha256:32e38f020aaf7749b6c2ffddbba1b9f3cd7b5b0721e1fec517fb327681866e9d",
   "stored": 326559
  },
  "l10.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:782e9633e2cdf404a973216e4cc57ff6e7629f7fd8ea1aeaaca06c7b88874360",
   "stored": 8649
  },
  "l10.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 2.558532953262329,
   "kappa": "sha256:ee1705a7f4838f5c7ad89dab7f0481d5be875ed58b897997437c7756f831a97a",
   "stored": 1299074
  },
  "l10.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:2147a35c821eb8a2eddd2b532d9214626deffe87cdb34597833d010d70d8553d",
   "stored": 8567
  },
  "l10.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:199d95cf8a43fa1f91966079b7cf07bf54e8129c6de265fd9f02af8d3bed0890",
   "stored": 24142
  },
  "l10.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.0193634033203125,
   "kappa": "sha256:96df3ffba35d1446e0c4f6905a8aa4ae284419b5ea2d0c49c8166fa101c6bf52",
   "stored": 3489313
  },
  "l10.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 1.9991511106491089,
   "kappa": "sha256:75416c379f04b83a7bdc3340925a75eb2bed503f94cc166c8e050e5ab7d74daf",
   "stored": 3498534
  },
  "l10.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.4574780464172363,
   "kappa": "sha256:57319912983f17e970ac534c57476a8a2ea9be3fa0ce7b5252d18719341fc5ab",
   "stored": 3503315
  },
  "l11.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:3dec9f0306414d8c8b22ff3fe806902617fad04c72efd6b41bed36feed595162",
   "stored": 8701
  },
  "l11.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.6803207397460938,
   "kappa": "sha256:be02a5ce53613a0f6831cc91a822083c422084d53c2b22b5f1b76b20cc75290b",
   "stored": 1233851
  },
  "l11.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 2.5544822216033936,
   "kappa": "sha256:492699f2da103515776826f9aa3826cbaf051210a3d4dd10e886831684281ef1",
   "stored": 311854
  },
  "l11.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 3.988284111022949,
   "kappa": "sha256:2c667556b397b39494fe165ae3384321b2ae5f88dd704363d56a19b367b1cf1f",
   "stored": 327202
  },
  "l11.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:437ee2d20a1f11d0cd19894382ccaa867b01e7fdd1ce80bfa39a926302a42189",
   "stored": 8811
  },
  "l11.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 2.748793125152588,
   "kappa": "sha256:6fe390ece67c120432f6f90977b518fe4a92e9ae341f55f8c95db53d3fafca31",
   "stored": 1301013
  },
  "l11.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:b13e9967d2ae1001079f4431f024b9041b1a06f1fafdbe6fd3f0ad13d88e8def",
   "stored": 8565
  },
  "l11.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:90d68e018bbfae238e6d981c50a2280bfed32903308fafbe9ec497ef5f43a096",
   "stored": 24307
  },
  "l11.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.069572925567627,
   "kappa": "sha256:e0e7f510fd742f0f13e6b1489ff85cd498ebb204f9faab0e78cdaf317ec8cbbb",
   "stored": 3479801
  },
  "l11.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.070842981338501,
   "kappa": "sha256:ea60f468a940a6c667f48c06a6b6c40778661c4a84eaae5ccf528fc12e5be1ab",
   "stored": 3490687
  },
  "l11.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.479630470275879,
   "kappa": "sha256:8ec9abfca02742b7029fa5e888faf50e35e1af180081dde6a703d3acb7514c41",
   "stored": 3502622
  },
  "l12.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:2a7106179f592b47270d98c9e759a57e282464b703c410cc403597f30ad84d39",
   "stored": 8665
  },
  "l12.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.8417397737503052,
   "kappa": "sha256:03a0c74878571d9e5e7718fe2e76b3f413dd86f3086202995a9ad23c52bbf400",
   "stored": 1237309
  },
  "l12.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 2.7830650806427,
   "kappa": "sha256:52d977dfa96f0c34ddb69c5b13e5348173ac0fa6d6c881442e746b360e4465c1",
   "stored": 312713
  },
  "l12.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 3.908966064453125,
   "kappa": "sha256:da301a0d6c1dad3dfc4d53013552034b4fd34ef8adbc6766d6dd3d3d8ca08b33",
   "stored": 326894
  },
  "l12.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:131e7975f0299431c81922ff167cc0f20eb35c0f1a4a4621220b7e0da01d6653",
   "stored": 8710
  },
  "l12.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 2.5834097862243652,
   "kappa": "sha256:ead1e4346c41f14ef58e7fba1dc630941a621c970feb993c98a075b8ec292f15",
   "stored": 1300895
  },
  "l12.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:fb2d84509acc043b7cd12935a8586fd338dcebea4490f2ec67c7ed1b7e454611",
   "stored": 8589
  },
  "l12.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:f29bf6f2557c0bee009a20a00a6830dcf0f71280099375d92ea063d56d1b6cc8",
   "stored": 24382
  },
  "l12.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.0987560749053955,
   "kappa": "sha256:c3f2de72772ca201f09a8573e152327fc4ab0223adabcb1d860f88fad0b2a688",
   "stored": 3478890
  },
  "l12.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.100825071334839,
   "kappa": "sha256:46c272c595e16ceae20edbec67a82c66188ae57a5e2558953d6b5fbbb312a950",
   "stored": 3491319
  },
  "l12.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.4586021900177,
   "kappa": "sha256:e97db1a670b1870a26f5841c7c861d2a4abcc88487c63b387c392ee71172795a",
   "stored": 3499883
  },
  "l13.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:0f0954b8e14cea1aae0ed8c78abbd9626877aecb8d27c171a143875036800510",
   "stored": 8683
  },
  "l13.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.6434295177459717,
   "kappa": "sha256:cc039ea7d8572341d2f5a120cf8b90983ec8308fae8574450b77209d8afd0f0d",
   "stored": 1269368
  },
  "l13.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 2.5182487964630127,
   "kappa": "sha256:e69dcfa83eebd60ebad06ccc038c101764ec139e7ff7c5600399b34d6c53ba44",
   "stored": 320625
  },
  "l13.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 4.165911674499512,
   "kappa": "sha256:ba5ee7eceef5070a066bdbe6e2b1ccead17173b4a02e7e397c1bb39bb2c26a16",
   "stored": 327382
  },
  "l13.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:149e4cbe031d0b04663857df0e82ba5190546b37045aa0d01aca2c4439d9d3c4",
   "stored": 8721
  },
  "l13.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 2.7986159324645996,
   "kappa": "sha256:d3d92965a8b15b2b693f0f60fdf3d1a8bbae03e495e44794d4efa09ec936e196",
   "stored": 1302530
  },
  "l13.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:5a083e22ccf316f43eca29117437dfb92384a0957f06973d8562039a48ce726d",
   "stored": 8592
  },
  "l13.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:730c5039d1aada00d4f55550c9c757ddf0a659489d3f4bd47294d28a973d6f08",
   "stored": 24460
  },
  "l13.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.115769863128662,
   "kappa": "sha256:1332dd6ce88f2933f62412f59c5cd17b467fd4bd05836029c5f1712f6e7bafed",
   "stored": 3475704
  },
  "l13.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.0921471118927,
   "kappa": "sha256:5a2b97090a6f9b29f68fdc68f01aeed181eccc160a99d61e1b3c91a2cb644cbb",
   "stored": 3488724
  },
  "l13.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.4288315773010254,
   "kappa": "sha256:d35ea0e8d7c9922b70d1302712524fd861ccca7f921a1871fc39e80a7ad54460",
   "stored": 3498956
  },
  "l14.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:a264bb5ecbac5937cc12b21f267a4a70a6e979dcf2ab7649afd444842ac52624",
   "stored": 8691
  },
  "l14.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.903365969657898,
   "kappa": "sha256:3bad331279a288104d892bba3a0e8ed5175082850b5827728a5b04cdf06ec43e",
   "stored": 1207552
  },
  "l14.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 3.034632921218872,
   "kappa": "sha256:e98c45cec51826359b25113f65db90c28839d4d6bc6f729cc983d9c48d817745",
   "stored": 303902
  },
  "l14.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 4.2067975997924805,
   "kappa": "sha256:d95bc5b2dfb66888b944eefc51a6ac7f852b7a0da5a60e1db41bb50920992503",
   "stored": 325952
  },
  "l14.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:fdc4396b4db79821762c8c0b919d714ca31bdbb933c39bb22fff20f70fb25c59",
   "stored": 8938
  },
  "l14.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 2.605940580368042,
   "kappa": "sha256:f4164ec457b9a9ad9e9f7c0096b6e46f6152f26b7d79a468642943c763c5d5e1",
   "stored": 1291367
  },
  "l14.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:1179bccfbe821def2c4ffd61cc98f0821035f45663d6ed2240187a47e4497eee",
   "stored": 8563
  },
  "l14.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:1b2dca0ba6d611ee443713dfebb348ae97625e128b9a264fea9c457a72594f0d",
   "stored": 24402
  },
  "l14.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.1594929695129395,
   "kappa": "sha256:5df37995fa6cf362b8208a9eb37c3f303c329ea520c313bf5b41f6e6cea418d4",
   "stored": 3482372
  },
  "l14.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.13179874420166,
   "kappa": "sha256:00904d6069702683877dca67c3ee8ac79a78ab203e25fd0d85987c4fafd56b39",
   "stored": 3494211
  },
  "l14.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.469046115875244,
   "kappa": "sha256:2aece1343e2c629f428331ee0b4c7b324c04c03194a406e8cae92933970d211f",
   "stored": 3503208
  },
  "l15.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:ded8e8312e67940c5af55a908065f298ea25fb01c847fc25479fdf710647bcb3",
   "stored": 8656
  },
  "l15.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.677456021308899,
   "kappa": "sha256:4b3dd548c05d8897350438e4364166f87c85e77677c578aecdb0f7b69df706ef",
   "stored": 1259872
  },
  "l15.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 2.7184839248657227,
   "kappa": "sha256:83a49f7a524d6d21068c0b807670b144be3ada2b996c706fe246a56652e5484f",
   "stored": 318226
  },
  "l15.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 4.493652820587158,
   "kappa": "sha256:5092b09a9dc207559930ae71596a806768cdbe419dbd5ea4badc61b320bc9ca1",
   "stored": 326215
  },
  "l15.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:aacba1bef1ca54367682cb7f27adf599942f83054eeddb59ee6865c6696afa4b",
   "stored": 9046
  },
  "l15.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 2.8107123374938965,
   "kappa": "sha256:bc2e1003d4239e3d6add2120a7ccbe62c9d012b7313f8fd90e2cdb251208975f",
   "stored": 1302692
  },
  "l15.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:750e282ef0bb0fc88e9c931b4855b19a41e01599458ff56c52e0fc0f7a3e310b",
   "stored": 8554
  },
  "l15.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:e8050e8d36560a87bed0ab7891b7b4154ef14729ccaca890186dcb837311f67e",
   "stored": 24399
  },
  "l15.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.173977851867676,
   "kappa": "sha256:24df807c88617ed524734893904b1d4a762fe1702ea0d196522ad3777dc6fb2b",
   "stored": 3478897
  },
  "l15.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.1298775672912598,
   "kappa": "sha256:92758fe435447026ae72fb53fbfb5b38add1c050c864d4952c8a31c316bd2716",
   "stored": 3489936
  },
  "l15.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.612762212753296,
   "kappa": "sha256:276eb4c9cc8ae430b093c26f1945e382ba85fc49a625ceea2ff9d3f911d7da24",
   "stored": 3507716
  },
  "l16.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:0f54af4af52942cfffb7946e06ff3c77498e564870d114169bcb645b61d1087a",
   "stored": 8578
  },
  "l16.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 2.0254507064819336,
   "kappa": "sha256:493452dcda405b4028168c794a368de18a4d2f122f8c7b06589cc0cd3ab1a8d2",
   "stored": 1239996
  },
  "l16.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 3.05548095703125,
   "kappa": "sha256:5ec8af543da79c2c623cf9dcd478090eea1669cb55e4c61a038dc450531bdeda",
   "stored": 313776
  },
  "l16.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 4.2704854011535645,
   "kappa": "sha256:f000f04d008105c24a870287716529d1ace57ed81ded44ee72f2a5b02ebce287",
   "stored": 326976
  },
  "l16.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:b6b99c74111928af88bed65ac9e9bbb27dbd4ee00cf7853d4728a1dbf1e64c79",
   "stored": 8880
  },
  "l16.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 2.815990686416626,
   "kappa": "sha256:6eef62d214b30f85321f84fa999770d5442452df0d31b9272d6b5214f43cb347",
   "stored": 1300219
  },
  "l16.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:1496c29a71c6fc14ae7650d39fbbd92df6033e0d5fed5e21cd94427b84a8d5cf",
   "stored": 8550
  },
  "l16.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:b23605a4c9420d202b283be11da4826f3ce096dd82eb87f521a0c87f2aa0ce7b",
   "stored": 24216
  },
  "l16.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.2153754234313965,
   "kappa": "sha256:30e331c1341cae2ace9c85e8a16586377409ae5e937fb6a756657c24c87ee262",
   "stored": 3466446
  },
  "l16.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.1473798751831055,
   "kappa": "sha256:344ec4167a8a7632e43f6764b38a68ca4215274b0eb02eb0b78464112855251f",
   "stored": 3477457
  },
  "l16.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.6256518363952637,
   "kappa": "sha256:990bb18dad432453257166b972d6cd47325995a3cb5174fce5ebb60c3a752d00",
   "stored": 3506553
  },
  "l17.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:0f20e28a629e981b79c3ceea71f8e30fc62596278dbe7c512a1f984bb5393485",
   "stored": 8622
  },
  "l17.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.9280686378479004,
   "kappa": "sha256:d7eb7e575ba7934ff322b9dc1b1bd0bfebf63bdc6337915c590f3fb7dc1ca13f",
   "stored": 1230092
  },
  "l17.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 3.032918930053711,
   "kappa": "sha256:ac8c0d682c8e61fc9fa3c461cbcb8797b4217fa4c294ea8666a46b0ee60e4a17",
   "stored": 309214
  },
  "l17.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 4.596096992492676,
   "kappa": "sha256:b649cec5999653626f25d09644c776ae0eb6966a7895bde9024576435c65ac31",
   "stored": 325382
  },
  "l17.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:f16418d40c73b5965dee50d65f1a1f24d97798b6be6012d2631aa3efd3b79d40",
   "stored": 8659
  },
  "l17.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 3.0144503116607666,
   "kappa": "sha256:267b8a6ab03f977ff7406c73162e8cefedf980a0a7fcddc4465936d8d4b53bc6",
   "stored": 1303571
  },
  "l17.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:ac0fa9d71d8d1da40ee5315f682a226ec46913bb8bc6502ecea2f52593643a30",
   "stored": 8540
  },
  "l17.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:d64f23299fe112fbfb3b4428bed51c4bfbb6748257a7f82785239875232bf2a9",
   "stored": 23985
  },
  "l17.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.238203287124634,
   "kappa": "sha256:54a8d87cb1d383171d2daab130849f10c0ebbb96d2cd517f1a70cb65abc761c8",
   "stored": 3446572
  },
  "l17.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.1656453609466553,
   "kappa": "sha256:828afde4626e04ff930bcbb402b12430a36ed0f225f4b2c84cd9133d61119caf",
   "stored": 3457141
  },
  "l17.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.6830811500549316,
   "kappa": "sha256:9c309b0405f795dddee2b67f36bdfad2c501fb44120066bbd16167e9ab9b89e1",
   "stored": 3506881
  },
  "l18.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:517104936088206453a9a54254aa38515b91d0551152461db17f061d92d3bd66",
   "stored": 8561
  },
  "l18.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.670782446861267,
   "kappa": "sha256:6874cc47a2d95cf8bbc38c7fdc5df1aed8a490ebf4feca24367f6c9e2849efb8",
   "stored": 1243693
  },
  "l18.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 2.648331642150879,
   "kappa": "sha256:dcc1c0d886f920de1376b5329e3a6c0545b4a54816ca48fba4fda52c24358055",
   "stored": 316453
  },
  "l18.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 4.337561130523682,
   "kappa": "sha256:e6f496710e8c6a9cb30136cc1894a448729afdb6bd50b704d4fa690508208698",
   "stored": 324300
  },
  "l18.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:e9dcc8e69ec044bcfc58092c79e08f20b07b49daf458eea2f5a2a02488605188",
   "stored": 8724
  },
  "l18.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 2.673166275024414,
   "kappa": "sha256:50bef903a19d791dbd4307680dea9d799d64bcaba54b4f1fcfe61f2a918ebd49",
   "stored": 1296641
  },
  "l18.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:4eef2ce59d99ec4e013073a2f87631f23c97499b5ca85c52fecbdda63218caaf",
   "stored": 8547
  },
  "l18.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:8846089de1b69f24ca084003083a7a76dbb91c92a030852dda6b4e7c43ca6d46",
   "stored": 23894
  },
  "l18.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.2899796962738037,
   "kappa": "sha256:ca77d6cebb41e9a66d40bfd701a33030b4a8b31984d7a14395117e8463fdb7d1",
   "stored": 3427966
  },
  "l18.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.1707653999328613,
   "kappa": "sha256:2f01981176c0598566378aae2a886774ab4602635bc9f89eb0428eaee9c0cf74",
   "stored": 3436621
  },
  "l18.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.7724497318267822,
   "kappa": "sha256:5f92278e7599b2bd58e4bc5549430cc64830d9905111c51f5733bfa883f4b902",
   "stored": 3505816
  },
  "l19.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:00106c7da56920948f89ce65dedab7a0f75d03dabc92b894decad1b71ea4bfb6",
   "stored": 8599
  },
  "l19.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.6231597661972046,
   "kappa": "sha256:4ad34a97b8fb441312850fb75fc07a9185bce856548d29018b5dc2562cc50701",
   "stored": 1268311
  },
  "l19.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 2.524646759033203,
   "kappa": "sha256:d78eb60a5cb4a7f66de0e13413ba25a45b3827174381696b342ea70ff7055234",
   "stored": 320160
  },
  "l19.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 4.454233169555664,
   "kappa": "sha256:a0eba26716db9d2b9ff7fb08adbf950a2bf976c702f8571f2c7332e2219bc1bc",
   "stored": 326926
  },
  "l19.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:d58ac61548347cc9d5bcaaac09cbd3a1e96076ab7073259ea739cc3e774228ca",
   "stored": 8698
  },
  "l19.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 2.8433892726898193,
   "kappa": "sha256:52627c2dadee3500e2643ede01671cf90731ee65a5266a3a65f50cb38ccde42d",
   "stored": 1300928
  },
  "l19.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:22c364a011f0ec1fa7f3d385a7339a352377b62e54e15d507205becd332ecbc2",
   "stored": 8540
  },
  "l19.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:507711f451e3ad786e0a0462ff90677030311f505e5950bc49f47c2fb2fcd87d",
   "stored": 23975
  },
  "l19.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.294785976409912,
   "kappa": "sha256:e93e6c869aa7369fe53be80149fa43bab10c54f65c2ea710609b4496bc796659",
   "stored": 3404690
  },
  "l19.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.1719775199890137,
   "kappa": "sha256:4fa49d77ce81f400fbe9d1d3b1ab872567f0b79bc431703081b817ccf401a1b5",
   "stored": 3412065
  },
  "l19.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.7865946292877197,
   "kappa": "sha256:c687e509f6ca2d8ce697d1bed8b5e032c8738656d48452f65022e92d68065df2",
   "stored": 3503942
  },
  "l20.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:8e3c6389b51bc756cc73ec651e430827458ebaf5274bf58381e2ad76cf457ad4",
   "stored": 8566
  },
  "l20.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.8413779735565186,
   "kappa": "sha256:560c7b16712e5ef195650b5f5c7e429802985e752148efdf53389eed5f8daa4b",
   "stored": 1223845
  },
  "l20.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 2.9852893352508545,
   "kappa": "sha256:da1ccee487cf5de7bae896367abbe900c7705d5b2f64eddaf7ac429e2032a28b",
   "stored": 308303
  },
  "l20.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 3.960118055343628,
   "kappa": "sha256:9fa541ea8b7ee59458cfe4f83e57b461d10fbca7538acd8be7d7afda08349f22",
   "stored": 326330
  },
  "l20.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:127743287e8ddc2b17670148f7eee9f365b688100bc923e9233fec2743319f96",
   "stored": 8722
  },
  "l20.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 2.505194902420044,
   "kappa": "sha256:82cd613b37484037a0a00de779c4fd8088747eadb6039dd1135c4998d027091f",
   "stored": 1295646
  },
  "l20.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:d6333fda0fcda1d719568427e1bc5e37dfc4bd18a3b9ad200da8dee38973ebfe",
   "stored": 8518
  },
  "l20.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:b87d77358cedeedc339ada895199e1bc1a79ad48e9af42ca87f705ac3a4d7555",
   "stored": 23973
  },
  "l20.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.320962429046631,
   "kappa": "sha256:c133b62423ac9c45d1d6e7ecd42b0e73c18626d76f61ba8ec9d1fedaef41772a",
   "stored": 3404621
  },
  "l20.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.174668788909912,
   "kappa": "sha256:94c55ffa1077470de6ae0b21e8e1992156209f1db671481b0d66c1e545689740",
   "stored": 3411028
  },
  "l20.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.8613626956939697,
   "kappa": "sha256:5b311fbb61aee7a40116f53a56058145bd9f4f0a59090b53da0aad10ac255612",
   "stored": 3505464
  },
  "l21.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:30cf1c3d05222e58ec1a470fccb701d424597b0f8afa6d155567fd64a37eef47",
   "stored": 8568
  },
  "l21.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.6465575695037842,
   "kappa": "sha256:b62ec365a6c19d2fb50022a3d8cbd71c14a8209ca38d5b52fb8dab9160dcade5",
   "stored": 1220151
  },
  "l21.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 2.761808156967163,
   "kappa": "sha256:eedc19c537279f080f520599f93dea11b0c43676a965d859ec3c5cd8ad53ca2b",
   "stored": 310100
  },
  "l21.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 3.8352205753326416,
   "kappa": "sha256:8658ff32cf9f495f7386962bb18e04325d11b3e3cab8723932dd52ec4ce95b27",
   "stored": 325277
  },
  "l21.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:6b9ec2571e281175b9b630a589b60186d96de29cfcf4f67fedb968763c861266",
   "stored": 8763
  },
  "l21.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 2.6626720428466797,
   "kappa": "sha256:f41bb62b4d000d6318c9a4c2b00bb5f1a902d0931172259d0ca33845df7a0a34",
   "stored": 1272990
  },
  "l21.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:349c4704429228880575cb52d0194c563fd4c4ad2b7788b45b61cb5dddf40547",
   "stored": 8488
  },
  "l21.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:40058be0829a80e29574fb7b24a039d451e3fd5b5330ab4d3af349cd0184d9bb",
   "stored": 24041
  },
  "l21.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.3371989727020264,
   "kappa": "sha256:cbf76a0cb2c0aaa666a91b575ca5678dff71b1f3ed74c23ae70837c3f5dd0dd9",
   "stored": 3433914
  },
  "l21.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.169843912124634,
   "kappa": "sha256:903bec8d8a182aa6f4c1c61874c37bdb8c1c699b8bf780df500249b94d9425b0",
   "stored": 3439530
  },
  "l21.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.823518753051758,
   "kappa": "sha256:5c2c91be76f3881703b83bf3ac35b899b95625e87a144f0d96be8458f911aa9b",
   "stored": 3510331
  },
  "l22.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:a73a8987c7cc27c59fcab1cdbff5b5d7035888700f9442271b67f54297b21ffc",
   "stored": 8531
  },
  "l22.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.528588056564331,
   "kappa": "sha256:abc0b300f88df6c475cdbf7e06857993beb950183d813944857f7d25658a123c",
   "stored": 1247452
  },
  "l22.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 2.4565179347991943,
   "kappa": "sha256:b4c681d094b1ae7992ea53ce55d217dd910793472b4adab70f7a65ae92af62cc",
   "stored": 318639
  },
  "l22.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 3.9668362140655518,
   "kappa": "sha256:224cc629b5a06de721dea89be0d88147278ff4d05972461182a8ac32661efba1",
   "stored": 326609
  },
  "l22.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:0c2c00df431d32b192708249c0cb55571b964c8dc60db363564b57b1659a0414",
   "stored": 8742
  },
  "l22.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 2.521069288253784,
   "kappa": "sha256:7174e54c88b883b37655b2aac63ac47b6ced4b6d6e04d786d309bbd39a2b205e",
   "stored": 1269258
  },
  "l22.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:e4f17f179b4237c7e91be89b507db87a425c0930107f97c0c1bb35272efdf130",
   "stored": 8461
  },
  "l22.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:4f2e7bc4baf5a4a9a54b8ddc1314edd819ba92805db16fc1f3bd83a6b1b1888b",
   "stored": 24071
  },
  "l22.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.3953351974487305,
   "kappa": "sha256:3bc149faa6c7602e01eea13cf5a3837173f3e08a1c382b7535897bd05503def7",
   "stored": 3469248
  },
  "l22.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.2749760150909424,
   "kappa": "sha256:b5ea7f1465b20b3e594fe020570bcd695fc21f19e080a90918bf2794f497e79d",
   "stored": 3476782
  },
  "l22.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.8763983249664307,
   "kappa": "sha256:6f9c135a7267176b61195221873e378b72368b7ab338ff47b513d81192558c27",
   "stored": 3515761
  },
  "l23.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:743c18ab6e0518ac8a5962e953752919a650a9b768a663147965b367e36ff716",
   "stored": 8562
  },
  "l23.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.692755103111267,
   "kappa": "sha256:3c8bb597863df34a3f82a4d1273ac2f62ee5bd8d3817123d08b63e4408c1ec0b",
   "stored": 1230694
  },
  "l23.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 2.8201286792755127,
   "kappa": "sha256:2c840dac40e5696a4df066766818d43323a46fa75e196d7c03880e5458a80deb",
   "stored": 308041
  },
  "l23.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 3.7103710174560547,
   "kappa": "sha256:3f686ed0ed0079fa099a20857d1d0a33c5459c6ec583aecb1a1d6fe1b97f4bb0",
   "stored": 324071
  },
  "l23.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:a90180ffb2d5aaa84f638c7f59e35d7550499d9debee7b8d7992a23eedea63f7",
   "stored": 8855
  },
  "l23.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 2.560340642929077,
   "kappa": "sha256:133ecaeb6f2a062ed4d32f7f9f9f5a38e29d681eb3c2c69e60ab860360b6f450",
   "stored": 1291366
  },
  "l23.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:509d17c71fa544fdf00ef93a2450b65b6a5a21dc522f30c0c4d718276e408d8f",
   "stored": 8440
  },
  "l23.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:f399b63dd99640938ece6401fc9162ee7b91c336f75967736e8001c3b6a59ba2",
   "stored": 24240
  },
  "l23.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.3736634254455566,
   "kappa": "sha256:04dbd4bff550b1bbdaf7b6a18ed29d3233d5b021ea4fed69805eae38b561a6c5",
   "stored": 3470924
  },
  "l23.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.259279251098633,
   "kappa": "sha256:1e8a92e4782dadc96a03b41934d89b9cb6bab0f775d5e889e67c2545a87aa0dd",
   "stored": 3477261
  },
  "l23.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.924910306930542,
   "kappa": "sha256:c297405a11b5831def2a9cb0037167a943dcb77a4f4be898fb08567ab9364456",
   "stored": 3516781
  },
  "l24.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:f0b19a0d276bd6d4ab9b659f480832b67626631745c33c56ce7aa66f359e8280",
   "stored": 8550
  },
  "l24.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.7878329753875732,
   "kappa": "sha256:d8c4f24a6729fd30107d1934a526773f2c5e4ba79498691a57197d980575aa27",
   "stored": 1218794
  },
  "l24.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 2.8064944744110107,
   "kappa": "sha256:14f3f0e7645a921aaa205e22dbed689bd0aec261ddfae5f7cd69033c4eaab748",
   "stored": 301918
  },
  "l24.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 3.817347526550293,
   "kappa": "sha256:1e87e8a72f350cd76305ba27be501b117c819178f8a2838431ea61525673d8e9",
   "stored": 323701
  },
  "l24.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:8144e7426c961d7703a43d04e946fdd15b9d7375efa96b12d59a0973ff80489b",
   "stored": 8872
  },
  "l24.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 2.772810935974121,
   "kappa": "sha256:f88a1fea167b8b9f99283155b982ce848165b07f9728f3638f6610ca7dabbaab",
   "stored": 1293811
  },
  "l24.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:29cc94d8b735f36c71daba9acef09a8daf4e55ee2a6429b0f745b28265aa26d4",
   "stored": 8413
  },
  "l24.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:7f03aa58bb4e7de4c4903fc8ac23bd7c47c5066e89ac69beff63ec50bc40d636",
   "stored": 24376
  },
  "l24.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.3518364429473877,
   "kappa": "sha256:5a48d35cfa2ef5ec109d2bee316d3c3a6537ddd41095d54a77d517d03d0d9be4",
   "stored": 3456817
  },
  "l24.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.2403571605682373,
   "kappa": "sha256:d234d9230a9f9ccc47d72a02ec7e2bdedaff76f3d998e8fae1d617d8e0077d85",
   "stored": 3462861
  },
  "l24.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.9294803142547607,
   "kappa": "sha256:b25493c731a2adc297043ff96dbaa5cbb37427153d9c33df628f402cacb3e571",
   "stored": 3516448
  },
  "l25.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:eb4de016dff57919b1bdd351efc0a27a87fa43fc46c92cdad245b4e76be4e44f",
   "stored": 8491
  },
  "l25.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.4664311408996582,
   "kappa": "sha256:ac920288b87ca4545bd4d30d2fc90905a90d8100e6a195432f8b6ab748ce8d77",
   "stored": 1265731
  },
  "l25.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 2.3266441822052,
   "kappa": "sha256:e7c45ae596c603530e29ca9c667c61a659e96618a4fa075b5bd7f925d0c5c757",
   "stored": 317412
  },
  "l25.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 3.356239080429077,
   "kappa": "sha256:ae0b2fa0a2454ceb89791fdaf60d98c9a1e1e6a4ba0fce3a3bf347a10fa02407",
   "stored": 325160
  },
  "l25.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:dc4a6ce354f97a954032b0e1169bd35e79cd962c0667054c799b3fa9bf2d7b89",
   "stored": 8984
  },
  "l25.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 2.084083318710327,
   "kappa": "sha256:eecf3ecfd1edd7f285c5d0de58aa52fed4763948954bf809bb139643903806f7",
   "stored": 1289589
  },
  "l25.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:05c76568bb678ead4bfb73bad8fb9be5069754cc1bcf9484ab526853385bdaf0",
   "stored": 8365
  },
  "l25.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:d726a5a55834d9e57740cbd6b0a08480f64a26b6ad2c11de442fe5441b2cd60a",
   "stored": 24435
  },
  "l25.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.405451774597168,
   "kappa": "sha256:8fae6f99ff235ebcb1b7769ce0e53e794f422717a1b3a171a0c4a0fe259b5334",
   "stored": 3460861
  },
  "l25.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.3046693801879883,
   "kappa": "sha256:21ec2d137c5e0ca3ae0f56245afa922d01fb29bf1ebb4480c0a915697fcdc1f7",
   "stored": 3467553
  },
  "l25.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.9113521575927734,
   "kappa": "sha256:bae4ab9bd5c58075e2ef3591a107694bd3fc51634dcab11d81e6971e9ce9c2d6",
   "stored": 3517866
  },
  "l26.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:83f085266b33846306a0e16ee5b74ba8355084e089be19b38f8f4f647aa3c60f",
   "stored": 8555
  },
  "l26.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.603651762008667,
   "kappa": "sha256:3d9b9734c1e945ac56bff10d8169b3cd2a8657bae1690df7e902b74712e795d7",
   "stored": 1266631
  },
  "l26.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 2.433367967605591,
   "kappa": "sha256:cdfebf9ccead909ed0bc6be77278b1d9c44cd3e94fc88da44d511a579e93805a",
   "stored": 316815
  },
  "l26.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 3.24575138092041,
   "kappa": "sha256:475d1c65d0d365d9644350537d39656d7fc837b94744e81bd852d3ec25ed2a73",
   "stored": 323487
  },
  "l26.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:d461b9bfd5221e632ecbffe44c2516c8aee3c8e1141d1c1a8d3164baa9fa2497",
   "stored": 8993
  },
  "l26.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 2.0014703273773193,
   "kappa": "sha256:9fea89cea504d6bb6ca2cbfbe539dd0965249f74b9ec4219ca745c2797dc29a5",
   "stored": 1288982
  },
  "l26.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:19077f98b1e6689e83425e3427c00a1d9cc889ced414e26a747ad565c8e3eae2",
   "stored": 8345
  },
  "l26.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:d504a971e507434a54e4b9331e99bdaf4eca05c9bf4b80a87fb5e283a44a9846",
   "stored": 24498
  },
  "l26.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.4668102264404297,
   "kappa": "sha256:c68f0c47d8aea1ddb2bf3e1a8009a88995c18dea60712016bf52a6c048e75c26",
   "stored": 3449930
  },
  "l26.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.360677480697632,
   "kappa": "sha256:44d0ccc8f8c4b429000e5022e7a1fbcc234b93e08374f7b0a38046213eb7e8d7",
   "stored": 3456823
  },
  "l26.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.857257127761841,
   "kappa": "sha256:14bbaaba92a70aeb828098d7c0ddd6391d981fbb4dd028047894c843760cc807",
   "stored": 3516292
  },
  "l27.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:461b1ec9e36906c7388a64e41813d238b3034263b8b9bd9bee4328bb256885e7",
   "stored": 8534
  },
  "l27.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.5555118322372437,
   "kappa": "sha256:7b8af3cb1981ea7ce57c583918161792ed16dc88e667be246dfc74cc049e2ec7",
   "stored": 1264784
  },
  "l27.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 2.5493197441101074,
   "kappa": "sha256:de36e7a63c247ab1d70583f5fd74984bd1d18ad746dfc4ceac4d5c2f508cf6c3",
   "stored": 319347
  },
  "l27.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 3.6846508979797363,
   "kappa": "sha256:8455cd66cb928b1fb724b2fcb77e7f6c3cf3b6e30cb7d12547ccde7a006c5ab6",
   "stored": 324856
  },
  "l27.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:c74de5999a150775e068568ce30fe916dde459aa046a86bedf6b364a71b8fec7",
   "stored": 8997
  },
  "l27.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 2.227123260498047,
   "kappa": "sha256:117e30fe70bb51f022011c49f791f461361e9de5994249448febeb095f670503",
   "stored": 1288074
  },
  "l27.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:576de6f306582f82137be9fb7fd374cbc89163b8aff4b45921d11e18b9325981",
   "stored": 8354
  },
  "l27.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:2311c9e4e1659c2746388cc24e2f1ffc7a7a7c18c2ce1ecc2eb96eb4d354a43e",
   "stored": 24540
  },
  "l27.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.518040657043457,
   "kappa": "sha256:c5a2403fc17aa441387208d4c3b79e1fe59040a814819c0b47a500e11ed75787",
   "stored": 3419538
  },
  "l27.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.4069950580596924,
   "kappa": "sha256:819611ec8b734136e76af930c3d7fb803f316adf7bcef9e37c285c5e81546f8c",
   "stored": 3427094
  },
  "l27.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.7643442153930664,
   "kappa": "sha256:7826b731f5dfa4601b7bcb09edbccc79cf87737fd331eb9da3bb49f70dc95b94",
   "stored": 3512578
  },
  "l28.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:fbcf6afd35b08aead3a8172a63e15e288217e50b3d5416965d684cf3d6b3a010",
   "stored": 8533
  },
  "l28.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.3656436204910278,
   "kappa": "sha256:825958482b4253f83a8d73f827f09648f4ab23126188839ead3f05744ebd29fb",
   "stored": 1283066
  },
  "l28.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 2.0675907135009766,
   "kappa": "sha256:1e31cee59a4e61bebd41947416310feeee63a8da865cee0c2b7c87941e1cf5bc",
   "stored": 322415
  },
  "l28.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 3.021132230758667,
   "kappa": "sha256:45db25240f93cabb269a60a8a55aac0bfd9c9f3a53276b6217f7db32c5a6409c",
   "stored": 325709
  },
  "l28.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:75048c023a7512cae301bda2104eedb6e80da9a474bb994ac7bf8dff85abac7e",
   "stored": 8950
  },
  "l28.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.8344933986663818,
   "kappa": "sha256:f9237a80fc688a7a9d17c1469fb32cf6aff3c11a956407fe7a11833a2c180964",
   "stored": 1276637
  },
  "l28.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:4e47460f932fb3de086403ed87c58f8c396ca71592c3e8802e284b0e8c97525f",
   "stored": 8343
  },
  "l28.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:ceef5b646b09c7ad474258a36341b56916c14bdd69caaff402ea2d8f9a986cc0",
   "stored": 24531
  },
  "l28.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.6670985221862793,
   "kappa": "sha256:56e55084168673cd450d5c573d90935233c37a051e96a06ac5d8c61ab297ecc3",
   "stored": 3437148
  },
  "l28.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.518381357192993,
   "kappa": "sha256:74d50c2f0973a716e1cfa1441244bb980ed2a28465c0d7cd89cf1f17bff2137d",
   "stored": 3445378
  },
  "l28.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.6306076049804688,
   "kappa": "sha256:804f0c55d0115b755b29899622bb93bf2ac2c1083c693580e0fdb31c835b04ba",
   "stored": 3511351
  },
  "l29.attn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:62e08a7584c82b53b9e06bbb439a0bc813f6deb70da05a5d84db8f6acc9dee14",
   "stored": 8618
  },
  "l29.wq": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 1.1274330615997314,
   "kappa": "sha256:e3801c026bc3665c300144cd314052358b2d28f3de18810e527078cacf8ebb1b",
   "stored": 1287315
  },
  "l29.wk": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 1.6596884727478027,
   "kappa": "sha256:d7c44c457b5ce46e0e950a10bc101051392c3bd8c1cea75e06b9ea4609229a1c",
   "stored": 324404
  },
  "l29.wv": {
   "fmt": "t2",
   "N": 640,
   "K": 2560,
   "s": 1.7768195867538452,
   "kappa": "sha256:8469b051c78fb5b20a83afce79bb050fd2aac70d0d5c1c143f85b3357d9ba565",
   "stored": 286956
  },
  "l29.attn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:c3f392431d9aff6e1ed3dfcc9d9509258dda1434a19d8072bc3f8957336db6b1",
   "stored": 9477
  },
  "l29.wo": {
   "fmt": "t2",
   "N": 2560,
   "K": 2560,
   "s": 0.9527681469917297,
   "kappa": "sha256:ddd5d3da391b17f6ec654ae6b5df9edf5c380667a8f868b2e7245fcaafa65c98",
   "stored": 1206227
  },
  "l29.ffn_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 2560,
   "kappa": "sha256:057906d0c07df226e44c8548315f606ffacbc88000385b8f180987d5036ffc8c",
   "stored": 8313
  },
  "l29.ffn_sub_norm": {
   "fmt": "f32",
   "N": 1,
   "K": 6912,
   "kappa": "sha256:435fc9aef5e13407a429e5d3a99d37665f37d7ac6927b9ba0562a6b4937cdde7",
   "stored": 24528
  },
  "l29.w_gate": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.8299777507781982,
   "kappa": "sha256:3e29552ae3748c2ca6970c5a0084268c67b5754c58561f59763010654afeebc4",
   "stored": 3404362
  },
  "l29.w_up": {
   "fmt": "t2",
   "N": 6912,
   "K": 2560,
   "s": 2.602970838546753,
   "kappa": "sha256:c7e0902606f5dd55a400ad2da9370a230b75c81dd8af6b5bf53e49c7e2d506fc",
   "stored": 3412728
  },
  "l29.w_down": {
   "fmt": "t2",
   "N": 2560,
   "K": 6912,
   "s": 2.288531541824341,
   "kappa": "sha256:8a7695baefc5a701788c4e4d1f539507ee1289d86c1943fb7ac94c860c946883",
   "stored": 3500362
  }
 }
}