| { |
| "format": "holo-2bit/1", |
| "mode": "bitnet", |
| "model": "bitnet2b", |
| "source": "https://huggingface.co/microsoft/BitNet-b1.58-2B-4T-gguf/resolve/main/ggml-model-i2_s.gguf", |
| "bits": 3, |
| "layout": "q3f", |
| "sub_norm": true, |
| "ffn_act": "relu2", |
| "twoBit": false, |
| "incoherent": false, |
| "root": "sha256:19e16e6d51b4a68f60861b80d8f279c212449bf181fbb10c200bc9c56fa40de8", |
| "d": 2560, |
| "n_heads": 20, |
| "n_kv_heads": 5, |
| "ff": 6912, |
| "vocab": 128256, |
| "n_layers": 30, |
| "hd": 128, |
| "rope_base": 500000, |
| "attn_bias": false, |
| "qk_norm": false, |
| "qk_norm_dim": 0, |
| "tied": true, |
| "tensors": { |
| "embed": { |
| "fmt": "q3", |
| "N": 128256, |
| "K": 2560, |
| "kappa": "sha256:86685c8b39cc4197dc10f1b9a23d16eb3c931e3d5dd3a75c745b35a167bbd922", |
| "stored": 145127032 |
| }, |
| "final_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:f3ed6ddffe28303f7f2612995807174954542cf7c98168884beb4dd750365a76", |
| "stored": 8573 |
| }, |
| "lm_head": { |
| "fmt": "q3", |
| "N": 128256, |
| "K": 2560, |
| "kappa": "sha256:86685c8b39cc4197dc10f1b9a23d16eb3c931e3d5dd3a75c745b35a167bbd922", |
| "stored": 145127032 |
| }, |
| "l0.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:d4284d126d8514ebc1225bb8961597206122b8365d83d3d11aa90ec6cfef8de8", |
| "stored": 8653 |
| }, |
| "l0.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.2188547849655151, |
| "kappa": "sha256:eedcfec0fab223567654c9e8cba0e1b4e5e74577cb92f83f1ff093d57672ec54", |
| "stored": 1170936 |
| }, |
| "l0.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 1.7940564155578613, |
| "kappa": "sha256:c64d7815926f964b99f7b8b8e5015194435eafcbb24212e8b5f7ed6e9331e67c", |
| "stored": 298793 |
| }, |
| "l0.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 2.2931101322174072, |
| "kappa": "sha256:520cc1864a0b2b176bfdb72e91a8dd7e7b4a52aa2de2172d0bb139243b7710e5", |
| "stored": 305533 |
| }, |
| "l0.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:6bccb42bec3f0d4df231babb3fba94a7c4b9cb8452ea7782715ac1bc2ffc7815", |
| "stored": 9416 |
| }, |
| "l0.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 0.965597927570343, |
| "kappa": "sha256:4a4d9737cfa843aa90a8b764260197db9ff7a8e6c7d5d94a217085a1ec0e1a7a", |
| "stored": 1187473 |
| }, |
| "l0.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:d6679d1308683791618105f311b33ad5f8712f9957fc31226b4a3632cbab4ff4", |
| "stored": 8362 |
| }, |
| "l0.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:e456d56b800c8f17ce57985fc6baa581a60ff16393fb1e1b29c74ec065a9a5a1", |
| "stored": 24416 |
| }, |
| "l0.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 1.5512433052062988, |
| "kappa": "sha256:cc7d52c53fb68ba9201935a4167b5b6c0f8dff8bebf291d5e0c0d595f5e8bce0", |
| "stored": 3373881 |
| }, |
| "l0.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 1.8309355974197388, |
| "kappa": "sha256:571acf66c108028bde670583ff93853ef98754d4b5e353f137f5a9b862854168", |
| "stored": 3381723 |
| }, |
| "l0.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.163161277770996, |
| "kappa": "sha256:1dfbf91a0d9d4b86dc5b7e694268cd2c69651a4cb55ccb0736b07a824fa65a26", |
| "stored": 3521138 |
| }, |
| "l1.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:692ffeaa496938f7d8fbb246f4727e1b9a8ba7eb1dfe0eb3d2822ac1395e20ec", |
| "stored": 8610 |
| }, |
| "l1.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 0.8356457352638245, |
| "kappa": "sha256:44caf920488c5af1ed0a5c79394e6090d0e35ad83cfbebdfc1112178d0f75cbb", |
| "stored": 1256231 |
| }, |
| "l1.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 1.3455936908721924, |
| "kappa": "sha256:4075ea3fc9ecec622a7ec4af242d843e66937985376ca12e0afc60f62dd70c7e", |
| "stored": 320250 |
| }, |
| "l1.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 2.25688099861145, |
| "kappa": "sha256:52e8bab774c8aaa60ef3dbd68838184ac53ee839e6dc198795ab1f60afb2e36d", |
| "stored": 311276 |
| }, |
| "l1.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:d6680a2abb414b60968d2d69681934998fdd6bfda67ec11e94461bc3accdf9bb", |
| "stored": 9243 |
| }, |
| "l1.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.6071064472198486, |
| "kappa": "sha256:aa5e364a81bc37d3d376669ecce76b1dd551a252011c7fde896b973478409a72", |
| "stored": 1267180 |
| }, |
| "l1.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:3e6862c455f450cfd04660812af02f505f64703abc933414f36270950cf238ef", |
| "stored": 8438 |
| }, |
| "l1.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:57284f98e494193ce830bf6a92cbcb89be1b4b508f95e4fb93b2686acb4f2856", |
| "stored": 24676 |
| }, |
| "l1.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 0.8077176213264465, |
| "kappa": "sha256:ffc322b414176fc97e37259f471a9c7efc3951526166d7d29251f5da0a7cbdf9", |
| "stored": 2006458 |
| }, |
| "l1.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 0.7442138195037842, |
| "kappa": "sha256:768d6296e7d5335cdcc324c4e21281fc52111bae79ee17ce00293db846f3de3d", |
| "stored": 2005495 |
| }, |
| "l1.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 1.265028715133667, |
| "kappa": "sha256:aabce90f8d5caa03a7b8eae9b576e4bb73660d386dfd0b0c7dbedee35d3c0e04", |
| "stored": 3132364 |
| }, |
| "l2.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:7794cb0da3c3aaab04d324091c45ede2819e6a9d2b2ee6890cb58ad6af608db0", |
| "stored": 8575 |
| }, |
| "l2.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 0.9968166947364807, |
| "kappa": "sha256:b819cf2aa22b25d06470640f50d25d5715dcb64e0f9b1473f78490cd0e243387", |
| "stored": 1280269 |
| }, |
| "l2.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 1.4827167987823486, |
| "kappa": "sha256:8b6b2a7670e2a04468ab68898467fc09bd34855d04558c756f7c7fcb8f1bd2fe", |
| "stored": 324212 |
| }, |
| "l2.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 2.706890344619751, |
| "kappa": "sha256:b70762c07fb0ab8912a43e60ba6a23e3a6e30821f8dc1ee89c3f7f59027477ca", |
| "stored": 322905 |
| }, |
| "l2.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:9e618587af45c53d49d0e9bf8529ca34aecf89bcaf9d4c1c86925c5393305b16", |
| "stored": 9098 |
| }, |
| "l2.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.9340922832489014, |
| "kappa": "sha256:563abfd7381f960a3b2eb3fab6272fadc5a18b3a75065c096cbc16c16b0fd58a", |
| "stored": 1273515 |
| }, |
| "l2.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:3e28b72d7dd7f19b005a4f93f684ae4bd3c48fa641c8ad13652a8f6f1e977e44", |
| "stored": 8482 |
| }, |
| "l2.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:3ee5263b59bdfff077ba4e3a3d38d1e38816abc52469df27cc33e0fccac3c570", |
| "stored": 24384 |
| }, |
| "l2.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 1.1282522678375244, |
| "kappa": "sha256:cbce5d17b05f295b2589d1f80267189bd3f118daa69dad2f81373f17f822fcc3", |
| "stored": 2586298 |
| }, |
| "l2.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 1.025633454322815, |
| "kappa": "sha256:886d6c83a74ddb05700d1bb12bfaa7e4928889b9096e9edc9f6b9f3e1bb8f590", |
| "stored": 2586385 |
| }, |
| "l2.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 1.7018623352050781, |
| "kappa": "sha256:a3f5480e02ea08533f6accb02305304dbd76b691fa40e6dcd49ee7a48906f52d", |
| "stored": 3312977 |
| }, |
| "l3.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:2f227ee74351331034d0d25d25a13727d9f7e8bc59f3a406f7ab52c52934a7e7", |
| "stored": 8578 |
| }, |
| "l3.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.0245068073272705, |
| "kappa": "sha256:a0745a319d3746c224f702efe23f24c1e20ce69ebbef4755f3a2e59a546b4e11", |
| "stored": 1270277 |
| }, |
| "l3.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 1.5373108386993408, |
| "kappa": "sha256:07569ce168db3d50cf391632f9ac2b3d4f44e6f3426a6ff7c78b5b8dd53f5a0b", |
| "stored": 321002 |
| }, |
| "l3.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 2.6927521228790283, |
| "kappa": "sha256:95bf010dd048b49028fabc4988da5d1e0e6e97be13b59d38f19dd2a955397c4c", |
| "stored": 322052 |
| }, |
| "l3.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:448aa10e6e669381f080f03a3a1dab6a9e46af610add6b9454ba6393318023c0", |
| "stored": 9100 |
| }, |
| "l3.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.9021772146224976, |
| "kappa": "sha256:3e43ee1b3dac78d34f2f48af4dfff7b98070bbcf79ba4dc14f6c06264e0f3840", |
| "stored": 1290528 |
| }, |
| "l3.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:4c91661088cdb8058b202eb4b8e0a4a2785fe77a7619977614bf3cf18a6e074d", |
| "stored": 8469 |
| }, |
| "l3.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:6f66043d29aea907fc9ebba9c99eaed244aa8fe9d2a8d7ea75d7d341d4e861b1", |
| "stored": 24504 |
| }, |
| "l3.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 1.4292961359024048, |
| "kappa": "sha256:730f4cc1b2fa77335f21d4938206cb2bdee421487c52a6c31de5d8391f355f9f", |
| "stored": 3027164 |
| }, |
| "l3.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 1.2692362070083618, |
| "kappa": "sha256:a6aeabc47065987ebc6b2181ce74cc592ef1d6135cafdbb1a6454a5612d2b4d7", |
| "stored": 3026512 |
| }, |
| "l3.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 1.9713046550750732, |
| "kappa": "sha256:36ffa2d25e4c56cceabaee0ce4c0d6fe1736704027c63aa0a50984b19405bfc8", |
| "stored": 3432754 |
| }, |
| "l4.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:aef70afa4e15c94014138edd713e11353f53c3d0104c6efcb06c2f18f329c838", |
| "stored": 8554 |
| }, |
| "l4.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.1376540660858154, |
| "kappa": "sha256:a7c2d195b7106780d6dc646a55adc8a37dfcb5a056b1cee43d200711e18fa6ba", |
| "stored": 1261583 |
| }, |
| "l4.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 1.8145620822906494, |
| "kappa": "sha256:8ae00c20de1bd5f6c13f478d1403877bdc7fdc43c0ad39dd549c7030a53679de", |
| "stored": 320169 |
| }, |
| "l4.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 3.0370285511016846, |
| "kappa": "sha256:ac1f28d22c420a602ba869fee175530d2f01642dfe428388eb94538d2c2222ba", |
| "stored": 326579 |
| }, |
| "l4.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:eb367f75b22575c830f0bcab0d24fd335f6930a9346ea235a61de4c724f07e73", |
| "stored": 9061 |
| }, |
| "l4.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 2.1330158710479736, |
| "kappa": "sha256:61b5f702f4e3bfd444d2dfba5151b18a6a330c7818409d54dfa66dde93e18d83", |
| "stored": 1293375 |
| }, |
| "l4.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:f479519120b05443a9e9f4faaf08bd9c4d4e5e3a98114fd0449d9e6d685cc727", |
| "stored": 8491 |
| }, |
| "l4.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:c31f2ed8859fc62267d313b9e04301a62c401653dabc4a88691a733d47d8887e", |
| "stored": 24558 |
| }, |
| "l4.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 1.6233707666397095, |
| "kappa": "sha256:e129f9c76ac0f244dc93b20eb4e524f52e745ba44ba64b0d7e458aca610315de", |
| "stored": 3221178 |
| }, |
| "l4.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 1.4957562685012817, |
| "kappa": "sha256:5199eef196960030362fe17bab617fbb1cff581aafbf25e0553498f13e9bcfb9", |
| "stored": 3222898 |
| }, |
| "l4.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.13490629196167, |
| "kappa": "sha256:ff81c04bcc1ab1351024224a516d7d1c9626de3d70b7ed33aefc11b18a1ae187", |
| "stored": 3472411 |
| }, |
| "l5.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:971a860e4b5bb450e32ecf49acc10b1ac7f5e99e5ff84d616315ddaea81fcca1", |
| "stored": 8579 |
| }, |
| "l5.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.2880576848983765, |
| "kappa": "sha256:705d9f3b71f29883191c8e7b551177ac69ced46308863c4cd0e5619198787996", |
| "stored": 1249387 |
| }, |
| "l5.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 1.9532663822174072, |
| "kappa": "sha256:6ed451a23d0b381885e5f276c7e2ffdac4c659cd1302b904e30440f130c9912c", |
| "stored": 316211 |
| }, |
| "l5.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 3.1583521366119385, |
| "kappa": "sha256:8155fb31d9b06cdf3864361ad3c68f77725534867dfed2a523ca51393524055e", |
| "stored": 323643 |
| }, |
| "l5.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:2abdf791b1319830d07e81386c6c5106188b7ecf99f8063db90bbdcdeee6b32e", |
| "stored": 8998 |
| }, |
| "l5.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 2.144779682159424, |
| "kappa": "sha256:72e908f9d71630e38c46d04be24ac4721527219c440a7bab373200aa9f3308f6", |
| "stored": 1288762 |
| }, |
| "l5.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:43dc31c0c267117271541de63cec7ce8bf592ac4a72ac6efa08b547ade082b08", |
| "stored": 8523 |
| }, |
| "l5.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:e8ed5c6c5ea6afc2995d6a65e5f1e538b0ba04d37743792ebfe52449c0c74ba8", |
| "stored": 24616 |
| }, |
| "l5.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 1.7476176023483276, |
| "kappa": "sha256:4f9035197f2933807c73b72c8111ead321842a2fd6157164125b56e28d2f9414", |
| "stored": 3367757 |
| }, |
| "l5.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 1.6368310451507568, |
| "kappa": "sha256:a59899440487f801a0e07470a782fbab9c74477a58e917b3bdc915952a2d9b8b", |
| "stored": 3369828 |
| }, |
| "l5.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.177177906036377, |
| "kappa": "sha256:873a0a0e7b7c772ef5c7785a74323eb7283bb715a170798631c2a7d66d7d3f7c", |
| "stored": 3488556 |
| }, |
| "l6.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:fd97f2f81d8fd792584b189bf7887855d09d5d8f6285ec58a3cab173c73b6cfc", |
| "stored": 8571 |
| }, |
| "l6.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.3680181503295898, |
| "kappa": "sha256:6e16bda0b604b7837899991199d8cc301442bda267e00b2b76e939f6f9e39de1", |
| "stored": 1245136 |
| }, |
| "l6.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 2.1691858768463135, |
| "kappa": "sha256:77e4d3562eee8f1bfd7accbb77f4afb7db530fb37baa08a5425381b3d7dea550", |
| "stored": 311065 |
| }, |
| "l6.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 3.480070114135742, |
| "kappa": "sha256:b5ba51de8180b296588c3627d3a0c38d2e4fce871718c22caf4889723b1ea071", |
| "stored": 325142 |
| }, |
| "l6.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:dd25aa4a8a78833ab9f5adaa4ab660fa747914dc5dedcbc4dca0a02313083c29", |
| "stored": 9006 |
| }, |
| "l6.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 2.3384294509887695, |
| "kappa": "sha256:f3b0526e24d1403a67ee1b35be1ecdc453f4ecb8696b7e2b6a6ab6643b77d46c", |
| "stored": 1295959 |
| }, |
| "l6.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:18009c00b2b0debcc1db961502d5e7dc5c630facaacd3c4483e8fc03e1b16f64", |
| "stored": 8538 |
| }, |
| "l6.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:2dbdb1b93dd74d82f9857eae5e997cf4ec2b30483f2baeaf0e8c4af49cc99631", |
| "stored": 24486 |
| }, |
| "l6.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 1.8770447969436646, |
| "kappa": "sha256:8c4a233e39c3b1c717446669c7cd4f452d3b337bc5b158ce5198b418a7ba70de", |
| "stored": 3421320 |
| }, |
| "l6.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 1.7816859483718872, |
| "kappa": "sha256:a3b7de9949de7bc843ddc237c837d395fd1b1d99a5e30eb21cbc2fb75ecd450b", |
| "stored": 3423850 |
| }, |
| "l6.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.3278253078460693, |
| "kappa": "sha256:8a109645c73199c149003d29145ba15e8b038140feefcddc208bf7d700bc9cf3", |
| "stored": 3495370 |
| }, |
| "l7.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:72aac3bcca02692efb9ad921d4349a891458c479fefbef1fafc44924fb8d0cbd", |
| "stored": 8600 |
| }, |
| "l7.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.4199094772338867, |
| "kappa": "sha256:8a0dc9f2c20a1c17efdb8e7c7c22174979ec497f4f1e0ae23c57a196d3256159", |
| "stored": 1246258 |
| }, |
| "l7.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 2.2624335289001465, |
| "kappa": "sha256:77bca20e1dc197fd480ee4ddcdbc7f5ba093ce9d1242a03608f5eceb79b4b653", |
| "stored": 312316 |
| }, |
| "l7.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 3.229888916015625, |
| "kappa": "sha256:1c10f530cae746ab5ea59d226388ea343146d82e3e8223c39e45f171eaf2a9fb", |
| "stored": 321444 |
| }, |
| "l7.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:d8e7963a5dbdef62a2f16c75e0df7236d4df9ab5ed88e3cb0e8a8baed768000e", |
| "stored": 8748 |
| }, |
| "l7.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 2.244603157043457, |
| "kappa": "sha256:780d1b78022e6bd680d647450927e92948a5c51e832935b0fcc41a4d18e9ba4f", |
| "stored": 1293210 |
| }, |
| "l7.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:65f742abac762cdad7942532137459bbd3810fa3471fa3bebc91682b930045ea", |
| "stored": 8522 |
| }, |
| "l7.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:15f48d29df002e1653c3da8b9eab8e38014a6ea7eb045b4b666ffcc59929772a", |
| "stored": 24415 |
| }, |
| "l7.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 1.921570897102356, |
| "kappa": "sha256:fb023a0b278fb708e11827e1ca0f9007c0e82cdf616bf3a59c56d170ae34d710", |
| "stored": 3406036 |
| }, |
| "l7.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 1.835010290145874, |
| "kappa": "sha256:e13b3ae8baff331918812dafce136fe62ddbc4fac1a5cac8419d673f80275537", |
| "stored": 3410662 |
| }, |
| "l7.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.358863592147827, |
| "kappa": "sha256:2886364019c1c76049929a048dab94c82f611c0985b92383d64f5ce492bb6fc0", |
| "stored": 3494492 |
| }, |
| "l8.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:bb4a8eff0b0a58bdf9ec2305b52da0c2b1fd8fef65bafe1b3f24ed1b3ab46a7b", |
| "stored": 8588 |
| }, |
| "l8.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.4686142206192017, |
| "kappa": "sha256:6526238a271557f08cc2bf6650526dec3e7fb5d299a9b04d650136208d520a61", |
| "stored": 1208280 |
| }, |
| "l8.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 2.274625301361084, |
| "kappa": "sha256:3076a90cd6b99295062744a7b10e39fe821e5dc42e89175b5d36fb78d0410a94", |
| "stored": 308315 |
| }, |
| "l8.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 3.369652271270752, |
| "kappa": "sha256:5bb0e7bf0d66130c22594c304c63c39d74ef3f0fe17f266b99eaae33ed275763", |
| "stored": 326110 |
| }, |
| "l8.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:94d6650642a953cc71c95b35ab73d2624b00e9c71a11532cdf430cdb03d2ce89", |
| "stored": 8809 |
| }, |
| "l8.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 2.2168290615081787, |
| "kappa": "sha256:fd6c6ec36a7c2133cb1ff973efe9ab077b686e0294ba78c1052af1ea21e99a77", |
| "stored": 1281107 |
| }, |
| "l8.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:634283223ba66c5961db00479366189c4bcde592448be8e4323648e23bc477d6", |
| "stored": 8541 |
| }, |
| "l8.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:0243ef3ad9aad2f638c518c5e07ca3ac324a79cc80bc7ef27c889ac49e41d5f7", |
| "stored": 24305 |
| }, |
| "l8.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 1.9750255346298218, |
| "kappa": "sha256:a6c49a94b00ddb2987642e7bf0a9527c7a9aab4a038853fac2c73c3e487685fa", |
| "stored": 3439969 |
| }, |
| "l8.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 1.8963160514831543, |
| "kappa": "sha256:1cd8040709067e02598e5d67cedec21bd82b6a661f5a2026f8d7211ce5679225", |
| "stored": 3445779 |
| }, |
| "l8.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.3891186714172363, |
| "kappa": "sha256:f4b9d3a4092fe968222519f3c5706166afc9169460c44ef52e52262774711dcb", |
| "stored": 3497860 |
| }, |
| "l9.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:90c44ce94849a4c3a1c7c93354ffacdf156651a5b1ef25f73effde32c39263eb", |
| "stored": 8699 |
| }, |
| "l9.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.4829719066619873, |
| "kappa": "sha256:465a37b73c3c262366cf432e1ce8ca08d9a6a4cf4880e830902053d460018c46", |
| "stored": 1265908 |
| }, |
| "l9.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 2.3452494144439697, |
| "kappa": "sha256:0886f776e68311b280df1cb667e369050a76ee5b3773651d2ca038ce485d8eb8", |
| "stored": 319513 |
| }, |
| "l9.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 3.61929988861084, |
| "kappa": "sha256:dea24f11b9d6b4e1e520f93e23651a8968801c22ecede45db21b097da495390b", |
| "stored": 324106 |
| }, |
| "l9.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:11732da6626328631c33e0ffb6ca1cdd50856805609644353eeac53965c02cb1", |
| "stored": 8704 |
| }, |
| "l9.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 2.453671455383301, |
| "kappa": "sha256:589de8df982857b58896c60c7128a2ac4dbb14ed4f35d3d811b1b66e296b881c", |
| "stored": 1297657 |
| }, |
| "l9.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:e25a9252436f05df8eb2304fdbf407233b0872ee5e33516103bf9b27f98da023", |
| "stored": 8546 |
| }, |
| "l9.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:4c637cf120d150027ae9ae759783e7c89f179fcf58c22ca44c971a98fc1434a2", |
| "stored": 24270 |
| }, |
| "l9.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 1.9943604469299316, |
| "kappa": "sha256:ca9f9acd4753f15e611f65ca2de793d698462933859120ff621118f50162a021", |
| "stored": 3469879 |
| }, |
| "l9.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 1.9441899061203003, |
| "kappa": "sha256:2ead90fe4d39cf276ac9e8fa5436c6639400ebc4b1d31db8202ccfa6d53ac7c5", |
| "stored": 3479023 |
| }, |
| "l9.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.404606580734253, |
| "kappa": "sha256:58a9f563ea631f405ce7114cdf6be883869956eb46e3087fccfebf2256d9ef33", |
| "stored": 3499823 |
| }, |
| "l10.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:ae2fe1bd96c37a2ee0edec1d508f48bca20a6f1ddbe65d8e7318d76f36b55f12", |
| "stored": 8640 |
| }, |
| "l10.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.6026175022125244, |
| "kappa": "sha256:ca364c90a7e08dbdd4b8868b95b4b61f593b9c047500ad1984f492720cde6c58", |
| "stored": 1232583 |
| }, |
| "l10.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 2.5188024044036865, |
| "kappa": "sha256:3b3453607402be049591ac77dc5a59069febc2b28ad7cba10c2487347cd313e7", |
| "stored": 307549 |
| }, |
| "l10.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 3.8300766944885254, |
| "kappa": "sha256:32e38f020aaf7749b6c2ffddbba1b9f3cd7b5b0721e1fec517fb327681866e9d", |
| "stored": 326559 |
| }, |
| "l10.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:782e9633e2cdf404a973216e4cc57ff6e7629f7fd8ea1aeaaca06c7b88874360", |
| "stored": 8649 |
| }, |
| "l10.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 2.558532953262329, |
| "kappa": "sha256:ee1705a7f4838f5c7ad89dab7f0481d5be875ed58b897997437c7756f831a97a", |
| "stored": 1299074 |
| }, |
| "l10.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:2147a35c821eb8a2eddd2b532d9214626deffe87cdb34597833d010d70d8553d", |
| "stored": 8567 |
| }, |
| "l10.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:199d95cf8a43fa1f91966079b7cf07bf54e8129c6de265fd9f02af8d3bed0890", |
| "stored": 24142 |
| }, |
| "l10.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.0193634033203125, |
| "kappa": "sha256:96df3ffba35d1446e0c4f6905a8aa4ae284419b5ea2d0c49c8166fa101c6bf52", |
| "stored": 3489313 |
| }, |
| "l10.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 1.9991511106491089, |
| "kappa": "sha256:75416c379f04b83a7bdc3340925a75eb2bed503f94cc166c8e050e5ab7d74daf", |
| "stored": 3498534 |
| }, |
| "l10.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.4574780464172363, |
| "kappa": "sha256:57319912983f17e970ac534c57476a8a2ea9be3fa0ce7b5252d18719341fc5ab", |
| "stored": 3503315 |
| }, |
| "l11.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:3dec9f0306414d8c8b22ff3fe806902617fad04c72efd6b41bed36feed595162", |
| "stored": 8701 |
| }, |
| "l11.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.6803207397460938, |
| "kappa": "sha256:be02a5ce53613a0f6831cc91a822083c422084d53c2b22b5f1b76b20cc75290b", |
| "stored": 1233851 |
| }, |
| "l11.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 2.5544822216033936, |
| "kappa": "sha256:492699f2da103515776826f9aa3826cbaf051210a3d4dd10e886831684281ef1", |
| "stored": 311854 |
| }, |
| "l11.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 3.988284111022949, |
| "kappa": "sha256:2c667556b397b39494fe165ae3384321b2ae5f88dd704363d56a19b367b1cf1f", |
| "stored": 327202 |
| }, |
| "l11.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:437ee2d20a1f11d0cd19894382ccaa867b01e7fdd1ce80bfa39a926302a42189", |
| "stored": 8811 |
| }, |
| "l11.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 2.748793125152588, |
| "kappa": "sha256:6fe390ece67c120432f6f90977b518fe4a92e9ae341f55f8c95db53d3fafca31", |
| "stored": 1301013 |
| }, |
| "l11.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:b13e9967d2ae1001079f4431f024b9041b1a06f1fafdbe6fd3f0ad13d88e8def", |
| "stored": 8565 |
| }, |
| "l11.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:90d68e018bbfae238e6d981c50a2280bfed32903308fafbe9ec497ef5f43a096", |
| "stored": 24307 |
| }, |
| "l11.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.069572925567627, |
| "kappa": "sha256:e0e7f510fd742f0f13e6b1489ff85cd498ebb204f9faab0e78cdaf317ec8cbbb", |
| "stored": 3479801 |
| }, |
| "l11.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.070842981338501, |
| "kappa": "sha256:ea60f468a940a6c667f48c06a6b6c40778661c4a84eaae5ccf528fc12e5be1ab", |
| "stored": 3490687 |
| }, |
| "l11.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.479630470275879, |
| "kappa": "sha256:8ec9abfca02742b7029fa5e888faf50e35e1af180081dde6a703d3acb7514c41", |
| "stored": 3502622 |
| }, |
| "l12.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:2a7106179f592b47270d98c9e759a57e282464b703c410cc403597f30ad84d39", |
| "stored": 8665 |
| }, |
| "l12.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.8417397737503052, |
| "kappa": "sha256:03a0c74878571d9e5e7718fe2e76b3f413dd86f3086202995a9ad23c52bbf400", |
| "stored": 1237309 |
| }, |
| "l12.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 2.7830650806427, |
| "kappa": "sha256:52d977dfa96f0c34ddb69c5b13e5348173ac0fa6d6c881442e746b360e4465c1", |
| "stored": 312713 |
| }, |
| "l12.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 3.908966064453125, |
| "kappa": "sha256:da301a0d6c1dad3dfc4d53013552034b4fd34ef8adbc6766d6dd3d3d8ca08b33", |
| "stored": 326894 |
| }, |
| "l12.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:131e7975f0299431c81922ff167cc0f20eb35c0f1a4a4621220b7e0da01d6653", |
| "stored": 8710 |
| }, |
| "l12.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 2.5834097862243652, |
| "kappa": "sha256:ead1e4346c41f14ef58e7fba1dc630941a621c970feb993c98a075b8ec292f15", |
| "stored": 1300895 |
| }, |
| "l12.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:fb2d84509acc043b7cd12935a8586fd338dcebea4490f2ec67c7ed1b7e454611", |
| "stored": 8589 |
| }, |
| "l12.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:f29bf6f2557c0bee009a20a00a6830dcf0f71280099375d92ea063d56d1b6cc8", |
| "stored": 24382 |
| }, |
| "l12.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.0987560749053955, |
| "kappa": "sha256:c3f2de72772ca201f09a8573e152327fc4ab0223adabcb1d860f88fad0b2a688", |
| "stored": 3478890 |
| }, |
| "l12.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.100825071334839, |
| "kappa": "sha256:46c272c595e16ceae20edbec67a82c66188ae57a5e2558953d6b5fbbb312a950", |
| "stored": 3491319 |
| }, |
| "l12.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.4586021900177, |
| "kappa": "sha256:e97db1a670b1870a26f5841c7c861d2a4abcc88487c63b387c392ee71172795a", |
| "stored": 3499883 |
| }, |
| "l13.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:0f0954b8e14cea1aae0ed8c78abbd9626877aecb8d27c171a143875036800510", |
| "stored": 8683 |
| }, |
| "l13.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.6434295177459717, |
| "kappa": "sha256:cc039ea7d8572341d2f5a120cf8b90983ec8308fae8574450b77209d8afd0f0d", |
| "stored": 1269368 |
| }, |
| "l13.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 2.5182487964630127, |
| "kappa": "sha256:e69dcfa83eebd60ebad06ccc038c101764ec139e7ff7c5600399b34d6c53ba44", |
| "stored": 320625 |
| }, |
| "l13.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 4.165911674499512, |
| "kappa": "sha256:ba5ee7eceef5070a066bdbe6e2b1ccead17173b4a02e7e397c1bb39bb2c26a16", |
| "stored": 327382 |
| }, |
| "l13.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:149e4cbe031d0b04663857df0e82ba5190546b37045aa0d01aca2c4439d9d3c4", |
| "stored": 8721 |
| }, |
| "l13.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 2.7986159324645996, |
| "kappa": "sha256:d3d92965a8b15b2b693f0f60fdf3d1a8bbae03e495e44794d4efa09ec936e196", |
| "stored": 1302530 |
| }, |
| "l13.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:5a083e22ccf316f43eca29117437dfb92384a0957f06973d8562039a48ce726d", |
| "stored": 8592 |
| }, |
| "l13.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:730c5039d1aada00d4f55550c9c757ddf0a659489d3f4bd47294d28a973d6f08", |
| "stored": 24460 |
| }, |
| "l13.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.115769863128662, |
| "kappa": "sha256:1332dd6ce88f2933f62412f59c5cd17b467fd4bd05836029c5f1712f6e7bafed", |
| "stored": 3475704 |
| }, |
| "l13.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.0921471118927, |
| "kappa": "sha256:5a2b97090a6f9b29f68fdc68f01aeed181eccc160a99d61e1b3c91a2cb644cbb", |
| "stored": 3488724 |
| }, |
| "l13.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.4288315773010254, |
| "kappa": "sha256:d35ea0e8d7c9922b70d1302712524fd861ccca7f921a1871fc39e80a7ad54460", |
| "stored": 3498956 |
| }, |
| "l14.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:a264bb5ecbac5937cc12b21f267a4a70a6e979dcf2ab7649afd444842ac52624", |
| "stored": 8691 |
| }, |
| "l14.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.903365969657898, |
| "kappa": "sha256:3bad331279a288104d892bba3a0e8ed5175082850b5827728a5b04cdf06ec43e", |
| "stored": 1207552 |
| }, |
| "l14.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 3.034632921218872, |
| "kappa": "sha256:e98c45cec51826359b25113f65db90c28839d4d6bc6f729cc983d9c48d817745", |
| "stored": 303902 |
| }, |
| "l14.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 4.2067975997924805, |
| "kappa": "sha256:d95bc5b2dfb66888b944eefc51a6ac7f852b7a0da5a60e1db41bb50920992503", |
| "stored": 325952 |
| }, |
| "l14.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:fdc4396b4db79821762c8c0b919d714ca31bdbb933c39bb22fff20f70fb25c59", |
| "stored": 8938 |
| }, |
| "l14.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 2.605940580368042, |
| "kappa": "sha256:f4164ec457b9a9ad9e9f7c0096b6e46f6152f26b7d79a468642943c763c5d5e1", |
| "stored": 1291367 |
| }, |
| "l14.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:1179bccfbe821def2c4ffd61cc98f0821035f45663d6ed2240187a47e4497eee", |
| "stored": 8563 |
| }, |
| "l14.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:1b2dca0ba6d611ee443713dfebb348ae97625e128b9a264fea9c457a72594f0d", |
| "stored": 24402 |
| }, |
| "l14.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.1594929695129395, |
| "kappa": "sha256:5df37995fa6cf362b8208a9eb37c3f303c329ea520c313bf5b41f6e6cea418d4", |
| "stored": 3482372 |
| }, |
| "l14.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.13179874420166, |
| "kappa": "sha256:00904d6069702683877dca67c3ee8ac79a78ab203e25fd0d85987c4fafd56b39", |
| "stored": 3494211 |
| }, |
| "l14.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.469046115875244, |
| "kappa": "sha256:2aece1343e2c629f428331ee0b4c7b324c04c03194a406e8cae92933970d211f", |
| "stored": 3503208 |
| }, |
| "l15.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:ded8e8312e67940c5af55a908065f298ea25fb01c847fc25479fdf710647bcb3", |
| "stored": 8656 |
| }, |
| "l15.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.677456021308899, |
| "kappa": "sha256:4b3dd548c05d8897350438e4364166f87c85e77677c578aecdb0f7b69df706ef", |
| "stored": 1259872 |
| }, |
| "l15.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 2.7184839248657227, |
| "kappa": "sha256:83a49f7a524d6d21068c0b807670b144be3ada2b996c706fe246a56652e5484f", |
| "stored": 318226 |
| }, |
| "l15.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 4.493652820587158, |
| "kappa": "sha256:5092b09a9dc207559930ae71596a806768cdbe419dbd5ea4badc61b320bc9ca1", |
| "stored": 326215 |
| }, |
| "l15.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:aacba1bef1ca54367682cb7f27adf599942f83054eeddb59ee6865c6696afa4b", |
| "stored": 9046 |
| }, |
| "l15.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 2.8107123374938965, |
| "kappa": "sha256:bc2e1003d4239e3d6add2120a7ccbe62c9d012b7313f8fd90e2cdb251208975f", |
| "stored": 1302692 |
| }, |
| "l15.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:750e282ef0bb0fc88e9c931b4855b19a41e01599458ff56c52e0fc0f7a3e310b", |
| "stored": 8554 |
| }, |
| "l15.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:e8050e8d36560a87bed0ab7891b7b4154ef14729ccaca890186dcb837311f67e", |
| "stored": 24399 |
| }, |
| "l15.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.173977851867676, |
| "kappa": "sha256:24df807c88617ed524734893904b1d4a762fe1702ea0d196522ad3777dc6fb2b", |
| "stored": 3478897 |
| }, |
| "l15.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.1298775672912598, |
| "kappa": "sha256:92758fe435447026ae72fb53fbfb5b38add1c050c864d4952c8a31c316bd2716", |
| "stored": 3489936 |
| }, |
| "l15.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.612762212753296, |
| "kappa": "sha256:276eb4c9cc8ae430b093c26f1945e382ba85fc49a625ceea2ff9d3f911d7da24", |
| "stored": 3507716 |
| }, |
| "l16.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:0f54af4af52942cfffb7946e06ff3c77498e564870d114169bcb645b61d1087a", |
| "stored": 8578 |
| }, |
| "l16.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 2.0254507064819336, |
| "kappa": "sha256:493452dcda405b4028168c794a368de18a4d2f122f8c7b06589cc0cd3ab1a8d2", |
| "stored": 1239996 |
| }, |
| "l16.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 3.05548095703125, |
| "kappa": "sha256:5ec8af543da79c2c623cf9dcd478090eea1669cb55e4c61a038dc450531bdeda", |
| "stored": 313776 |
| }, |
| "l16.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 4.2704854011535645, |
| "kappa": "sha256:f000f04d008105c24a870287716529d1ace57ed81ded44ee72f2a5b02ebce287", |
| "stored": 326976 |
| }, |
| "l16.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:b6b99c74111928af88bed65ac9e9bbb27dbd4ee00cf7853d4728a1dbf1e64c79", |
| "stored": 8880 |
| }, |
| "l16.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 2.815990686416626, |
| "kappa": "sha256:6eef62d214b30f85321f84fa999770d5442452df0d31b9272d6b5214f43cb347", |
| "stored": 1300219 |
| }, |
| "l16.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:1496c29a71c6fc14ae7650d39fbbd92df6033e0d5fed5e21cd94427b84a8d5cf", |
| "stored": 8550 |
| }, |
| "l16.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:b23605a4c9420d202b283be11da4826f3ce096dd82eb87f521a0c87f2aa0ce7b", |
| "stored": 24216 |
| }, |
| "l16.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.2153754234313965, |
| "kappa": "sha256:30e331c1341cae2ace9c85e8a16586377409ae5e937fb6a756657c24c87ee262", |
| "stored": 3466446 |
| }, |
| "l16.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.1473798751831055, |
| "kappa": "sha256:344ec4167a8a7632e43f6764b38a68ca4215274b0eb02eb0b78464112855251f", |
| "stored": 3477457 |
| }, |
| "l16.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.6256518363952637, |
| "kappa": "sha256:990bb18dad432453257166b972d6cd47325995a3cb5174fce5ebb60c3a752d00", |
| "stored": 3506553 |
| }, |
| "l17.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:0f20e28a629e981b79c3ceea71f8e30fc62596278dbe7c512a1f984bb5393485", |
| "stored": 8622 |
| }, |
| "l17.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.9280686378479004, |
| "kappa": "sha256:d7eb7e575ba7934ff322b9dc1b1bd0bfebf63bdc6337915c590f3fb7dc1ca13f", |
| "stored": 1230092 |
| }, |
| "l17.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 3.032918930053711, |
| "kappa": "sha256:ac8c0d682c8e61fc9fa3c461cbcb8797b4217fa4c294ea8666a46b0ee60e4a17", |
| "stored": 309214 |
| }, |
| "l17.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 4.596096992492676, |
| "kappa": "sha256:b649cec5999653626f25d09644c776ae0eb6966a7895bde9024576435c65ac31", |
| "stored": 325382 |
| }, |
| "l17.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:f16418d40c73b5965dee50d65f1a1f24d97798b6be6012d2631aa3efd3b79d40", |
| "stored": 8659 |
| }, |
| "l17.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 3.0144503116607666, |
| "kappa": "sha256:267b8a6ab03f977ff7406c73162e8cefedf980a0a7fcddc4465936d8d4b53bc6", |
| "stored": 1303571 |
| }, |
| "l17.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:ac0fa9d71d8d1da40ee5315f682a226ec46913bb8bc6502ecea2f52593643a30", |
| "stored": 8540 |
| }, |
| "l17.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:d64f23299fe112fbfb3b4428bed51c4bfbb6748257a7f82785239875232bf2a9", |
| "stored": 23985 |
| }, |
| "l17.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.238203287124634, |
| "kappa": "sha256:54a8d87cb1d383171d2daab130849f10c0ebbb96d2cd517f1a70cb65abc761c8", |
| "stored": 3446572 |
| }, |
| "l17.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.1656453609466553, |
| "kappa": "sha256:828afde4626e04ff930bcbb402b12430a36ed0f225f4b2c84cd9133d61119caf", |
| "stored": 3457141 |
| }, |
| "l17.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.6830811500549316, |
| "kappa": "sha256:9c309b0405f795dddee2b67f36bdfad2c501fb44120066bbd16167e9ab9b89e1", |
| "stored": 3506881 |
| }, |
| "l18.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:517104936088206453a9a54254aa38515b91d0551152461db17f061d92d3bd66", |
| "stored": 8561 |
| }, |
| "l18.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.670782446861267, |
| "kappa": "sha256:6874cc47a2d95cf8bbc38c7fdc5df1aed8a490ebf4feca24367f6c9e2849efb8", |
| "stored": 1243693 |
| }, |
| "l18.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 2.648331642150879, |
| "kappa": "sha256:dcc1c0d886f920de1376b5329e3a6c0545b4a54816ca48fba4fda52c24358055", |
| "stored": 316453 |
| }, |
| "l18.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 4.337561130523682, |
| "kappa": "sha256:e6f496710e8c6a9cb30136cc1894a448729afdb6bd50b704d4fa690508208698", |
| "stored": 324300 |
| }, |
| "l18.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:e9dcc8e69ec044bcfc58092c79e08f20b07b49daf458eea2f5a2a02488605188", |
| "stored": 8724 |
| }, |
| "l18.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 2.673166275024414, |
| "kappa": "sha256:50bef903a19d791dbd4307680dea9d799d64bcaba54b4f1fcfe61f2a918ebd49", |
| "stored": 1296641 |
| }, |
| "l18.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:4eef2ce59d99ec4e013073a2f87631f23c97499b5ca85c52fecbdda63218caaf", |
| "stored": 8547 |
| }, |
| "l18.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:8846089de1b69f24ca084003083a7a76dbb91c92a030852dda6b4e7c43ca6d46", |
| "stored": 23894 |
| }, |
| "l18.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.2899796962738037, |
| "kappa": "sha256:ca77d6cebb41e9a66d40bfd701a33030b4a8b31984d7a14395117e8463fdb7d1", |
| "stored": 3427966 |
| }, |
| "l18.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.1707653999328613, |
| "kappa": "sha256:2f01981176c0598566378aae2a886774ab4602635bc9f89eb0428eaee9c0cf74", |
| "stored": 3436621 |
| }, |
| "l18.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.7724497318267822, |
| "kappa": "sha256:5f92278e7599b2bd58e4bc5549430cc64830d9905111c51f5733bfa883f4b902", |
| "stored": 3505816 |
| }, |
| "l19.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:00106c7da56920948f89ce65dedab7a0f75d03dabc92b894decad1b71ea4bfb6", |
| "stored": 8599 |
| }, |
| "l19.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.6231597661972046, |
| "kappa": "sha256:4ad34a97b8fb441312850fb75fc07a9185bce856548d29018b5dc2562cc50701", |
| "stored": 1268311 |
| }, |
| "l19.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 2.524646759033203, |
| "kappa": "sha256:d78eb60a5cb4a7f66de0e13413ba25a45b3827174381696b342ea70ff7055234", |
| "stored": 320160 |
| }, |
| "l19.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 4.454233169555664, |
| "kappa": "sha256:a0eba26716db9d2b9ff7fb08adbf950a2bf976c702f8571f2c7332e2219bc1bc", |
| "stored": 326926 |
| }, |
| "l19.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:d58ac61548347cc9d5bcaaac09cbd3a1e96076ab7073259ea739cc3e774228ca", |
| "stored": 8698 |
| }, |
| "l19.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 2.8433892726898193, |
| "kappa": "sha256:52627c2dadee3500e2643ede01671cf90731ee65a5266a3a65f50cb38ccde42d", |
| "stored": 1300928 |
| }, |
| "l19.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:22c364a011f0ec1fa7f3d385a7339a352377b62e54e15d507205becd332ecbc2", |
| "stored": 8540 |
| }, |
| "l19.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:507711f451e3ad786e0a0462ff90677030311f505e5950bc49f47c2fb2fcd87d", |
| "stored": 23975 |
| }, |
| "l19.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.294785976409912, |
| "kappa": "sha256:e93e6c869aa7369fe53be80149fa43bab10c54f65c2ea710609b4496bc796659", |
| "stored": 3404690 |
| }, |
| "l19.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.1719775199890137, |
| "kappa": "sha256:4fa49d77ce81f400fbe9d1d3b1ab872567f0b79bc431703081b817ccf401a1b5", |
| "stored": 3412065 |
| }, |
| "l19.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.7865946292877197, |
| "kappa": "sha256:c687e509f6ca2d8ce697d1bed8b5e032c8738656d48452f65022e92d68065df2", |
| "stored": 3503942 |
| }, |
| "l20.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:8e3c6389b51bc756cc73ec651e430827458ebaf5274bf58381e2ad76cf457ad4", |
| "stored": 8566 |
| }, |
| "l20.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.8413779735565186, |
| "kappa": "sha256:560c7b16712e5ef195650b5f5c7e429802985e752148efdf53389eed5f8daa4b", |
| "stored": 1223845 |
| }, |
| "l20.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 2.9852893352508545, |
| "kappa": "sha256:da1ccee487cf5de7bae896367abbe900c7705d5b2f64eddaf7ac429e2032a28b", |
| "stored": 308303 |
| }, |
| "l20.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 3.960118055343628, |
| "kappa": "sha256:9fa541ea8b7ee59458cfe4f83e57b461d10fbca7538acd8be7d7afda08349f22", |
| "stored": 326330 |
| }, |
| "l20.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:127743287e8ddc2b17670148f7eee9f365b688100bc923e9233fec2743319f96", |
| "stored": 8722 |
| }, |
| "l20.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 2.505194902420044, |
| "kappa": "sha256:82cd613b37484037a0a00de779c4fd8088747eadb6039dd1135c4998d027091f", |
| "stored": 1295646 |
| }, |
| "l20.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:d6333fda0fcda1d719568427e1bc5e37dfc4bd18a3b9ad200da8dee38973ebfe", |
| "stored": 8518 |
| }, |
| "l20.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:b87d77358cedeedc339ada895199e1bc1a79ad48e9af42ca87f705ac3a4d7555", |
| "stored": 23973 |
| }, |
| "l20.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.320962429046631, |
| "kappa": "sha256:c133b62423ac9c45d1d6e7ecd42b0e73c18626d76f61ba8ec9d1fedaef41772a", |
| "stored": 3404621 |
| }, |
| "l20.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.174668788909912, |
| "kappa": "sha256:94c55ffa1077470de6ae0b21e8e1992156209f1db671481b0d66c1e545689740", |
| "stored": 3411028 |
| }, |
| "l20.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.8613626956939697, |
| "kappa": "sha256:5b311fbb61aee7a40116f53a56058145bd9f4f0a59090b53da0aad10ac255612", |
| "stored": 3505464 |
| }, |
| "l21.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:30cf1c3d05222e58ec1a470fccb701d424597b0f8afa6d155567fd64a37eef47", |
| "stored": 8568 |
| }, |
| "l21.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.6465575695037842, |
| "kappa": "sha256:b62ec365a6c19d2fb50022a3d8cbd71c14a8209ca38d5b52fb8dab9160dcade5", |
| "stored": 1220151 |
| }, |
| "l21.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 2.761808156967163, |
| "kappa": "sha256:eedc19c537279f080f520599f93dea11b0c43676a965d859ec3c5cd8ad53ca2b", |
| "stored": 310100 |
| }, |
| "l21.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 3.8352205753326416, |
| "kappa": "sha256:8658ff32cf9f495f7386962bb18e04325d11b3e3cab8723932dd52ec4ce95b27", |
| "stored": 325277 |
| }, |
| "l21.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:6b9ec2571e281175b9b630a589b60186d96de29cfcf4f67fedb968763c861266", |
| "stored": 8763 |
| }, |
| "l21.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 2.6626720428466797, |
| "kappa": "sha256:f41bb62b4d000d6318c9a4c2b00bb5f1a902d0931172259d0ca33845df7a0a34", |
| "stored": 1272990 |
| }, |
| "l21.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:349c4704429228880575cb52d0194c563fd4c4ad2b7788b45b61cb5dddf40547", |
| "stored": 8488 |
| }, |
| "l21.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:40058be0829a80e29574fb7b24a039d451e3fd5b5330ab4d3af349cd0184d9bb", |
| "stored": 24041 |
| }, |
| "l21.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.3371989727020264, |
| "kappa": "sha256:cbf76a0cb2c0aaa666a91b575ca5678dff71b1f3ed74c23ae70837c3f5dd0dd9", |
| "stored": 3433914 |
| }, |
| "l21.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.169843912124634, |
| "kappa": "sha256:903bec8d8a182aa6f4c1c61874c37bdb8c1c699b8bf780df500249b94d9425b0", |
| "stored": 3439530 |
| }, |
| "l21.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.823518753051758, |
| "kappa": "sha256:5c2c91be76f3881703b83bf3ac35b899b95625e87a144f0d96be8458f911aa9b", |
| "stored": 3510331 |
| }, |
| "l22.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:a73a8987c7cc27c59fcab1cdbff5b5d7035888700f9442271b67f54297b21ffc", |
| "stored": 8531 |
| }, |
| "l22.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.528588056564331, |
| "kappa": "sha256:abc0b300f88df6c475cdbf7e06857993beb950183d813944857f7d25658a123c", |
| "stored": 1247452 |
| }, |
| "l22.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 2.4565179347991943, |
| "kappa": "sha256:b4c681d094b1ae7992ea53ce55d217dd910793472b4adab70f7a65ae92af62cc", |
| "stored": 318639 |
| }, |
| "l22.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 3.9668362140655518, |
| "kappa": "sha256:224cc629b5a06de721dea89be0d88147278ff4d05972461182a8ac32661efba1", |
| "stored": 326609 |
| }, |
| "l22.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:0c2c00df431d32b192708249c0cb55571b964c8dc60db363564b57b1659a0414", |
| "stored": 8742 |
| }, |
| "l22.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 2.521069288253784, |
| "kappa": "sha256:7174e54c88b883b37655b2aac63ac47b6ced4b6d6e04d786d309bbd39a2b205e", |
| "stored": 1269258 |
| }, |
| "l22.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:e4f17f179b4237c7e91be89b507db87a425c0930107f97c0c1bb35272efdf130", |
| "stored": 8461 |
| }, |
| "l22.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:4f2e7bc4baf5a4a9a54b8ddc1314edd819ba92805db16fc1f3bd83a6b1b1888b", |
| "stored": 24071 |
| }, |
| "l22.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.3953351974487305, |
| "kappa": "sha256:3bc149faa6c7602e01eea13cf5a3837173f3e08a1c382b7535897bd05503def7", |
| "stored": 3469248 |
| }, |
| "l22.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.2749760150909424, |
| "kappa": "sha256:b5ea7f1465b20b3e594fe020570bcd695fc21f19e080a90918bf2794f497e79d", |
| "stored": 3476782 |
| }, |
| "l22.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.8763983249664307, |
| "kappa": "sha256:6f9c135a7267176b61195221873e378b72368b7ab338ff47b513d81192558c27", |
| "stored": 3515761 |
| }, |
| "l23.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:743c18ab6e0518ac8a5962e953752919a650a9b768a663147965b367e36ff716", |
| "stored": 8562 |
| }, |
| "l23.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.692755103111267, |
| "kappa": "sha256:3c8bb597863df34a3f82a4d1273ac2f62ee5bd8d3817123d08b63e4408c1ec0b", |
| "stored": 1230694 |
| }, |
| "l23.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 2.8201286792755127, |
| "kappa": "sha256:2c840dac40e5696a4df066766818d43323a46fa75e196d7c03880e5458a80deb", |
| "stored": 308041 |
| }, |
| "l23.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 3.7103710174560547, |
| "kappa": "sha256:3f686ed0ed0079fa099a20857d1d0a33c5459c6ec583aecb1a1d6fe1b97f4bb0", |
| "stored": 324071 |
| }, |
| "l23.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:a90180ffb2d5aaa84f638c7f59e35d7550499d9debee7b8d7992a23eedea63f7", |
| "stored": 8855 |
| }, |
| "l23.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 2.560340642929077, |
| "kappa": "sha256:133ecaeb6f2a062ed4d32f7f9f9f5a38e29d681eb3c2c69e60ab860360b6f450", |
| "stored": 1291366 |
| }, |
| "l23.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:509d17c71fa544fdf00ef93a2450b65b6a5a21dc522f30c0c4d718276e408d8f", |
| "stored": 8440 |
| }, |
| "l23.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:f399b63dd99640938ece6401fc9162ee7b91c336f75967736e8001c3b6a59ba2", |
| "stored": 24240 |
| }, |
| "l23.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.3736634254455566, |
| "kappa": "sha256:04dbd4bff550b1bbdaf7b6a18ed29d3233d5b021ea4fed69805eae38b561a6c5", |
| "stored": 3470924 |
| }, |
| "l23.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.259279251098633, |
| "kappa": "sha256:1e8a92e4782dadc96a03b41934d89b9cb6bab0f775d5e889e67c2545a87aa0dd", |
| "stored": 3477261 |
| }, |
| "l23.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.924910306930542, |
| "kappa": "sha256:c297405a11b5831def2a9cb0037167a943dcb77a4f4be898fb08567ab9364456", |
| "stored": 3516781 |
| }, |
| "l24.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:f0b19a0d276bd6d4ab9b659f480832b67626631745c33c56ce7aa66f359e8280", |
| "stored": 8550 |
| }, |
| "l24.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.7878329753875732, |
| "kappa": "sha256:d8c4f24a6729fd30107d1934a526773f2c5e4ba79498691a57197d980575aa27", |
| "stored": 1218794 |
| }, |
| "l24.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 2.8064944744110107, |
| "kappa": "sha256:14f3f0e7645a921aaa205e22dbed689bd0aec261ddfae5f7cd69033c4eaab748", |
| "stored": 301918 |
| }, |
| "l24.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 3.817347526550293, |
| "kappa": "sha256:1e87e8a72f350cd76305ba27be501b117c819178f8a2838431ea61525673d8e9", |
| "stored": 323701 |
| }, |
| "l24.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:8144e7426c961d7703a43d04e946fdd15b9d7375efa96b12d59a0973ff80489b", |
| "stored": 8872 |
| }, |
| "l24.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 2.772810935974121, |
| "kappa": "sha256:f88a1fea167b8b9f99283155b982ce848165b07f9728f3638f6610ca7dabbaab", |
| "stored": 1293811 |
| }, |
| "l24.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:29cc94d8b735f36c71daba9acef09a8daf4e55ee2a6429b0f745b28265aa26d4", |
| "stored": 8413 |
| }, |
| "l24.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:7f03aa58bb4e7de4c4903fc8ac23bd7c47c5066e89ac69beff63ec50bc40d636", |
| "stored": 24376 |
| }, |
| "l24.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.3518364429473877, |
| "kappa": "sha256:5a48d35cfa2ef5ec109d2bee316d3c3a6537ddd41095d54a77d517d03d0d9be4", |
| "stored": 3456817 |
| }, |
| "l24.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.2403571605682373, |
| "kappa": "sha256:d234d9230a9f9ccc47d72a02ec7e2bdedaff76f3d998e8fae1d617d8e0077d85", |
| "stored": 3462861 |
| }, |
| "l24.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.9294803142547607, |
| "kappa": "sha256:b25493c731a2adc297043ff96dbaa5cbb37427153d9c33df628f402cacb3e571", |
| "stored": 3516448 |
| }, |
| "l25.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:eb4de016dff57919b1bdd351efc0a27a87fa43fc46c92cdad245b4e76be4e44f", |
| "stored": 8491 |
| }, |
| "l25.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.4664311408996582, |
| "kappa": "sha256:ac920288b87ca4545bd4d30d2fc90905a90d8100e6a195432f8b6ab748ce8d77", |
| "stored": 1265731 |
| }, |
| "l25.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 2.3266441822052, |
| "kappa": "sha256:e7c45ae596c603530e29ca9c667c61a659e96618a4fa075b5bd7f925d0c5c757", |
| "stored": 317412 |
| }, |
| "l25.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 3.356239080429077, |
| "kappa": "sha256:ae0b2fa0a2454ceb89791fdaf60d98c9a1e1e6a4ba0fce3a3bf347a10fa02407", |
| "stored": 325160 |
| }, |
| "l25.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:dc4a6ce354f97a954032b0e1169bd35e79cd962c0667054c799b3fa9bf2d7b89", |
| "stored": 8984 |
| }, |
| "l25.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 2.084083318710327, |
| "kappa": "sha256:eecf3ecfd1edd7f285c5d0de58aa52fed4763948954bf809bb139643903806f7", |
| "stored": 1289589 |
| }, |
| "l25.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:05c76568bb678ead4bfb73bad8fb9be5069754cc1bcf9484ab526853385bdaf0", |
| "stored": 8365 |
| }, |
| "l25.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:d726a5a55834d9e57740cbd6b0a08480f64a26b6ad2c11de442fe5441b2cd60a", |
| "stored": 24435 |
| }, |
| "l25.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.405451774597168, |
| "kappa": "sha256:8fae6f99ff235ebcb1b7769ce0e53e794f422717a1b3a171a0c4a0fe259b5334", |
| "stored": 3460861 |
| }, |
| "l25.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.3046693801879883, |
| "kappa": "sha256:21ec2d137c5e0ca3ae0f56245afa922d01fb29bf1ebb4480c0a915697fcdc1f7", |
| "stored": 3467553 |
| }, |
| "l25.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.9113521575927734, |
| "kappa": "sha256:bae4ab9bd5c58075e2ef3591a107694bd3fc51634dcab11d81e6971e9ce9c2d6", |
| "stored": 3517866 |
| }, |
| "l26.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:83f085266b33846306a0e16ee5b74ba8355084e089be19b38f8f4f647aa3c60f", |
| "stored": 8555 |
| }, |
| "l26.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.603651762008667, |
| "kappa": "sha256:3d9b9734c1e945ac56bff10d8169b3cd2a8657bae1690df7e902b74712e795d7", |
| "stored": 1266631 |
| }, |
| "l26.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 2.433367967605591, |
| "kappa": "sha256:cdfebf9ccead909ed0bc6be77278b1d9c44cd3e94fc88da44d511a579e93805a", |
| "stored": 316815 |
| }, |
| "l26.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 3.24575138092041, |
| "kappa": "sha256:475d1c65d0d365d9644350537d39656d7fc837b94744e81bd852d3ec25ed2a73", |
| "stored": 323487 |
| }, |
| "l26.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:d461b9bfd5221e632ecbffe44c2516c8aee3c8e1141d1c1a8d3164baa9fa2497", |
| "stored": 8993 |
| }, |
| "l26.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 2.0014703273773193, |
| "kappa": "sha256:9fea89cea504d6bb6ca2cbfbe539dd0965249f74b9ec4219ca745c2797dc29a5", |
| "stored": 1288982 |
| }, |
| "l26.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:19077f98b1e6689e83425e3427c00a1d9cc889ced414e26a747ad565c8e3eae2", |
| "stored": 8345 |
| }, |
| "l26.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:d504a971e507434a54e4b9331e99bdaf4eca05c9bf4b80a87fb5e283a44a9846", |
| "stored": 24498 |
| }, |
| "l26.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.4668102264404297, |
| "kappa": "sha256:c68f0c47d8aea1ddb2bf3e1a8009a88995c18dea60712016bf52a6c048e75c26", |
| "stored": 3449930 |
| }, |
| "l26.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.360677480697632, |
| "kappa": "sha256:44d0ccc8f8c4b429000e5022e7a1fbcc234b93e08374f7b0a38046213eb7e8d7", |
| "stored": 3456823 |
| }, |
| "l26.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.857257127761841, |
| "kappa": "sha256:14bbaaba92a70aeb828098d7c0ddd6391d981fbb4dd028047894c843760cc807", |
| "stored": 3516292 |
| }, |
| "l27.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:461b1ec9e36906c7388a64e41813d238b3034263b8b9bd9bee4328bb256885e7", |
| "stored": 8534 |
| }, |
| "l27.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.5555118322372437, |
| "kappa": "sha256:7b8af3cb1981ea7ce57c583918161792ed16dc88e667be246dfc74cc049e2ec7", |
| "stored": 1264784 |
| }, |
| "l27.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 2.5493197441101074, |
| "kappa": "sha256:de36e7a63c247ab1d70583f5fd74984bd1d18ad746dfc4ceac4d5c2f508cf6c3", |
| "stored": 319347 |
| }, |
| "l27.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 3.6846508979797363, |
| "kappa": "sha256:8455cd66cb928b1fb724b2fcb77e7f6c3cf3b6e30cb7d12547ccde7a006c5ab6", |
| "stored": 324856 |
| }, |
| "l27.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:c74de5999a150775e068568ce30fe916dde459aa046a86bedf6b364a71b8fec7", |
| "stored": 8997 |
| }, |
| "l27.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 2.227123260498047, |
| "kappa": "sha256:117e30fe70bb51f022011c49f791f461361e9de5994249448febeb095f670503", |
| "stored": 1288074 |
| }, |
| "l27.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:576de6f306582f82137be9fb7fd374cbc89163b8aff4b45921d11e18b9325981", |
| "stored": 8354 |
| }, |
| "l27.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:2311c9e4e1659c2746388cc24e2f1ffc7a7a7c18c2ce1ecc2eb96eb4d354a43e", |
| "stored": 24540 |
| }, |
| "l27.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.518040657043457, |
| "kappa": "sha256:c5a2403fc17aa441387208d4c3b79e1fe59040a814819c0b47a500e11ed75787", |
| "stored": 3419538 |
| }, |
| "l27.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.4069950580596924, |
| "kappa": "sha256:819611ec8b734136e76af930c3d7fb803f316adf7bcef9e37c285c5e81546f8c", |
| "stored": 3427094 |
| }, |
| "l27.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.7643442153930664, |
| "kappa": "sha256:7826b731f5dfa4601b7bcb09edbccc79cf87737fd331eb9da3bb49f70dc95b94", |
| "stored": 3512578 |
| }, |
| "l28.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:fbcf6afd35b08aead3a8172a63e15e288217e50b3d5416965d684cf3d6b3a010", |
| "stored": 8533 |
| }, |
| "l28.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.3656436204910278, |
| "kappa": "sha256:825958482b4253f83a8d73f827f09648f4ab23126188839ead3f05744ebd29fb", |
| "stored": 1283066 |
| }, |
| "l28.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 2.0675907135009766, |
| "kappa": "sha256:1e31cee59a4e61bebd41947416310feeee63a8da865cee0c2b7c87941e1cf5bc", |
| "stored": 322415 |
| }, |
| "l28.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 3.021132230758667, |
| "kappa": "sha256:45db25240f93cabb269a60a8a55aac0bfd9c9f3a53276b6217f7db32c5a6409c", |
| "stored": 325709 |
| }, |
| "l28.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:75048c023a7512cae301bda2104eedb6e80da9a474bb994ac7bf8dff85abac7e", |
| "stored": 8950 |
| }, |
| "l28.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.8344933986663818, |
| "kappa": "sha256:f9237a80fc688a7a9d17c1469fb32cf6aff3c11a956407fe7a11833a2c180964", |
| "stored": 1276637 |
| }, |
| "l28.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:4e47460f932fb3de086403ed87c58f8c396ca71592c3e8802e284b0e8c97525f", |
| "stored": 8343 |
| }, |
| "l28.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:ceef5b646b09c7ad474258a36341b56916c14bdd69caaff402ea2d8f9a986cc0", |
| "stored": 24531 |
| }, |
| "l28.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.6670985221862793, |
| "kappa": "sha256:56e55084168673cd450d5c573d90935233c37a051e96a06ac5d8c61ab297ecc3", |
| "stored": 3437148 |
| }, |
| "l28.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.518381357192993, |
| "kappa": "sha256:74d50c2f0973a716e1cfa1441244bb980ed2a28465c0d7cd89cf1f17bff2137d", |
| "stored": 3445378 |
| }, |
| "l28.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.6306076049804688, |
| "kappa": "sha256:804f0c55d0115b755b29899622bb93bf2ac2c1083c693580e0fdb31c835b04ba", |
| "stored": 3511351 |
| }, |
| "l29.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:62e08a7584c82b53b9e06bbb439a0bc813f6deb70da05a5d84db8f6acc9dee14", |
| "stored": 8618 |
| }, |
| "l29.wq": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 1.1274330615997314, |
| "kappa": "sha256:e3801c026bc3665c300144cd314052358b2d28f3de18810e527078cacf8ebb1b", |
| "stored": 1287315 |
| }, |
| "l29.wk": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 1.6596884727478027, |
| "kappa": "sha256:d7c44c457b5ce46e0e950a10bc101051392c3bd8c1cea75e06b9ea4609229a1c", |
| "stored": 324404 |
| }, |
| "l29.wv": { |
| "fmt": "t2", |
| "N": 640, |
| "K": 2560, |
| "s": 1.7768195867538452, |
| "kappa": "sha256:8469b051c78fb5b20a83afce79bb050fd2aac70d0d5c1c143f85b3357d9ba565", |
| "stored": 286956 |
| }, |
| "l29.attn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:c3f392431d9aff6e1ed3dfcc9d9509258dda1434a19d8072bc3f8957336db6b1", |
| "stored": 9477 |
| }, |
| "l29.wo": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 2560, |
| "s": 0.9527681469917297, |
| "kappa": "sha256:ddd5d3da391b17f6ec654ae6b5df9edf5c380667a8f868b2e7245fcaafa65c98", |
| "stored": 1206227 |
| }, |
| "l29.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2560, |
| "kappa": "sha256:057906d0c07df226e44c8548315f606ffacbc88000385b8f180987d5036ffc8c", |
| "stored": 8313 |
| }, |
| "l29.ffn_sub_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 6912, |
| "kappa": "sha256:435fc9aef5e13407a429e5d3a99d37665f37d7ac6927b9ba0562a6b4937cdde7", |
| "stored": 24528 |
| }, |
| "l29.w_gate": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.8299777507781982, |
| "kappa": "sha256:3e29552ae3748c2ca6970c5a0084268c67b5754c58561f59763010654afeebc4", |
| "stored": 3404362 |
| }, |
| "l29.w_up": { |
| "fmt": "t2", |
| "N": 6912, |
| "K": 2560, |
| "s": 2.602970838546753, |
| "kappa": "sha256:c7e0902606f5dd55a400ad2da9370a230b75c81dd8af6b5bf53e49c7e2d506fc", |
| "stored": 3412728 |
| }, |
| "l29.w_down": { |
| "fmt": "t2", |
| "N": 2560, |
| "K": 6912, |
| "s": 2.288531541824341, |
| "kappa": "sha256:8a7695baefc5a701788c4e4d1f539507ee1289d86c1943fb7ac94c860c946883", |
| "stored": 3500362 |
| } |
| } |
| } |
