| { |
| "format": "holo-2bit/1", |
| "mode": "bitnet", |
| "model": "1b-instruct", |
| "source": "https://huggingface.co/tiiuae/Falcon-E-1B-Instruct-GGUF/resolve/main/ggml-model-i2_s.gguf", |
| "bits": 3, |
| "layout": "q3f", |
| "twoBit": false, |
| "incoherent": false, |
| "root": "sha256:57ffdc131d47d668c2eac105e95e5bfed953441f43ae735cc5184733e70aaf5d", |
| "d": 2048, |
| "n_heads": 16, |
| "n_kv_heads": 2, |
| "ff": 9216, |
| "vocab": 32768, |
| "n_layers": 24, |
| "hd": 128, |
| "rope_base": 1000000, |
| "attn_bias": false, |
| "qk_norm": false, |
| "qk_norm_dim": 0, |
| "tied": false, |
| "tensors": { |
| "embed": { |
| "fmt": "q3", |
| "N": 32768, |
| "K": 2048, |
| "kappa": "sha256:ee7e77035b8931d98bb8c0feb48f9aaae8af965b603dd8b9ed9f1fca7a9b14cb", |
| "stored": 28694263 |
| }, |
| "final_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:a27dc19f684e4a7390d6601ce899a94b12a9a3734f623e08785d96200003cbb0", |
| "stored": 2203 |
| }, |
| "lm_head": { |
| "fmt": "q3", |
| "N": 32768, |
| "K": 2048, |
| "kappa": "sha256:0db3c5f7f86e395c06c925ff0bace79d9cd517741cd24c991b0902c0d5fbecea", |
| "stored": 28361893 |
| }, |
| "l0.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:a818b76f18d81fce5a2f925f5652948aff0817a1aefaca371fadfc21b4f276aa", |
| "stored": 3259 |
| }, |
| "l0.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.057553958147764206, |
| "kappa": "sha256:9740f7f1bf113b10b1ab0f30f544ee183fec20e588334cd050e4fa330d4466e4", |
| "stored": 832881 |
| }, |
| "l0.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.11594203114509583, |
| "kappa": "sha256:d0fad13bb97f9d91030df611592e451275af7fc103f39bb5d2175a7f35c7ae60", |
| "stored": 104355 |
| }, |
| "l0.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.01587301678955555, |
| "kappa": "sha256:14d2c60af1adbe5d19cb9751224ae57f3547e3301ab5fe42bd8793d34f51d702", |
| "stored": 104589 |
| }, |
| "l0.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.02666666731238365, |
| "kappa": "sha256:ccd01cd61c94ba0dab729733bbd99bffad862409dddcca4e0124fb59d4d20d45", |
| "stored": 835101 |
| }, |
| "l0.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:ad622590b745d7a5357a5186557a69c39cb1417776a203a5d09252558d817a85", |
| "stored": 2287 |
| }, |
| "l0.w_gate": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.041450776159763336, |
| "kappa": "sha256:22b954669d24869a88476872268601600c6bcdc45146053581a033ecb4b5446d", |
| "stored": 3786649 |
| }, |
| "l0.w_up": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.04301075264811516, |
| "kappa": "sha256:7f54e9086e636e78e2b45979d8f9e4cbd186e3b31931bd2dc3e05f6ad4c9640c", |
| "stored": 3787662 |
| }, |
| "l0.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 9216, |
| "s": 0.048192769289016724, |
| "kappa": "sha256:00709178bef25ba2e7c763e7496b430c1bb19aee0fc73d5795e41ce25d3baf86", |
| "stored": 3778332 |
| }, |
| "l1.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:1954e860c5b1720b9097a9c38aab63518db1bb6e10d1ead51531d0a53301e151", |
| "stored": 2568 |
| }, |
| "l1.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.058394160121679306, |
| "kappa": "sha256:68936e7d0bfe435fbdf4fc91403092cccd8f26fbdf8b5eb35cb7693211e16091", |
| "stored": 835107 |
| }, |
| "l1.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.0658436194062233, |
| "kappa": "sha256:b5cf32f7666928936fe6a888be671b78140b0a8351daeae8242519e4e67883b7", |
| "stored": 104665 |
| }, |
| "l1.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.042328041046857834, |
| "kappa": "sha256:5a4b5f048488abbbbbf21ab2ae197479c129adff3f55ae6d9266e9a5d9da9405", |
| "stored": 104110 |
| }, |
| "l1.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.04494382068514824, |
| "kappa": "sha256:443a01bc85b551a3a35bea41be8d0d653e8efcadef354c54731c95d01d073765", |
| "stored": 837700 |
| }, |
| "l1.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:18cd5b5ea0047a59729664e1b32ff5afa0174087e7a942472ac3981aafa5a454", |
| "stored": 2407 |
| }, |
| "l1.w_gate": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.04848484694957733, |
| "kappa": "sha256:07b7a250680ace71d4aa66ee818cf731ba7d3925d8e1517a5075f1835f9d442f", |
| "stored": 3779326 |
| }, |
| "l1.w_up": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.0470588244497776, |
| "kappa": "sha256:5f5fa1fae66de68121fe4c6363ce0887e8d10d0867229122025df6f7000decd6", |
| "stored": 3778219 |
| }, |
| "l1.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 9216, |
| "s": 0.050314463675022125, |
| "kappa": "sha256:5e7cf7295d4b0c6981bf211cdd2d06ad7d1a1f9e4b671cf86fb1d35f92757891", |
| "stored": 3764980 |
| }, |
| "l2.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:a5044717b363f92a029fd35f573aea1a1ff46267dd52beafdc537088f201903c", |
| "stored": 2397 |
| }, |
| "l2.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.06299212574958801, |
| "kappa": "sha256:3da0047cfb1b5afa64efc17d08f32d39395d5fc216b18e2c333fdc7c76f38a49", |
| "stored": 825540 |
| }, |
| "l2.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.0824742242693901, |
| "kappa": "sha256:c47624e6c2c3c62d475917bf04087cef17bec542444ec1b6daa3b4e39a13e28a", |
| "stored": 100486 |
| }, |
| "l2.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.05161290243268013, |
| "kappa": "sha256:4c097bbec294ff98aec7a0174e51293e248e9a19502783a94c5f48184823aaf5", |
| "stored": 104013 |
| }, |
| "l2.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.047337278723716736, |
| "kappa": "sha256:0ce4ae955cf639d3270db2283d77bd9fe28fd292c6b3055781d92aa42d58f1f6", |
| "stored": 835291 |
| }, |
| "l2.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:fbfefe75b36d899a58c3ee9f512cc90e66ae91ab95f413cc70bff7f9f0df62e0", |
| "stored": 2323 |
| }, |
| "l2.w_gate": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.054054055362939835, |
| "kappa": "sha256:4ed0b802d6b3375e48853e03cb5af1f4fdc3384dc29a759d192b63f696e0af84", |
| "stored": 3770392 |
| }, |
| "l2.w_up": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.05161290243268013, |
| "kappa": "sha256:101ee407e196724b074f40e587e6644a9bff534bcdd6f1cfc87e4e39704c5d93", |
| "stored": 3767637 |
| }, |
| "l2.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 9216, |
| "s": 0.057553958147764206, |
| "kappa": "sha256:a438939c575b4595b13b6924209ef714f554d80733143e4f5b8f93822f753b65", |
| "stored": 3754706 |
| }, |
| "l3.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:988b1812b0bd8b6aa0e461b8a9d48d666e0dc9541928ea61bc9830ecf68d5e13", |
| "stored": 2304 |
| }, |
| "l3.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.06530611962080002, |
| "kappa": "sha256:86c67adebf3552db1a153e4900a51bc2ce6861ab1e4c8aa0ad1ecedd9de05030", |
| "stored": 822872 |
| }, |
| "l3.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.095238097012043, |
| "kappa": "sha256:3a0becfbc338c85fcab1995882553048b0438fc10f4e53a6a1f81623882b6df3", |
| "stored": 102466 |
| }, |
| "l3.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.05797101557254791, |
| "kappa": "sha256:df474a23c6d5d0e5333f7c31cdc582210b343d7dc352f65d319b1fccf282e7a8", |
| "stored": 103862 |
| }, |
| "l3.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.04968944191932678, |
| "kappa": "sha256:af5c4e41ebe9d1d2f48877b9f3a1102a5489d7fe77e7f906337ce6a2372a8586", |
| "stored": 831828 |
| }, |
| "l3.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:06de28dd070cf140e58e6cc4be6616858602b53f005812ba51aee788798f58b3", |
| "stored": 2318 |
| }, |
| "l3.w_gate": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.058394160121679306, |
| "kappa": "sha256:607eaaa3f97e8844a9fe2668d12792bdba9e9dfae82596f51343f5561dcd1478", |
| "stored": 3769978 |
| }, |
| "l3.w_up": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.0555555559694767, |
| "kappa": "sha256:602a1b14105c4745aad1ae79a5fa7f6b6768562b1a44f5a260c3201a63134002", |
| "stored": 3759244 |
| }, |
| "l3.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 9216, |
| "s": 0.0615384615957737, |
| "kappa": "sha256:a9494219e7dca5808fd79bbca4807e42985c58256d381add53dd5588adf8ca70", |
| "stored": 3761449 |
| }, |
| "l4.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:a8e75e03b145ee538bb4ad1cffb4d0649cbae1eee0a737efa7dd56f1c47cd5f8", |
| "stored": 2318 |
| }, |
| "l4.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07017543911933899, |
| "kappa": "sha256:72ba8993b2d58eddcda29185d6ecdecb8f9f779393fa98afc920a56f75149bce", |
| "stored": 822973 |
| }, |
| "l4.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.0903954803943634, |
| "kappa": "sha256:44ae2e0bd806c9bbe1359d23fab51fffcfe9c8f307b9ea0e0f224a7a0bcfac70", |
| "stored": 103705 |
| }, |
| "l4.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.07373271882534027, |
| "kappa": "sha256:daf1a2184fce0e2de41502437c9d11c023b8449eea7eee95b870cd1bc9b1e5be", |
| "stored": 104859 |
| }, |
| "l4.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.054794520139694214, |
| "kappa": "sha256:f3e7d5a88c17002979a7e5c40d6b429b55ca7f3cfaaba126f320465b70acfbf4", |
| "stored": 830882 |
| }, |
| "l4.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:0d079308c9a0c7102783d8e8c258880f402ec0ec79c7a2c5fc06531d7401135e", |
| "stored": 2368 |
| }, |
| "l4.w_gate": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.0634920671582222, |
| "kappa": "sha256:83f9489bf7786f89bd11d49d305b0b610bcde7851c779104bcefcf2028130d9e", |
| "stored": 3761151 |
| }, |
| "l4.w_up": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.05925925821065903, |
| "kappa": "sha256:4496f104dfdbb4dbabd9865f662c55b1cd025fffbf36bc22e5f93851173805c6", |
| "stored": 3761637 |
| }, |
| "l4.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 9216, |
| "s": 0.06639004498720169, |
| "kappa": "sha256:0fc3582848b80dc9e474eb2e44c7e3b3914c823bf60907bcb81a3fd95ec7cc00", |
| "stored": 3740920 |
| }, |
| "l5.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:3db948fca45342feaef50e2ab5c946e3c9d8c4388ea7e73c715c8a5539687feb", |
| "stored": 2440 |
| }, |
| "l5.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.0615384615957737, |
| "kappa": "sha256:788fff67cc5cedfbe73d9472db0de4e5875736d37bcc4593b2d753297fbf2e03", |
| "stored": 824570 |
| }, |
| "l5.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.07373271882534027, |
| "kappa": "sha256:cd7308f3529d35785437a978a37f160d686352461cdd6c19f9d63fe6578b64a3", |
| "stored": 103319 |
| }, |
| "l5.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.06504064798355103, |
| "kappa": "sha256:9cf25f6b7d068cb7bb79983446421cbcea3140d36ba5fac4de0b6875309b2e99", |
| "stored": 103425 |
| }, |
| "l5.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.050314463675022125, |
| "kappa": "sha256:d6e30100e030377f4c6b33f00caf4d6151775b6097f61ab34d445f2126cdb08d", |
| "stored": 833362 |
| }, |
| "l5.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:0a5803819f432452589afdf1ae94b21cc14b369f073d8eefc1b6351c247446da", |
| "stored": 2380 |
| }, |
| "l5.w_gate": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.06299212574958801, |
| "kappa": "sha256:c6495c419a20d6f4384b461309cad6dcaad8bfebd055d4c4deb3392d96e009dc", |
| "stored": 3758851 |
| }, |
| "l5.w_up": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.057553958147764206, |
| "kappa": "sha256:cc6c9b653dc106f25603ec0f3729aaaac8be106a527f36d05180493cd0da8960", |
| "stored": 3746318 |
| }, |
| "l5.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 9216, |
| "s": 0.06530611962080002, |
| "kappa": "sha256:3218c04e7941cdd3fd7ee5b1ec7b8e07475893547a827b6e843a369748152360", |
| "stored": 3753997 |
| }, |
| "l6.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:bb88bf1c71eb2faa18469d5332d15845e9083d8ff3d06f814dc5b632abe0918e", |
| "stored": 2490 |
| }, |
| "l6.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.06896551698446274, |
| "kappa": "sha256:6a115a1d6313166b903cbd6faff818784c56a0960f5e282927ae0b2ba630651c", |
| "stored": 780850 |
| }, |
| "l6.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.1066666692495346, |
| "kappa": "sha256:f4a7858eff376e6f839bdbf709b590d147bb0ea919f1359118ec444e74886d29", |
| "stored": 95719 |
| }, |
| "l6.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.07079645991325378, |
| "kappa": "sha256:53366973ecf788caa0ca381c121a1deda198d7d0ae3b2cd2b95f2bf2089944e4", |
| "stored": 103690 |
| }, |
| "l6.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.05161290243268013, |
| "kappa": "sha256:1f02cd985e8c395f01cd4340c55f99af84574ae9d2a85fdd88460b697599fe54", |
| "stored": 832299 |
| }, |
| "l6.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:39d545c3b96ed4e012e53f6c36970686ad6e44f0c18eea473a5fd9009bb5f05b", |
| "stored": 2407 |
| }, |
| "l6.w_gate": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.06530611962080002, |
| "kappa": "sha256:c8d96401a8b1ea243b10c18b300febac7988daaeafe054dfcae34278d9698564", |
| "stored": 3756252 |
| }, |
| "l6.w_up": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.060150377452373505, |
| "kappa": "sha256:c323f670e12162cceec497887736aa0e3b9a2d0a8afa18e4919ce2d329d1faed", |
| "stored": 3737767 |
| }, |
| "l6.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 9216, |
| "s": 0.06751054525375366, |
| "kappa": "sha256:146e14451fff70f4b4a3bbe267fc7bf7250887cee17e04cd50ebd2e405824065", |
| "stored": 3749427 |
| }, |
| "l7.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:3b69291efd52e2a3f12ea5a3a90df9d9c60d3b521f4d55a8743483eef3544c7f", |
| "stored": 2590 |
| }, |
| "l7.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.0714285746216774, |
| "kappa": "sha256:c19a1ddc4b41e29faca6084476d778fc0c08da4f65ada68ce899bf1933038cd0", |
| "stored": 806212 |
| }, |
| "l7.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.08938547223806381, |
| "kappa": "sha256:4d8a9b4d58bc2559dba38c0596be1d9baa452593992641af9d75888169c42ba0", |
| "stored": 102441 |
| }, |
| "l7.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.0883977934718132, |
| "kappa": "sha256:4d567fa30d8be0104f2b158109efdf6f139e66249e75ff6948c68a0f47a51c1b", |
| "stored": 103137 |
| }, |
| "l7.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.0615384615957737, |
| "kappa": "sha256:52a7f0c4574e0056cdfadde5768fb9b8e2e84749d18894cb246ce6f2cbbbaacf", |
| "stored": 833232 |
| }, |
| "l7.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:fc952ddb6589d2a6364ed4b16446f4ec5a9dd4663adfe88e81f459db4dcc89be", |
| "stored": 2390 |
| }, |
| "l7.w_gate": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.06896551698446274, |
| "kappa": "sha256:e9b9074c331c0478d32058e9e216ced06c7ff2216f6523ed13014c59069175b4", |
| "stored": 3732593 |
| }, |
| "l7.w_up": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.06201550364494324, |
| "kappa": "sha256:eaa925dc4d9466dcfd6800ab12747e16269b7e3a53644f9f05a45d758f62bfef", |
| "stored": 3739084 |
| }, |
| "l7.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 9216, |
| "s": 0.07079645991325378, |
| "kappa": "sha256:a9fab967a9df87dabf988f0d0c420f55f4134fdd649224e0d8c5f1689b8d005e", |
| "stored": 3747368 |
| }, |
| "l8.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:05a03b0d78654456af45ee27d54ed25e1838fa5e06e2360a32cfaf1a01050925", |
| "stored": 2546 |
| }, |
| "l8.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07804878056049347, |
| "kappa": "sha256:2fc1bfc3ce8130ecacfa74f50a07e787dde4a36d5e501d761d48c1c54f9e6cd6", |
| "stored": 786326 |
| }, |
| "l8.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.13389120995998383, |
| "kappa": "sha256:526dd344c5c5903d6a83b77f2fa15f038e5a163ad8ce99d3c4128dd43b593c7b", |
| "stored": 96016 |
| }, |
| "l8.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.09302325546741486, |
| "kappa": "sha256:fcfc467765af2a18f0ec753f2729fb62eafe5f297ffec5e8f10069c61163dfe7", |
| "stored": 104308 |
| }, |
| "l8.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.06400000303983688, |
| "kappa": "sha256:2dd03c380207d0e0e58bedc2221ab1c42ab9ba8aa3c0a1c62a774c830cd57248", |
| "stored": 831670 |
| }, |
| "l8.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:5a296fed42e34267f3f3bf836434b727307858523220df67d64b5b2606febeac", |
| "stored": 2401 |
| }, |
| "l8.w_gate": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.07407407462596893, |
| "kappa": "sha256:42d84eb9d80446645daa21df098e2f320be0d24c275141d6f2f5aa474fc16529", |
| "stored": 3727518 |
| }, |
| "l8.w_up": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.06451612710952759, |
| "kappa": "sha256:132a4ff94ad4be557ca12ff4ef72bc622bcaf0eaf4e288c3253741985b585361", |
| "stored": 3727175 |
| }, |
| "l8.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 9216, |
| "s": 0.07476635277271271, |
| "kappa": "sha256:e4896caa6a4fb926629858a4ca3ad6c14464c958a25b726d9af858b3d084b07b", |
| "stored": 3724042 |
| }, |
| "l9.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:91641a998556a165bb22d740a2c7fcbb116bc831d92652694b5e8e9582c1bb30", |
| "stored": 2614 |
| }, |
| "l9.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.08290155231952667, |
| "kappa": "sha256:c323963605aa0cc0a206bec15c27878e36aa493344b06ba9a0f2ee23cb75c968", |
| "stored": 800988 |
| }, |
| "l9.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.12800000607967377, |
| "kappa": "sha256:8ea97a4e08f995686ecca118aaaffcb64fd6d8389d46c04458d1d1bdd14b9cdf", |
| "stored": 96236 |
| }, |
| "l9.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.10191082954406738, |
| "kappa": "sha256:ca89603dadc417ba4d1bc03e8847186429636b3cadc7a1fc12691bff372ceeb2", |
| "stored": 104775 |
| }, |
| "l9.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07804878056049347, |
| "kappa": "sha256:5e2afb5d4dca5f32a0ad7b238dfe107583bcd0be7548291ffd9821cb86d12a18", |
| "stored": 818160 |
| }, |
| "l9.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:6a937ed3c0bc91ba306f921d8dfe4cb4371e039f6a6310b34d9dce58da3e2408", |
| "stored": 2420 |
| }, |
| "l9.w_gate": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.07804878056049347, |
| "kappa": "sha256:b2ce17d20182647c4a6a8371d0e6dfad89aebfb85e915c69150c66a0f150bf46", |
| "stored": 3718570 |
| }, |
| "l9.w_up": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.0658436194062233, |
| "kappa": "sha256:0adc77ef0f656fbe052a80c06898492245f61ca19daa1e33e901415bd6774920", |
| "stored": 3743760 |
| }, |
| "l9.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 9216, |
| "s": 0.07960198819637299, |
| "kappa": "sha256:8130b508e96f9fd96630ef6f2a0e3aebc358f3258fa5e4936f895e4508af35f6", |
| "stored": 3735821 |
| }, |
| "l10.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:ef33a74356f8543ac3db2cc7ef2e01a217c3797559f5cebc55eece4f1384afb1", |
| "stored": 2544 |
| }, |
| "l10.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07017543911933899, |
| "kappa": "sha256:a74b78648a2f071da8d7e94872ceef88a8a7cab3a869344cf7d1b823a690de02", |
| "stored": 804039 |
| }, |
| "l10.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.13008129596710205, |
| "kappa": "sha256:e997f71728ced97ef71c0f36667e32e72e280f46c6884a6c6aa341eba472a489", |
| "stored": 101852 |
| }, |
| "l10.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.09638553857803345, |
| "kappa": "sha256:a4f6a2944d221225a9aa11d0d83b8c5bb7a3de8ef17f3f652fb5257bd3bf863d", |
| "stored": 101049 |
| }, |
| "l10.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.05797101557254791, |
| "kappa": "sha256:6d2c2e05e3ee07bdf9f2cb6a59a777e6d86983264683446b744756a8c56cb7af", |
| "stored": 825273 |
| }, |
| "l10.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:6697c24c75a4ae3efe0a47f46917b0f9f0fd9b16b71d753cd6ddc1658b759146", |
| "stored": 2352 |
| }, |
| "l10.w_gate": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.07619047909975052, |
| "kappa": "sha256:eee0dccf742a17c721ffcd7056796e0b77d8fa524f9ca6aeed722302b0f308f4", |
| "stored": 3738505 |
| }, |
| "l10.w_up": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.06896551698446274, |
| "kappa": "sha256:c2ea44458d016970d35d346da527dfa35a214066df4bcc697f69d3d6ad2a0e8e", |
| "stored": 3723088 |
| }, |
| "l10.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 9216, |
| "s": 0.0776699036359787, |
| "kappa": "sha256:d59474bdca40cdaf79ee8a47bfbc9deeb3ae6cb051c5d9ffa627a5056dd3e698", |
| "stored": 3732769 |
| }, |
| "l11.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:509de66e49773f3deb749b8e423f61ba20f637ecdcb2307130013f503dfd4b26", |
| "stored": 2499 |
| }, |
| "l11.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07017543911933899, |
| "kappa": "sha256:6f11378c2df3ac6be84b928555d0f0c1a8a379b0612b6d39b9fb412104ef17a2", |
| "stored": 807484 |
| }, |
| "l11.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.11347517371177673, |
| "kappa": "sha256:0f553a29d8452a0650c13655a21cf3804b69f3ab3979bef848adc4e34762e5ad", |
| "stored": 100386 |
| }, |
| "l11.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.09467455744743347, |
| "kappa": "sha256:4a28b5d67a6f86596802f7ca6f99d722cde0b38ca4437e9d1db6041746e78298", |
| "stored": 104603 |
| }, |
| "l11.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.058394160121679306, |
| "kappa": "sha256:6cbdce57d47596a86401689fad94de4d74078d4ce860401cc7aff00c96b25028", |
| "stored": 831352 |
| }, |
| "l11.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:3b2545e7ed62ae28ee4f75b354670dee8ae0e124d208456063109b0f5fda09b3", |
| "stored": 2302 |
| }, |
| "l11.w_gate": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.07407407462596893, |
| "kappa": "sha256:26c1e541af00bf0aa8d792e7701b16fc41db770504923953aa81b85f5c879d7e", |
| "stored": 3708600 |
| }, |
| "l11.w_up": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.06751054525375366, |
| "kappa": "sha256:d5b343faa13c8a2627c9a71c6bec91ac6f6af4a81185927101d6b85cf15c6d27", |
| "stored": 3742205 |
| }, |
| "l11.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 9216, |
| "s": 0.07619047909975052, |
| "kappa": "sha256:ad7c435e52076fd08226771fc103b8aa6b6838c8fd8e23f8200e26f6a1fd7ad9", |
| "stored": 3724034 |
| }, |
| "l12.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:535d05bfd3d176bb19b8afeeaf8e1722e8d855767aa0ba4396637040ed4b164d", |
| "stored": 2569 |
| }, |
| "l12.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07960198819637299, |
| "kappa": "sha256:478b3ab1ac3028515124fc187f645b1ec91f599b24a8820bc0baa1365c6b40f0", |
| "stored": 821628 |
| }, |
| "l12.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.12800000607967377, |
| "kappa": "sha256:d6a50f85804ee13181d122954c4214ca2cbb617bf72a3adacb2f40be6d1105a9", |
| "stored": 100509 |
| }, |
| "l12.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.11594203114509583, |
| "kappa": "sha256:e929ae0fddff4b80351f133ca690643c86fade29aa6c1d8a20192b62552f1046", |
| "stored": 101374 |
| }, |
| "l12.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07511737197637558, |
| "kappa": "sha256:e2e1f91fd05ee55efae5c0928e3aa522e38f2dff467bb1f5907b766cb207bfa6", |
| "stored": 830181 |
| }, |
| "l12.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:f33579d2b8315228c5ba59af2ced41d84a19d3c2c9164c33c2f4e7ee35642f89", |
| "stored": 2337 |
| }, |
| "l12.w_gate": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.07339449226856232, |
| "kappa": "sha256:26e62a8a98530b60433f9c5a5eefa30c12f89f4670f590b7217a394cb89df05f", |
| "stored": 3698242 |
| }, |
| "l12.w_up": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.06530611962080002, |
| "kappa": "sha256:d512ab906e64605106a15514f8a49ea0faa7d6127991b8201aef0eaecc9192a3", |
| "stored": 3742990 |
| }, |
| "l12.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 9216, |
| "s": 0.0758293867111206, |
| "kappa": "sha256:15d72382c7538c405924bd59748513d19d4732c6541718797c4f0734e10e742d", |
| "stored": 3735514 |
| }, |
| "l13.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:b732dad889042d272c136ffb557935a5b4f3d3bbe51a9e74113e8ac3fdd23973", |
| "stored": 2614 |
| }, |
| "l13.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07804878056049347, |
| "kappa": "sha256:5f00bffd6c38909ea17f20f65a05c1f8b20869431480bcd581d5d039ba687745", |
| "stored": 810238 |
| }, |
| "l13.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.125, |
| "kappa": "sha256:2001f419671b2cdf12b010268c4c313ecb5331c58d9aefb2e3f73b90fdc503b5", |
| "stored": 99117 |
| }, |
| "l13.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.11347517371177673, |
| "kappa": "sha256:c093a1bb426eaecd748b4b9a87b4cc160272f81a2303dac4bcab097f493926e6", |
| "stored": 100913 |
| }, |
| "l13.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07373271882534027, |
| "kappa": "sha256:7d45919226132abf1d0d542e0bc701d59000ecc7005b825a0055160d8d750540", |
| "stored": 830828 |
| }, |
| "l13.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:9d6ae6f1df0f3c00fe247a81c80a2bc93002ea1c52db11873bfd28296257e854", |
| "stored": 2358 |
| }, |
| "l13.w_gate": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.07339449226856232, |
| "kappa": "sha256:aeac2bc320374f2b664e53afa8753cafdc5003b415be897a00527731bb513ddf", |
| "stored": 3720133 |
| }, |
| "l13.w_up": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.06779661029577255, |
| "kappa": "sha256:7c5a023fa78ce1d3834e0e909e0660fcf70aef82f4ceea38d69cb5afb8180a03", |
| "stored": 3716475 |
| }, |
| "l13.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 9216, |
| "s": 0.07729468494653702, |
| "kappa": "sha256:fa0925843a93b86e2dd7d2e6d53d578ebd3879ab800446c329e99c26f0926069", |
| "stored": 3729866 |
| }, |
| "l14.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:00b0c9ab2be1b0ee5f1a13267a0f7fb994a17a74a34dd09415ec84e32b1876d8", |
| "stored": 2654 |
| }, |
| "l14.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.08163265138864517, |
| "kappa": "sha256:0a2b332567ff994a2ec261bcc4136eb47ff58695b9fddd61fdaf9582e8937462", |
| "stored": 786505 |
| }, |
| "l14.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.1367521435022354, |
| "kappa": "sha256:330b80cda38f5cccd22808665a147156aae31ae87f0cbbe45a6e010c79ccf20e", |
| "stored": 99377 |
| }, |
| "l14.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.11594203114509583, |
| "kappa": "sha256:9e3573287d2b14d7c395bf03333a0474fa43a7d40f8982fea05c3063cfb81d2c", |
| "stored": 104748 |
| }, |
| "l14.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07476635277271271, |
| "kappa": "sha256:1d1f2f696b3c59b296417813a2961d3225620f49dee3b0bfa2b5a7f77dbc6467", |
| "stored": 832246 |
| }, |
| "l14.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:6d4cad832e2b90e857e3a9afd1e8620723b8b3b16a27ba56cc95a690f6ec464e", |
| "stored": 2305 |
| }, |
| "l14.w_gate": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.07373271882534027, |
| "kappa": "sha256:b56dc98ab4941e796d1516e6b2859311975105cc2afceecb1bf41ffc4ebdf09e", |
| "stored": 3687726 |
| }, |
| "l14.w_up": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.06896551698446274, |
| "kappa": "sha256:46242e46c80c6d3a9a1c44b1b34f8d7845d2d024cb94d27e71c05d0971f28840", |
| "stored": 3741488 |
| }, |
| "l14.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 9216, |
| "s": 0.0784313753247261, |
| "kappa": "sha256:d3ff0438b00f77d62c4a3f37576845b03a954db8ccab2bed21c18936c66cacd1", |
| "stored": 3735731 |
| }, |
| "l15.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:506cfa5c7408f8fc3a9d24762882c5391b7a934b5cc90bccc253db8dc1e86048", |
| "stored": 2572 |
| }, |
| "l15.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.0883977934718132, |
| "kappa": "sha256:6363cca17f749c8e4d65ea0a770ab45a6b970413d63f5ef369621d013d8015a8", |
| "stored": 761635 |
| }, |
| "l15.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.14814814925193787, |
| "kappa": "sha256:bc7fc75ae5dccd3c50dc80d468fa1f2b95da5bf6b8be89ff67f6732f6f9e8d6d", |
| "stored": 86141 |
| }, |
| "l15.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.1221374049782753, |
| "kappa": "sha256:a9dfc1f8ddd54ae9eb8026ce1c8bd61c66384679ec6dd54641bfc2b77ebef88f", |
| "stored": 101312 |
| }, |
| "l15.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.0776699036359787, |
| "kappa": "sha256:c55271acc72108cc37969c86094d0937862bd4478ee1b80f005d6b1c7adefd3d", |
| "stored": 830489 |
| }, |
| "l15.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:dbb2d6ce5732e8b68176298a889ec6caeb82fafdee312937e271cf14ea9f4b82", |
| "stored": 2279 |
| }, |
| "l15.w_gate": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.07476635277271271, |
| "kappa": "sha256:3d0c03676c3eae2f8c75c11df76e5840468e76e86bc99a1d2a05eb2ea293f1ec", |
| "stored": 3721234 |
| }, |
| "l15.w_up": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.07017543911933899, |
| "kappa": "sha256:259838aa95bdb77cfd1508fcc03222631090a5d1801ee25bed931cd203905b34", |
| "stored": 3741376 |
| }, |
| "l15.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 9216, |
| "s": 0.0784313753247261, |
| "kappa": "sha256:611ad490466c760fbf19af3e8577fb6b26e87adb3e2988c7cd5408b09749e677", |
| "stored": 3733577 |
| }, |
| "l16.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:db3f643b3a786e87bc5f3fb40c4e88663141c084172a64e1ea18b6137ddf1e94", |
| "stored": 2417 |
| }, |
| "l16.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.08648648858070374, |
| "kappa": "sha256:e71673a6be25357e95a6ee2b8caeb8fd1e04c12d6492656bee73476bc8be7a2c", |
| "stored": 761649 |
| }, |
| "l16.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.14159291982650757, |
| "kappa": "sha256:80f38c149fa9df965a94dc360163842571f34de19eca9198c32d777cb6682432", |
| "stored": 92963 |
| }, |
| "l16.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.12800000607967377, |
| "kappa": "sha256:fe51184404eed8cb89c4fba6605d96f0aaa0f5a96934f70d42057d724f458309", |
| "stored": 99337 |
| }, |
| "l16.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07079645991325378, |
| "kappa": "sha256:1bbb7c7d2b6ab61d82f5886f710c40551bd512bb445eb9c56e716af55d5dccb9", |
| "stored": 814822 |
| }, |
| "l16.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:b900358b956562d6fe7e514d6f024facf4d421f235b4ab6abf290eac2908dc94", |
| "stored": 2261 |
| }, |
| "l16.w_gate": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.07729468494653702, |
| "kappa": "sha256:664d719e642a7d5ce60361e6f8c0070e0363dad453547aefa74f0e37261c59ef", |
| "stored": 3675497 |
| }, |
| "l16.w_up": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.0714285746216774, |
| "kappa": "sha256:ce8478a86db5e9e9634a3119385661aee8dc1e4aa923ebd89b58a494321c8c5b", |
| "stored": 3739659 |
| }, |
| "l16.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 9216, |
| "s": 0.07999999821186066, |
| "kappa": "sha256:5aaf679a87bf218ff814dabffd0c3b61da7c6e4aab788f7852be379c20d011e6", |
| "stored": 3699508 |
| }, |
| "l17.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:a3f6a727ed433237f88d4a986c19d7cfd64981bc4cc08171d9f945637fa3a2c4", |
| "stored": 2473 |
| }, |
| "l17.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.0824742242693901, |
| "kappa": "sha256:269a431524e9807d4be49aad71f646ff543cf19e48a0144ba571876635d243fa", |
| "stored": 765925 |
| }, |
| "l17.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.12121212482452393, |
| "kappa": "sha256:2c4caa83c3b143547388a121f7117ffe557c3ea3bf8e4d16eb86872a21324617", |
| "stored": 93679 |
| }, |
| "l17.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.1269841343164444, |
| "kappa": "sha256:e93ab15208bb77a971ab7a9fab3f4bfbf9077c6b5b963f0cede570ef0da53ea4", |
| "stored": 101890 |
| }, |
| "l17.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07920791953802109, |
| "kappa": "sha256:6bbbff80aff8645fc29f00e56121578bdf897c750cc8ddb0332f9628abc563c3", |
| "stored": 833187 |
| }, |
| "l17.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:8978197a5cdaae7a572fd36855569f2447a9d26d440f948b6267075d75b06365", |
| "stored": 2187 |
| }, |
| "l17.w_gate": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.08121827244758606, |
| "kappa": "sha256:d42b53eb886c46969e92e04a7d5e4980ef9a74a0a1c4dc6468c61b90aeabb7cf", |
| "stored": 3725011 |
| }, |
| "l17.w_up": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.07407407462596893, |
| "kappa": "sha256:ea96cf6cbcfd28e44d86bf468ae7e455e6bf42de715efd9fbe8ce9a3f8b70ad9", |
| "stored": 3709716 |
| }, |
| "l17.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 9216, |
| "s": 0.0824742242693901, |
| "kappa": "sha256:4d5319bba8dd0ff738377b54c08add2189e6977956d05a873eb59c7d76856936", |
| "stored": 3693677 |
| }, |
| "l18.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:b3b34d96407e06fc2536bc64f69d064c7a0dda731ee021e5e2191da20a0a3521", |
| "stored": 2441 |
| }, |
| "l18.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07407407462596893, |
| "kappa": "sha256:cee5757e2812a1277f912229e2e54bb44a5ee2cbd2b8f14223777c1e97a795a4", |
| "stored": 797540 |
| }, |
| "l18.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.11851851642131805, |
| "kappa": "sha256:bfd4a97c69d211c8e22c6b347c3d55965b0952247d50635007f69e1a687b879c", |
| "stored": 98682 |
| }, |
| "l18.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.13502109050750732, |
| "kappa": "sha256:0600af2631da35111c94b3aa963173a198afa5a8adf9be59908a0a3d77ecb7e1", |
| "stored": 104881 |
| }, |
| "l18.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.0758293867111206, |
| "kappa": "sha256:75d0a3a6b48beb19f9acfbbc5ca6ad04f14d44bf32b9b6f8577e30455ba8ffbb", |
| "stored": 834981 |
| }, |
| "l18.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:7714b192aca49c88552a3769da4ba049845b017d025e1c29acb0e8c811f0e680", |
| "stored": 2100 |
| }, |
| "l18.w_gate": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.0824742242693901, |
| "kappa": "sha256:e0ce01a774c58d1e4da27bd0416b35ab093cf7624595edb48b0c93ab4924d591", |
| "stored": 3724036 |
| }, |
| "l18.w_up": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.0758293867111206, |
| "kappa": "sha256:f0761c5dfed24af60a0dfa0da89d3866a603ee623ec2a6896cecd54eab018b3f", |
| "stored": 3735205 |
| }, |
| "l18.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 9216, |
| "s": 0.08465608209371567, |
| "kappa": "sha256:cf3eb1c4ad9d1867fcd49b58d89fe758495b6281c0f34626de045f533e848cab", |
| "stored": 3685969 |
| }, |
| "l19.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:57798af629bd71094f8c6c71ee61d3da0cd81e54cfc147c0c1524f653ec4e8c1", |
| "stored": 2338 |
| }, |
| "l19.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.0833333358168602, |
| "kappa": "sha256:2eb15cdce52fa8a2a6c5ab5fc90bf97f304fef8bae6021b2feb40d9a646b1506", |
| "stored": 777241 |
| }, |
| "l19.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.14159291982650757, |
| "kappa": "sha256:25bc9060f1e48b17837d74efd762f387c8f129605fcc16c0e18d7becc5afd776", |
| "stored": 94811 |
| }, |
| "l19.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.15458936989307404, |
| "kappa": "sha256:8ec8402fe8fab3d70e702371a0ce3c712c6b8b2d6940812ef82f5c281d6d1f71", |
| "stored": 104473 |
| }, |
| "l19.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.09696969389915466, |
| "kappa": "sha256:876179fd8de0b9774af53bd964fdbc29767105b63b2e9a6d04efae04d61df8fd", |
| "stored": 818501 |
| }, |
| "l19.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:2cc1ff56ad134ab0849fbbc742c42581fd21b9d0fa94a21c4a88ba6e693e12b4", |
| "stored": 1958 |
| }, |
| "l19.w_gate": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.08648648858070374, |
| "kappa": "sha256:57cddd837bd1f666ce5451716cad7e06cd6a23f19e52c8da948905573e2d6191", |
| "stored": 3720942 |
| }, |
| "l19.w_up": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.07960198819637299, |
| "kappa": "sha256:4f64981a34394bd1f2002a7fd06ea837260dc5977e8678ec3d14fc0b282c8a93", |
| "stored": 3709179 |
| }, |
| "l19.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 9216, |
| "s": 0.09142857044935226, |
| "kappa": "sha256:9b5c7eeb8d32858a2bd504809d963a59e9c607be4ae04412b243349e72a396a1", |
| "stored": 3700406 |
| }, |
| "l20.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:ebd3c102dc0599cd56391cb5b864cf0d06821a4e471b02dcd47c895afd355452", |
| "stored": 2037 |
| }, |
| "l20.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.072727270424366, |
| "kappa": "sha256:5afa69ff2383cba102d8d7d839599294235ad5ba1cf0f0e1805842837d00f2e7", |
| "stored": 777337 |
| }, |
| "l20.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.11594203114509583, |
| "kappa": "sha256:2bd4c465a1da4e206c99c6c1ce1339c7c13b87cc0fdaf89183f24f2e1bb34e20", |
| "stored": 91429 |
| }, |
| "l20.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.1516587734222412, |
| "kappa": "sha256:7e8f31aadced40735fb7f9ad500e16b672941baa392c2a21f543e109fe59a438", |
| "stored": 104570 |
| }, |
| "l20.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.08695652335882187, |
| "kappa": "sha256:77dc2f8478527e4c4abd412b4c7db247420b5a7d01f30059200b9ca60919dcf6", |
| "stored": 833120 |
| }, |
| "l20.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:5707e90fb064de58196b7555c61bc2d1d6210865475b49631e34133aaa307b2d", |
| "stored": 1910 |
| }, |
| "l20.w_gate": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.09248554706573486, |
| "kappa": "sha256:39989fb0113047688ac3a4f31e3bcf2c71466796e8d6d8d386a1c050987018c5", |
| "stored": 3651375 |
| }, |
| "l20.w_up": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.08510638028383255, |
| "kappa": "sha256:ac6b086d9045a721d037b912c7540b5c54c34c3f4028afa7103d542e9e6c7962", |
| "stored": 3712208 |
| }, |
| "l20.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 9216, |
| "s": 0.09815950691699982, |
| "kappa": "sha256:65d7804d571e55571054fb5b90a947af45d9855a7e955e5c794e6347382dd9b4", |
| "stored": 3693321 |
| }, |
| "l21.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:84a0fc100663aebcff2028645bb9dae20db6fb8f0e79689d087184ec1cfb572e", |
| "stored": 2173 |
| }, |
| "l21.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07999999821186066, |
| "kappa": "sha256:5d747b175949ba84825fa6cee2f6f922af3b4bc2d4203fba632646df4e20e3ed", |
| "stored": 769949 |
| }, |
| "l21.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.125, |
| "kappa": "sha256:60a835369da878e8768709ad545014a2abd68806858f025275ddb31a5e370514", |
| "stored": 87584 |
| }, |
| "l21.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.19875776767730713, |
| "kappa": "sha256:505e02954ae9d99662dd81b62cb0abdc96f00b4a8f4ee7f24a32a6ebac531c2a", |
| "stored": 103388 |
| }, |
| "l21.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.11678832024335861, |
| "kappa": "sha256:a1bc0cf83b47a58642842e4b752519301b3220c9ecd9028b60039d12a4d3d17f", |
| "stored": 824118 |
| }, |
| "l21.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:4a4a52fc406482533444df3395591c6e2802fad3edbfece89b1e3480dc592410", |
| "stored": 1848 |
| }, |
| "l21.w_gate": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.10126582533121109, |
| "kappa": "sha256:18a264b98447eef159e1dd1e530e6cd2a55a511d9513ea53bbf492011b232432", |
| "stored": 3685668 |
| }, |
| "l21.w_up": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.09142857044935226, |
| "kappa": "sha256:bd4581ecb20d353c198faf96b28d01c47d969bd8e07eeb5d206eeec0ccd62831", |
| "stored": 3742570 |
| }, |
| "l21.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 9216, |
| "s": 0.10457516461610794, |
| "kappa": "sha256:f97d2d7464e6d0383e8ac416767d0c05e759e649fd899ee34a9b08ef6d9ad608", |
| "stored": 3652397 |
| }, |
| "l22.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:f9d6bea338a30abf1ba252c948cd2e45afc89740cd15a060a73c575a1bc76c71", |
| "stored": 2378 |
| }, |
| "l22.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.05263157933950424, |
| "kappa": "sha256:50a541f4fdde88f0e1e4d0b9d620324128376cc67ab2c80cceb86e83944eebe8", |
| "stored": 803400 |
| }, |
| "l22.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.06926406919956207, |
| "kappa": "sha256:26a74c3bc63f91eb4e1f023087c20e174cce8d24a0c28a524207a783f8393744", |
| "stored": 102659 |
| }, |
| "l22.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.1066666692495346, |
| "kappa": "sha256:467796af3c702e315d1b360a34f11602a95f3bfbcd4175112a8d397810d2653d", |
| "stored": 103562 |
| }, |
| "l22.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.0615384615957737, |
| "kappa": "sha256:e3a0ecdf50e55641211456d947f39a68b96e91d7761cb2630b85669d85dc84f0", |
| "stored": 835203 |
| }, |
| "l22.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:a5fea58d0ab6473744261f93495f8567793b5e06610d545d30cfd379a51d4071", |
| "stored": 1841 |
| }, |
| "l22.w_gate": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.10526315867900848, |
| "kappa": "sha256:e503a51b45ed429a4848eeda32afa8646dd45e9defdb36941dde7afcb8794c5d", |
| "stored": 3729155 |
| }, |
| "l22.w_up": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.095238097012043, |
| "kappa": "sha256:ed80f1801cefc87b3889c2011b40ba91663dedd96873139abb05a8449a21a6ca", |
| "stored": 3713485 |
| }, |
| "l22.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 9216, |
| "s": 0.09937888383865356, |
| "kappa": "sha256:096a1fab63d763adae93f3851e130311f1ddcabba38bf54c6c12efa33a10f039", |
| "stored": 3705098 |
| }, |
| "l23.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:1fe978523d813d48466c27f2ea2884d991bd1cc20fd4849f4289ea411e8a4408", |
| "stored": 2145 |
| }, |
| "l23.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.0658436194062233, |
| "kappa": "sha256:b0366b4fa7e7d99bad3ced48f17d92e26df32fc4ff2174e686aeddd47cedd16d", |
| "stored": 796192 |
| }, |
| "l23.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.10256410390138626, |
| "kappa": "sha256:a54a655e8cc8a464659d26c78d44ce94386f4be4379d2abd7057dcd623a1b77c", |
| "stored": 96598 |
| }, |
| "l23.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.15311004221439362, |
| "kappa": "sha256:c0fdb2aecb013df89b930531af4c33346649a263fbec17a4d2c7072683301242", |
| "stored": 102055 |
| }, |
| "l23.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.0833333358168602, |
| "kappa": "sha256:9a056d4039fe1c5567ac55494c914f9d7392fe76e0e8326dcf82bf2c35944d52", |
| "stored": 834707 |
| }, |
| "l23.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:3e1f33eed063c11453a91a243e25f8006163b47354fea45a52a537a83d7bf57e", |
| "stored": 1899 |
| }, |
| "l23.w_gate": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.09756097197532654, |
| "kappa": "sha256:5890ddd5306e0cde2b2a64c94c8040fe2f204a43cdf5ff9e3853a0bbf0e52927", |
| "stored": 3732416 |
| }, |
| "l23.w_up": { |
| "fmt": "t2", |
| "N": 9216, |
| "K": 2048, |
| "s": 0.08648648858070374, |
| "kappa": "sha256:da4d855e442a0823d3f9816f4ac677f45a73ed1cc8928fa31e7bf07c7ecd0177", |
| "stored": 3746737 |
| }, |
| "l23.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 9216, |
| "s": 0.08163265138864517, |
| "kappa": "sha256:927dc1aef18206c270525bb4158482b3b8600531b28f5a43f69514b82251654c", |
| "stored": 3693996 |
| } |
| }, |
| "frame": { |
| "v": 1, |
| "id": "holo-quant/q3f-3bit", |
| "fingerprint": "e520483e7f1579ed6357ac331fd68109", |
| "codec": "holo-2bit/1", |
| "layout": "q3f", |
| "bits": 3, |
| "mode": "bitnet", |
| "twoBit": false, |
| "incoherent": false, |
| "grid": "scalar" |
| } |
| } |
