| { |
| "format": "holo-2bit/1", |
| "mode": "bitnet", |
| "model": "3b-instruct", |
| "source": "https://huggingface.co/tiiuae/Falcon-E-3B-Instruct-GGUF/resolve/main/ggml-model-i2_s.gguf", |
| "bits": 3, |
| "layout": "q3f", |
| "twoBit": false, |
| "incoherent": false, |
| "root": "sha256:4ba652d365e6c7791eea5ef87af738692e2902a6af72f27e18ef98f438638333", |
| "d": 2048, |
| "n_heads": 16, |
| "n_kv_heads": 2, |
| "ff": 13312, |
| "vocab": 32768, |
| "n_layers": 32, |
| "hd": 128, |
| "rope_base": 1000000, |
| "attn_bias": false, |
| "qk_norm": false, |
| "qk_norm_dim": 0, |
| "tied": false, |
| "tensors": { |
| "embed": { |
| "fmt": "q3", |
| "N": 32768, |
| "K": 2048, |
| "kappa": "sha256:82d0d02778a711c943aa18ce50da502a453da186e0a27fccd8b51eeaa375ed87", |
| "stored": 28709021 |
| }, |
| "final_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:76f6b10788d09bbf9f4074c135b21efefca6e30f888549d212cf974258b16986", |
| "stored": 2150 |
| }, |
| "lm_head": { |
| "fmt": "q3", |
| "N": 32768, |
| "K": 2048, |
| "kappa": "sha256:b1e6a1ad2d3518124e5fffad067f4bd341cdfe2bca5191c59d43a922ca2255d6", |
| "stored": 28346714 |
| }, |
| "l0.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:506e3247c1550689166815e0b6fe6f27cc36f435a15f0e1dd67f9c7882fa7dd3", |
| "stored": 3688 |
| }, |
| "l0.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.04907975345849991, |
| "kappa": "sha256:b7635224e4687ffca695ad8bc2f358ca09849b78679b730e57c9c1f715e6938b", |
| "stored": 825187 |
| }, |
| "l0.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.1111111119389534, |
| "kappa": "sha256:6f01c95b724402f79242050dad3120a810f5922dc7a8653f641be0b2def27d12", |
| "stored": 102312 |
| }, |
| "l0.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.017241379246115685, |
| "kappa": "sha256:910fe7295fdaa82ef706956041a3d959bec01662d7342cbed221d97bd51b2980", |
| "stored": 104953 |
| }, |
| "l0.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.02469135820865631, |
| "kappa": "sha256:845c56b67f28a8eefc2ce6691ea41b66ce8986391c86bad47d4e271ac4b6e369", |
| "stored": 837403 |
| }, |
| "l0.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:4f2e0c2ea5348a9d7af92fffb5e476b3d96a534c2161d522f48774597dfaa98a", |
| "stored": 2416 |
| }, |
| "l0.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.035874441266059875, |
| "kappa": "sha256:575ec16d68c2e2e0b2318daf80c3864c8f4a86fce86d7571c9a5a2e9dd997bcc", |
| "stored": 5472297 |
| }, |
| "l0.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.036363635212183, |
| "kappa": "sha256:910f2119d9ecfbce824bb11677465d982ddd3166d62a3cc41819b044a12796cc", |
| "stored": 5469890 |
| }, |
| "l0.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.04123711213469505, |
| "kappa": "sha256:55a9e4e1ac44ba85de4fafb44a1622c639196aa926c101f20c6b69ddef8345f3", |
| "stored": 5465154 |
| }, |
| "l1.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:91dbd27b39bc23c4ba36706efc65458de4b7f5b1b0d605852ee44b132047b28a", |
| "stored": 2899 |
| }, |
| "l1.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.05000000074505806, |
| "kappa": "sha256:48b0d5291f2da7338f73b299d0fee70063d63e5d7c9f2573822b108de8ca72a3", |
| "stored": 833698 |
| }, |
| "l1.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.07339449226856232, |
| "kappa": "sha256:364945645d35012d764471f09e1e2229c56db53323c80aa1a39b50239d089633", |
| "stored": 104148 |
| }, |
| "l1.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.030534351244568825, |
| "kappa": "sha256:c5e4c69561da274535c899690d0f5770b20fa4012806719ba689c67b64d27c9b", |
| "stored": 104065 |
| }, |
| "l1.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.036866359412670135, |
| "kappa": "sha256:9afd084ea294da602e30bf30c7879c6324cb653c3b24103bb5e43d885e8540ce", |
| "stored": 838426 |
| }, |
| "l1.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:f4d094f36af775f04da7954a04464765783e101bad59162fa90ac339f27df451", |
| "stored": 2576 |
| }, |
| "l1.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.040816325694322586, |
| "kappa": "sha256:322be0b9150a3f1705fcd47b3c523388208b3b6ee30830d76d63cd58f47e8fee", |
| "stored": 5469301 |
| }, |
| "l1.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.03980099409818649, |
| "kappa": "sha256:b366574f256761224eec4ec84194b909fee17f452d898dc9b4c6f4dde9f5f1ba", |
| "stored": 5469294 |
| }, |
| "l1.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.043478261679410934, |
| "kappa": "sha256:2bcb929cbdac65f552500c37426da807da552e8e52530efa156a58fc68dea6e5", |
| "stored": 5451687 |
| }, |
| "l2.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:7057fb6b819480b20e9234ab66108527124ce590993bb4c2f76fa7790fee2ff5", |
| "stored": 2532 |
| }, |
| "l2.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.05517241358757019, |
| "kappa": "sha256:c2e98406194cb6048ea95f2a53419534a36b918e9ef729359237fdb0cc1a7999", |
| "stored": 825867 |
| }, |
| "l2.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.08040200918912888, |
| "kappa": "sha256:5132568284822949b7acc55e1e8e792df70791ed56649ae1c8f26f21f33a6268", |
| "stored": 101652 |
| }, |
| "l2.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.03864734247326851, |
| "kappa": "sha256:1befbd2ca4a01348edc5cb1f8be5b4559f2e67b9a3509116a3ef6f6bc9046643", |
| "stored": 104555 |
| }, |
| "l2.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.0416666679084301, |
| "kappa": "sha256:b2798e038b76fcc96c3b9da883b6aecad7acc1da586323dd6096adfc7ab39d09", |
| "stored": 838393 |
| }, |
| "l2.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:aa875a605720a7df23e1993599694c1b5d7c82777fce3eb5290cb3f527d0a017", |
| "stored": 2455 |
| }, |
| "l2.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.04624277353286743, |
| "kappa": "sha256:8573b5153f0afb670f047e0a534b99303d763b8e9055b637725897e8a04cda2e", |
| "stored": 5458711 |
| }, |
| "l2.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.0451977401971817, |
| "kappa": "sha256:81482f3c027c5d93c4a8da03e283ed64e2edf9f9d4d30fe8ddd809ddda7779db", |
| "stored": 5460775 |
| }, |
| "l2.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.04848484694957733, |
| "kappa": "sha256:1d27009cd65693b16eb6c4e5eed0f3370d3c7a025a07fc28372f3f1c5f49995d", |
| "stored": 5438103 |
| }, |
| "l3.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:52153d8d4e92003da1c02afba69c1d895d0690762cb7522ab2d698fa63cec88c", |
| "stored": 2475 |
| }, |
| "l3.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.05797101557254791, |
| "kappa": "sha256:25fc98d16317367ee0a590415f202337f852b6d9af0a112447811599147317ef", |
| "stored": 823468 |
| }, |
| "l3.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.08556149899959564, |
| "kappa": "sha256:cc7ba34b8aea743dffe76516a0f6392265d167daea420abca3d7c1a0bb45acf0", |
| "stored": 100414 |
| }, |
| "l3.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.0476190485060215, |
| "kappa": "sha256:57c5a6915b627d58e4dd890452ab53a15b7330ce1c5b25ab5096dedfc3b0d198", |
| "stored": 104701 |
| }, |
| "l3.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.04324324429035187, |
| "kappa": "sha256:111d5011929fbcb0205c70afc36a52d7abfdbc938b26944cc47935dee8e7ef34", |
| "stored": 837561 |
| }, |
| "l3.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:12043c5905749b0f70c25a698ee5fdd36136bf68435c0848b1fe75fb1e5b5498", |
| "stored": 2460 |
| }, |
| "l3.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.05095541477203369, |
| "kappa": "sha256:d86a959238d708f74b5366ca4e99886303beafe83ee825102edb551a69b48a7f", |
| "stored": 5457080 |
| }, |
| "l3.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.04938271641731262, |
| "kappa": "sha256:6e8cf9e14cd8cd3b6c0e899134e2998258ad86db159f86aca62ba6f1a46bf8d0", |
| "stored": 5446026 |
| }, |
| "l3.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.0533333346247673, |
| "kappa": "sha256:18699a4ce29f480ac7b06f84fe7ed46c7289d61fe54c89f2a99d10d03a4ae78c", |
| "stored": 5437664 |
| }, |
| "l4.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:11e903e01ff01898caf6109f9da8d12b6e4bc1e96f6a7d40550fdc1a6e78a56d", |
| "stored": 2460 |
| }, |
| "l4.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.05882352963089943, |
| "kappa": "sha256:59bc5a05cca71d945472fc6ccd665fb36c56a671075f9c8b33878b4f4525b534", |
| "stored": 825647 |
| }, |
| "l4.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.0776699036359787, |
| "kappa": "sha256:e1539e5b27ee48ad6fedd2bde5245e25b8454dce550e1ff33356bdd423464b87", |
| "stored": 103422 |
| }, |
| "l4.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.0559440553188324, |
| "kappa": "sha256:83433d7839731544d5f237249fc9ea77a30c3b492e7cc72d4f63137c52c9ca65", |
| "stored": 104272 |
| }, |
| "l4.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.0470588244497776, |
| "kappa": "sha256:1ba62c2a2bce8fa6e3621392575c22f7e779f90fba9c72db22c38cbb90942a66", |
| "stored": 836641 |
| }, |
| "l4.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:d43eba2942b161d4241483b8313dae9e7a260e6b17f8d722369d5cadc5cf9340", |
| "stored": 2399 |
| }, |
| "l4.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.0533333346247673, |
| "kappa": "sha256:eb2ba4cff0c7ed0566ccd45c50ef633fe13dfb70c1de2377956accd3da6fc462", |
| "stored": 5451989 |
| }, |
| "l4.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.05000000074505806, |
| "kappa": "sha256:4fd5d8d7eeadd5d20e9d5c7aef1ff51cf07414f129ebf50c0e2f86ffc50f875d", |
| "stored": 5445196 |
| }, |
| "l4.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.0533333346247673, |
| "kappa": "sha256:5effdf374dd107a2930f49b42893cac1c88568827df1aba68704b9d218d03689", |
| "stored": 5431678 |
| }, |
| "l5.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:5e63ba8e2d38a71a617ba5ee5ae242b2bb007838e88f2e77111691d89e19f3e7", |
| "stored": 2552 |
| }, |
| "l5.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.060150377452373505, |
| "kappa": "sha256:065ac9554eb3d31c6a146bd3fc9ee4ff54158a35e670975e9c31601f0d2c99be", |
| "stored": 820163 |
| }, |
| "l5.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.08602150529623032, |
| "kappa": "sha256:9e46518a7e44234c599600d1cf09134de156fff6e4e764abcc0a441ed759dda9", |
| "stored": 101257 |
| }, |
| "l5.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.06106870248913765, |
| "kappa": "sha256:61eec78f59b3509fa9e0066f0461ea52bbce70d6d67b70c8ae2a07ce5e23dc72", |
| "stored": 104622 |
| }, |
| "l5.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.04878048598766327, |
| "kappa": "sha256:584f75a85a830c12cf6c8ab79f277596f03a69f5f52f18be812dc1773f07453c", |
| "stored": 836076 |
| }, |
| "l5.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:3970e9ef2abd507c0d6dffd41f6f7e2e82bb15a939be9fe807b3a0c15d8d8398", |
| "stored": 2399 |
| }, |
| "l5.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.0559440553188324, |
| "kappa": "sha256:fdad43763b7ea0a71d92c2a3b0124090584965b4fe41b6624164360552b6f591", |
| "stored": 5438965 |
| }, |
| "l5.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.05161290243268013, |
| "kappa": "sha256:9527a9dec898946a9c2e7b26e1285f0e603cf62741a630a8ad4f39055daadc8b", |
| "stored": 5437537 |
| }, |
| "l5.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.054794520139694214, |
| "kappa": "sha256:81bac27218e1f1f53aafb81eed9664b0a94696c9667d90711440f64420c3bfdc", |
| "stored": 5428265 |
| }, |
| "l6.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:34644c13b07aed0c3b246f6fd859bcbc557d65aaa06ed772f092367a39828ddd", |
| "stored": 2475 |
| }, |
| "l6.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.06530611962080002, |
| "kappa": "sha256:dd3b0c39e32fa7c0b175feefcb61a22b45d34865a2d0e31ebcd6bea36a16655d", |
| "stored": 788496 |
| }, |
| "l6.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.10958904027938843, |
| "kappa": "sha256:fc3f913035420c90670f01a91aa65b2a3007f5e0d6b96c74e6211ccbc8ff5a9f", |
| "stored": 96983 |
| }, |
| "l6.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.060150377452373505, |
| "kappa": "sha256:ee83383aecb3116b9f2e82bf7469653e9ebb5d426fce52dcc49548fe77f5d42f", |
| "stored": 101783 |
| }, |
| "l6.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.044692736119031906, |
| "kappa": "sha256:5a83f2cdb298c5bf3b18885aa63f3b0769a8dee9dd5c1aebe3fe47e0626023a6", |
| "stored": 832334 |
| }, |
| "l6.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:01663a28f8aa8e2e2164638c6f5e4e1949f3d093bc7882bc02bc3fee25cfefa0", |
| "stored": 2348 |
| }, |
| "l6.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.05797101557254791, |
| "kappa": "sha256:70d93d4fbc18bb15343aa91181c733c3f6d7409169bdf64cb536402354be5e7a", |
| "stored": 5435478 |
| }, |
| "l6.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.05263157933950424, |
| "kappa": "sha256:df3ba1b468194a9430dff4ee7ec7f8dfa1645435c6c690e9f88ba13c70415008", |
| "stored": 5421302 |
| }, |
| "l6.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.05673758685588837, |
| "kappa": "sha256:bf58ca9c4333c96de8eacf33832bf3454aef0a30f85043f1849ba32c470a20f6", |
| "stored": 5418821 |
| }, |
| "l7.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:2d7ac6fa36b6d482e04aa943d99d3b89ab41715eccc4129f7cbf324d36313e3a", |
| "stored": 2424 |
| }, |
| "l7.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.05714285746216774, |
| "kappa": "sha256:d3a944f7c0a811b67d214eec5daeeb6b05ac54f76a4491f7ec4658c30cf8ae6b", |
| "stored": 823043 |
| }, |
| "l7.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.08290155231952667, |
| "kappa": "sha256:db6946ab5a570f1e5609df789896336c82d0fd2d80004f1edeedaaf7ac5cb594", |
| "stored": 102300 |
| }, |
| "l7.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.06504064798355103, |
| "kappa": "sha256:4717ff735aecfefaf72c84c89a8d3a7f746e288289ba3121b81bd9c766abfc4e", |
| "stored": 104738 |
| }, |
| "l7.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.0476190485060215, |
| "kappa": "sha256:caabc7c51690301066c62b2b142ada2da72002660e4bd30a0c69c217db8241fd", |
| "stored": 835169 |
| }, |
| "l7.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:cbd9fb45c29abeaf4a42deb3670ec91ef26965226cc6706858e7267d7c83b9ad", |
| "stored": 2315 |
| }, |
| "l7.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.058394160121679306, |
| "kappa": "sha256:08727b560850d27ec0c7c20c9b3747789460757d56a6f8aaefb697e525e28687", |
| "stored": 5430226 |
| }, |
| "l7.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.05298013240098953, |
| "kappa": "sha256:2ce44f1d8bbd32e5133b29ecb9e9de0512b3fdc148a0eb052d7f66f3eba03411", |
| "stored": 5437035 |
| }, |
| "l7.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.05673758685588837, |
| "kappa": "sha256:4f2b750f56c4fe174af71ae0de33429f1b15e5264f8b693ae308ec4bbb4db31d", |
| "stored": 5415130 |
| }, |
| "l8.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:f2fd536b12432ce065e02d2b0af669ad702780ee3624a4f671ea17d86bb073f2", |
| "stored": 2527 |
| }, |
| "l8.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.0634920671582222, |
| "kappa": "sha256:8a86d0614d2a7f76825d28ecd64108992045b174fcb224a67ed80287002187fc", |
| "stored": 820282 |
| }, |
| "l8.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.09937888383865356, |
| "kappa": "sha256:60816d366d616a590b8beb04e66fdc6732556c2b650fff94c0ffde34549e5197", |
| "stored": 102238 |
| }, |
| "l8.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.08205128461122513, |
| "kappa": "sha256:83d496433130be5678ba66cc6f0b3751b53ed93b5e53cdf90e425af2a7da6bac", |
| "stored": 102084 |
| }, |
| "l8.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.056338027119636536, |
| "kappa": "sha256:e849fff68500962c2bd5d685249ba8bfa576dbef7063099846bfbeaa6e6e89ac", |
| "stored": 825822 |
| }, |
| "l8.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:7fe1e4aa0ae89742d802a633e0ca32564dd8f87783c5e61cd4032d3a97f8b8f3", |
| "stored": 2403 |
| }, |
| "l8.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.058394160121679306, |
| "kappa": "sha256:ac6571d7e3a33a45ac303e4534dd71c512794f9f6eed38ba12a00bbbdbfcb9bf", |
| "stored": 5432764 |
| }, |
| "l8.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.0533333346247673, |
| "kappa": "sha256:56f0656b761316f5e578ad3942e2b7a1e6f4080cdb2e7d5df132d9fd8e36758b", |
| "stored": 5423838 |
| }, |
| "l8.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.05797101557254791, |
| "kappa": "sha256:a6612d26e2c889f32878e19c869b990e9b86c67ca5fcb8d4015f2bfbb2e76d5f", |
| "stored": 5400669 |
| }, |
| "l9.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:406251bec1e19a8bb0a2e5b3eb69443885109f06d06d145d800bbea895e6f95c", |
| "stored": 2400 |
| }, |
| "l9.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.06639004498720169, |
| "kappa": "sha256:20eb94a60729a198ecf4092d331451eec8a0bdcc2e5233d3e3c97b2579d45959", |
| "stored": 784107 |
| }, |
| "l9.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.09876543283462524, |
| "kappa": "sha256:60e1fee18e3c11d94e679b87fcfdf71a1c736a26a63313f8f4f02862b3e7f6b8", |
| "stored": 91887 |
| }, |
| "l9.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.07079645991325378, |
| "kappa": "sha256:1237d2ce986102ebffbbf0b0bb174f32a81733cf10e83e6152fc108f9f079d35", |
| "stored": 104420 |
| }, |
| "l9.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.050632912665605545, |
| "kappa": "sha256:b075a4952d3637b9f77eb83a90f3db64f5c7cc4ed9a70f370e155f013d9413e1", |
| "stored": 828821 |
| }, |
| "l9.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:77a44703f469940d642ff8518c2967f7db9afd84d4b85d682ca1af17ca8b8d9f", |
| "stored": 2401 |
| }, |
| "l9.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.05970149114727974, |
| "kappa": "sha256:2357de830475c0e7a1e38b7ce9926fa8941cf32fe8bc9cd323199306dd5ebb20", |
| "stored": 5425187 |
| }, |
| "l9.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.054054055362939835, |
| "kappa": "sha256:e9c89be52e1ddfbc9089c6a0e29181c5188244015c1c821e4fd846e1b7025e2a", |
| "stored": 5429734 |
| }, |
| "l9.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.05797101557254791, |
| "kappa": "sha256:408af34789d279f3d5628ea868e5a8a5da0ff02ba22322480667b9e6e5dd0624", |
| "stored": 5418591 |
| }, |
| "l10.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:e53899053ca876b5e022bb4d943681a10482b5ad4c82e6ea982ac8d2e59c92ba", |
| "stored": 2723 |
| }, |
| "l10.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07239819318056107, |
| "kappa": "sha256:8c7e9b6fc5eac20a0391c7d7bd45e957fa0562d50b199855d8cbe9c159eceb34", |
| "stored": 812297 |
| }, |
| "l10.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.11594203114509583, |
| "kappa": "sha256:766811298a29322636543293935211c4e15f65747079fdc05b0b324736c491ef", |
| "stored": 100778 |
| }, |
| "l10.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.09302325546741486, |
| "kappa": "sha256:19a42f02f970b83303ace6e78463a9cd6c028a07ca676c0ce71c88daf73460a2", |
| "stored": 100138 |
| }, |
| "l10.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.06400000303983688, |
| "kappa": "sha256:60823176cabcb22c94957d2a3aa77c13c8a2ddb1f011cd5e25f42ed202fca2e2", |
| "stored": 826346 |
| }, |
| "l10.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:67bc277a0ccb0667bcef7f59902a4ad1c951c22129a37b39a5fb7b967dcb233a", |
| "stored": 2439 |
| }, |
| "l10.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.06060606241226196, |
| "kappa": "sha256:fca860b9aefd7163005b472e1fd35e5d830325a86e87f4e5c889ede623814492", |
| "stored": 5406887 |
| }, |
| "l10.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.054421767592430115, |
| "kappa": "sha256:88a1ade2e951f75200a9fbd44d2b1dfb68ed86ca14357a3ee9107175fe9566d9", |
| "stored": 5424364 |
| }, |
| "l10.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.05925925821065903, |
| "kappa": "sha256:620870d2ab35e2f0b66788e09e131800f2059db9070a58fb4e5d589252ae7cb0", |
| "stored": 5415218 |
| }, |
| "l11.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:21508eb58b51086f1b2f13c137ee78876de9ed0ebd3a0d66457527c89d548480", |
| "stored": 2741 |
| }, |
| "l11.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07174888253211975, |
| "kappa": "sha256:e2eab81775bcbf2dd180fb38fdd7971bf7754be7f554a9cb12b9190db39b5982", |
| "stored": 793143 |
| }, |
| "l11.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.12903225421905518, |
| "kappa": "sha256:edea27ea5367edd469621e8cdcf68d9a57947f1770ee177cd8e7e35bcf8db14a", |
| "stored": 98629 |
| }, |
| "l11.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.09302325546741486, |
| "kappa": "sha256:3d9900daeb17c6eb11938fd140706820c6fd35d464265aeffaa8ef254de95907", |
| "stored": 101917 |
| }, |
| "l11.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.058394160121679306, |
| "kappa": "sha256:4ca362e908560b66a6019f3a66bac717a566c5b18efcb0455399630e17fad278", |
| "stored": 832188 |
| }, |
| "l11.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:921c4a33e53dc08a976ab7710d5ec6a60b18a74e6c93515b50429feee9fe65d1", |
| "stored": 2512 |
| }, |
| "l11.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.05882352963089943, |
| "kappa": "sha256:2651ded2abcfcd8e8650b19354dff7f51543b95987c9263f59419222b99affed", |
| "stored": 5404044 |
| }, |
| "l11.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.05263157933950424, |
| "kappa": "sha256:38716fade9aed7b8b1d6ee2c0f6ae6edb0eee9c4cb5e0e8fc6e8680cfc96d282", |
| "stored": 5424225 |
| }, |
| "l11.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.058394160121679306, |
| "kappa": "sha256:d51674e37be2e0dc9af9e876f7f44f09d78cbbd528c26819a0010f4de3762521", |
| "stored": 5393257 |
| }, |
| "l12.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:ca051419013aa9cf777f65bcd78cfb22a825b6c02f9d0e29f7890f5730a9e3f5", |
| "stored": 2805 |
| }, |
| "l12.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.0683760717511177, |
| "kappa": "sha256:e9146abcd84ec3996864abcc5936c07666f1eb995e8d40f623392bb49e346c5e", |
| "stored": 813513 |
| }, |
| "l12.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.10322580486536026, |
| "kappa": "sha256:aea889f6d01dc88debdede8405c117698e800f3e18dfececeb46fd6fbaec5382", |
| "stored": 101578 |
| }, |
| "l12.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.09815950691699982, |
| "kappa": "sha256:99b5bb8dc6b3a902555dfcb76248dba386205ef19d67befd147adfdd291b489f", |
| "stored": 102172 |
| }, |
| "l12.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.0658436194062233, |
| "kappa": "sha256:4984842a641daa8ec533ff87087f8396b70ab088676af9899e367b1757559467", |
| "stored": 832871 |
| }, |
| "l12.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:0100dbe8ff7de9f4e9555ebb238abb847d391a32c4007e5f0663b55a4609b6eb", |
| "stored": 2586 |
| }, |
| "l12.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.05714285746216774, |
| "kappa": "sha256:bf3c0f841390cab2f010ca3e62939fa93f1d3f2b846ac98025583584fccd505d", |
| "stored": 5400515 |
| }, |
| "l12.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.05263157933950424, |
| "kappa": "sha256:24a5b6374318feb473e0023305d0ea0ee82c812946e05d609588a6b62161895c", |
| "stored": 5409997 |
| }, |
| "l12.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.05925925821065903, |
| "kappa": "sha256:e41c7fc139561f54a3f88a398561e15b18d0e4921dec0dbdeb64a6048ba7216e", |
| "stored": 5393047 |
| }, |
| "l13.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:ce8dd26105dd5a327cc52bb5d1d676bb262efa8ba8791fdc7f4b58c021b1a728", |
| "stored": 2655 |
| }, |
| "l13.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.06896551698446274, |
| "kappa": "sha256:9cf0a87568b09e4cb2240d0403e38c0023383361656365c8fd22349334d2f343", |
| "stored": 765828 |
| }, |
| "l13.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.11510791629552841, |
| "kappa": "sha256:c63476d8e78d4e1a94a73a732fc3965f842c6cab64b80fb06e2c0993be42c205", |
| "stored": 91733 |
| }, |
| "l13.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.09195402264595032, |
| "kappa": "sha256:d601afac666fe79044f21a1a6f70908234bba5893d5a1debf125b81981d3cfbf", |
| "stored": 101655 |
| }, |
| "l13.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.0615384615957737, |
| "kappa": "sha256:e4a38e2413849c338c2f70876f734e674587cf5be4964efb2d33dfff80a4bca7", |
| "stored": 827239 |
| }, |
| "l13.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:fdd5100e2c6d705b2469a933fb46d0c24a95b25f866b92c87b5b955933f94adb", |
| "stored": 2595 |
| }, |
| "l13.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.05797101557254791, |
| "kappa": "sha256:bb4921233992205ae5cf8d5c8b02f618680eff7987ecd45d0642e3cd78850d9a", |
| "stored": 5372005 |
| }, |
| "l13.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.054054055362939835, |
| "kappa": "sha256:9ff8d17790d8f5774b96a10e2135c6e065bb67330d1d105239f492d74a6322a0", |
| "stored": 5403935 |
| }, |
| "l13.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.05970149114727974, |
| "kappa": "sha256:16f190d1a16023d56782c817ae9397df9b04ad5ec43ac80283245d228bfd4662", |
| "stored": 5391844 |
| }, |
| "l14.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:32c494f3f9eda964bc0d6328233dc0ee418c395513b7a8b5ec27b5f7fddbbab0", |
| "stored": 2545 |
| }, |
| "l14.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07511737197637558, |
| "kappa": "sha256:2b77957bd20bcddab0820fe8c5d0a8690d5f0666ea409113b61bbdaa0539851b", |
| "stored": 762445 |
| }, |
| "l14.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.11940298229455948, |
| "kappa": "sha256:eeff6833290d0b9da4e15e3b5830ba101f750b1d72ecc686c49bbf6427a1a1f3", |
| "stored": 91814 |
| }, |
| "l14.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.09696969389915466, |
| "kappa": "sha256:4e43281a5a89e6231faef777e4e0942719f9619c7c548cc5b2019eda45f777ba", |
| "stored": 99439 |
| }, |
| "l14.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.0615384615957737, |
| "kappa": "sha256:42b5253f3465dfbd26fe9e8ec081f189ab1ea380beeb594979389dbc97503bbd", |
| "stored": 822985 |
| }, |
| "l14.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:bdefcdfe01c62381db73d60ca7029830c131ead793003cdeee7a9c967cdbb3cd", |
| "stored": 2564 |
| }, |
| "l14.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.05970149114727974, |
| "kappa": "sha256:4c3575fb0560f92a1cb2d011c6ff0c805430258d91ad6671d932c9d15d0ac073", |
| "stored": 5386592 |
| }, |
| "l14.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.0555555559694767, |
| "kappa": "sha256:1e24df169663f2b147da6e5b515128aea38d5d5128fef3f739f73c1d9e68f9c5", |
| "stored": 5402904 |
| }, |
| "l14.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.06106870248913765, |
| "kappa": "sha256:3a2c02c56033b25436aeaca00d77066e6f9d3793d1916f5547ad6373cfee6448", |
| "stored": 5386870 |
| }, |
| "l15.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:4a2be0bbb665047014c72e1d5964440e689f1a752cc24312542cd12539d5f339", |
| "stored": 2771 |
| }, |
| "l15.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.08040200918912888, |
| "kappa": "sha256:e0eb8c7058054f24e7055c1489d54ef3ab4d8ec0ec53f3c5af2a96dc9814d4d9", |
| "stored": 797576 |
| }, |
| "l15.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.13389120995998383, |
| "kappa": "sha256:036df97cc8e5f4a74cbeec6ca5359b7ecb86d2e481c266dca4e7fef54a1f3843", |
| "stored": 92653 |
| }, |
| "l15.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.10958904027938843, |
| "kappa": "sha256:14f8f062c240f04f6a31c336c0c5c6da80401e61ef526276b875592c4e04996a", |
| "stored": 101302 |
| }, |
| "l15.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07729468494653702, |
| "kappa": "sha256:257ba8f0be2b24c6c09cb763cf879e6612b3787a30e965f7b5f7b8b4d8e1606b", |
| "stored": 828173 |
| }, |
| "l15.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:02a59cfbbf21f3261141c547987ae1f0d2b5f49f4a85b653bbb25cbccf79200b", |
| "stored": 2523 |
| }, |
| "l15.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.06060606241226196, |
| "kappa": "sha256:f08eb8ba0c73a95dd1e8b501765b697cf3e966ed7f8a43c7cec05d8c60e47da6", |
| "stored": 5379223 |
| }, |
| "l15.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.05673758685588837, |
| "kappa": "sha256:1f9dec274208fcd42e8f018dc8f4c66aafcb9e38495ccfc74a269e5ecd79fc54", |
| "stored": 5393740 |
| }, |
| "l15.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.06201550364494324, |
| "kappa": "sha256:433727699d91d2909698278850d83b23a5b30fab69bbc606a7991affc345135b", |
| "stored": 5387950 |
| }, |
| "l16.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:e091fb8a3a412b873716ca08f2a64956be983b0ed4e012898ceaa1707f120804", |
| "stored": 2739 |
| }, |
| "l16.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07655502110719681, |
| "kappa": "sha256:346c454188b8f9a8df2782decc23dddcc9fd0f67c255439d72429e44e323657b", |
| "stored": 786209 |
| }, |
| "l16.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.12800000607967377, |
| "kappa": "sha256:38d7c30e4f3f95a788fbd4b3293abe5344cf52af06c97f3fe223a21bbc98288c", |
| "stored": 99756 |
| }, |
| "l16.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.11594203114509583, |
| "kappa": "sha256:dd11fca2d4e0164a07abe05c231c93e98cd19970806fc7c2ba79eb68b3816b78", |
| "stored": 101220 |
| }, |
| "l16.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.06896551698446274, |
| "kappa": "sha256:52cec24809db795835332142afbebb6c0c7c95b400fe9a8134a0d2d0c78b537c", |
| "stored": 831751 |
| }, |
| "l16.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:6ee1d3ed27c90fd1f174810d0bc593480c2fa202ddd8a8293d04ff2c0da51b06", |
| "stored": 2495 |
| }, |
| "l16.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.05970149114727974, |
| "kappa": "sha256:58d13d08dd5c3dccdb3599e91edee5578b1604f8e3f26aa9edf11ae016cb6ed2", |
| "stored": 5381741 |
| }, |
| "l16.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.05673758685588837, |
| "kappa": "sha256:9f119123cc792055522956f4d771f260e2a696246f139701ea04568b3cc58c06", |
| "stored": 5415985 |
| }, |
| "l16.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.06201550364494324, |
| "kappa": "sha256:c3f3621fe0777bfed8370820f2a4947420b8e60cebc6ea6c9f59306414741aeb", |
| "stored": 5411083 |
| }, |
| "l17.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:cf43f129af274415f531fd675ea21099b70d7694c7b97ae8df7c194fec016b17", |
| "stored": 2506 |
| }, |
| "l17.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.08163265138864517, |
| "kappa": "sha256:d41fc5ebf4e75fe65c8b51ad9779ab3e075bfb43139b07b04f13ef9cf1506ac0", |
| "stored": 767244 |
| }, |
| "l17.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.14678898453712463, |
| "kappa": "sha256:42bb031799125a15f9505b35c906177aa219d6961bd1472a79c385f5995e1f2b", |
| "stored": 91859 |
| }, |
| "l17.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.11678832024335861, |
| "kappa": "sha256:cd44a6d55786d27a6a1841cc36b98a5a3505cdde22b044d0239903c426356417", |
| "stored": 100973 |
| }, |
| "l17.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.06926406919956207, |
| "kappa": "sha256:617a31c23a6391dc5cc37926dcfcb4c20741bf867344608a1e5f48ce385c49d3", |
| "stored": 830415 |
| }, |
| "l17.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:344f65305e562b11d138edcf3e858573305798784a83e2eb05339385f08baefe", |
| "stored": 2458 |
| }, |
| "l17.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.06060606241226196, |
| "kappa": "sha256:d387eb5034c2143239ddc23320fbdb5a26c6b5752b49d0358f2a9ca1162cd51a", |
| "stored": 5386954 |
| }, |
| "l17.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.057553958147764206, |
| "kappa": "sha256:5921aa1fea357c853236ff3be95dd4850d344eb62392a0082f9632aaefdb2878", |
| "stored": 5405575 |
| }, |
| "l17.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.06400000303983688, |
| "kappa": "sha256:5bddd691747ee8b42e0355057d6c0c2feba6755f3b17622fcecc623fc7625ab0", |
| "stored": 5378838 |
| }, |
| "l18.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:5430688bae354caf60b9168dc6cb96d1493f0e29dbc930ed2afaa0d598013663", |
| "stored": 2581 |
| }, |
| "l18.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.06751054525375366, |
| "kappa": "sha256:746cf145c79db9e4b24d551326c29639daa335166455895b98a2b29a475483f1", |
| "stored": 809456 |
| }, |
| "l18.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.10322580486536026, |
| "kappa": "sha256:79726f2eda70c0ffb152ead00d34e6d7294b0734c5cd95c625cfd69f48eb7e40", |
| "stored": 98694 |
| }, |
| "l18.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.11851851642131805, |
| "kappa": "sha256:e260ca02e50993047b3b9912f5a8e45ec95943baad0917942533650b66e30a99", |
| "stored": 100860 |
| }, |
| "l18.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07511737197637558, |
| "kappa": "sha256:01b8f36768de8914a73bc6dae2e4731a29a6af19e60697536b1b538353e979e6", |
| "stored": 832887 |
| }, |
| "l18.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:174006f661ed9468d676e9b95487dcbf0fe51f75ccb94696539299bb0051b7bc", |
| "stored": 2425 |
| }, |
| "l18.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.06299212574958801, |
| "kappa": "sha256:61cbea1cc8b3521b329c6664466a818d18085e2b9d180176945a2a685b118a0e", |
| "stored": 5399862 |
| }, |
| "l18.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.058394160121679306, |
| "kappa": "sha256:a400abd098a7b6cc5a6daa5755e33a6940b09bbd6dd2d3e452bcb8f2b2833ba5", |
| "stored": 5410467 |
| }, |
| "l18.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.06639004498720169, |
| "kappa": "sha256:f8785e0b947f8eff17bdcd262b843a52c0452d1d469d12556c8c369119c2b6d4", |
| "stored": 5377826 |
| }, |
| "l19.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:88394d4fbe66d503807292a76ee90ac578efcb5355e67c5b3111a7d44daf47a8", |
| "stored": 2533 |
| }, |
| "l19.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07373271882534027, |
| "kappa": "sha256:117796a7c49f78cfd5488db90af974e8abe8f72bef48fe6e548eda65c8749a3e", |
| "stored": 800945 |
| }, |
| "l19.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.1111111119389534, |
| "kappa": "sha256:ae751b78ab29d3f70dab8d0714627f8d136c8ebfae68538a089b9e0c9ed39433", |
| "stored": 96894 |
| }, |
| "l19.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.1111111119389534, |
| "kappa": "sha256:1bc32fb3e144cb70ce4cc7f9f4f9ca059ec46676ba39e0be3d010c689c8a6ceb", |
| "stored": 101496 |
| }, |
| "l19.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07960198819637299, |
| "kappa": "sha256:43691717ded6279a53e9c65dded5aa539bc6112ca576a502ea0bcdf8fec1bd63", |
| "stored": 820666 |
| }, |
| "l19.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:fdaf0b2f3963907035eb9d97e9532ec1b5f4731c80b5271a1fcb7b7fbd8e3951", |
| "stored": 2403 |
| }, |
| "l19.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.06779661029577255, |
| "kappa": "sha256:28918b44d3938e1329e4edcb95e50b86367f7d84614f61364ae5e2868e9b39d4", |
| "stored": 5398902 |
| }, |
| "l19.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.06106870248913765, |
| "kappa": "sha256:c89189dec9c7c0405d3742f1ec8eac9bbb319369d6e5775495bb54a3b9dc0bbe", |
| "stored": 5401552 |
| }, |
| "l19.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.06986899673938751, |
| "kappa": "sha256:b86ca7c2005c25533005a81358c48e9935ede59d62aa003281b42a8dd54d6ebc", |
| "stored": 5409435 |
| }, |
| "l20.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:e249cc38a517f196352a6bbca8d95bcea7d6ef9942979b1c461da91886ca175f", |
| "stored": 2474 |
| }, |
| "l20.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07804878056049347, |
| "kappa": "sha256:49a915481fed16a6e087d2c013c32dcac31517d1bba99ae66fa1f9cae241e35d", |
| "stored": 786914 |
| }, |
| "l20.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.13502109050750732, |
| "kappa": "sha256:7fa87f42bc0051ec6a50b7073ccfb329e3a6e787c7d7641353577c779584282d", |
| "stored": 97958 |
| }, |
| "l20.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.12403100728988647, |
| "kappa": "sha256:8e9433d24d5557c38d5440be40563428610782815791581132f945bb8e8f5325", |
| "stored": 104594 |
| }, |
| "l20.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.08121827244758606, |
| "kappa": "sha256:78e98e31e58149b5bedbfcb45052f7270f430676bffd039d7b8fab65973132c0", |
| "stored": 818200 |
| }, |
| "l20.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:2c69d2561a37206bdc3aadcc803ecccc20f596bda1d6e631e68ac19f67eddfcb", |
| "stored": 2358 |
| }, |
| "l20.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.07079645991325378, |
| "kappa": "sha256:cfee2266cddadbafc693ae838b435c2ef92e09155f34dc51674553ebe241c42f", |
| "stored": 5390241 |
| }, |
| "l20.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.06299212574958801, |
| "kappa": "sha256:9c2f748364dfd945eb37254c0ae2237112594a472c5ee241817deb1d89e2ec2c", |
| "stored": 5387846 |
| }, |
| "l20.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.0714285746216774, |
| "kappa": "sha256:bda9bb1100c10defbf9266d7a9987b9a6ee040cff6d56808ae0813e9da71122f", |
| "stored": 5406651 |
| }, |
| "l21.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:6d0b3702ba5742305f3056fdceca83466e0739c8e52401aa55b7f6e938ea3d90", |
| "stored": 2237 |
| }, |
| "l21.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.06694560497999191, |
| "kappa": "sha256:26bad1aa2153bb72dc7077fddc19f8b94ff58733a30427bdec18d624be028099", |
| "stored": 796117 |
| }, |
| "l21.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.11267605423927307, |
| "kappa": "sha256:79c0a1ed9163e52c87110e48bddcee3296f6562afb6d12b5e94a19f49da9acd9", |
| "stored": 100300 |
| }, |
| "l21.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.11347517371177673, |
| "kappa": "sha256:0c989adc8377dec8157865c519ac6ec73ed9fa615b9fbfe11b0c704fd758ba3e", |
| "stored": 101916 |
| }, |
| "l21.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.06530611962080002, |
| "kappa": "sha256:9ec9d08004f8bdf34d37f62da61c4f4633cd45c49b377fa126f452ab69acfbc9", |
| "stored": 825555 |
| }, |
| "l21.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:3ae3a8c9cbe98d5ab19895d5f9f2036976abe2e1d2105285f225cb48eef78d1b", |
| "stored": 2326 |
| }, |
| "l21.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.0714285746216774, |
| "kappa": "sha256:5149081ca648a589ff099983521afd90cf1143ee75e97f0fc048d9d660eb3262", |
| "stored": 5401077 |
| }, |
| "l21.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.06400000303983688, |
| "kappa": "sha256:1a212225a6ce5b574dc5dfa44ec2a0c17e3fdc83aae9a66ac0fedd268eb0a5df", |
| "stored": 5415088 |
| }, |
| "l21.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.07079645991325378, |
| "kappa": "sha256:bd19e790d5955de2f2eded6a4ac6cca2c4bf5f371c38fa4f87dab011395e326a", |
| "stored": 5405706 |
| }, |
| "l22.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:015c9c0c47cfd76938a11f4d8e537aac3a301ed300e2a91b4e06c7fdccf06537", |
| "stored": 2161 |
| }, |
| "l22.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.0634920671582222, |
| "kappa": "sha256:8433ee36a45b1d0f864e54dfab18777b18a54e2525b0344d6168f1edf81fcd87", |
| "stored": 810716 |
| }, |
| "l22.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.095238097012043, |
| "kappa": "sha256:2034bb9e15c382170d2e9527cc1441c960587bfa372cb650b7594d9e49c1ee3a", |
| "stored": 102269 |
| }, |
| "l22.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.10958904027938843, |
| "kappa": "sha256:ad3e2a2065b5bf70f854460f4f952cb6f2857e62e0df9e61c6b1a985b665ba67", |
| "stored": 104990 |
| }, |
| "l22.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.06504064798355103, |
| "kappa": "sha256:7207dcd120db855f41c6d5f84383cb626955abd3d0ae3efcff74aaf32ad8b32a", |
| "stored": 833758 |
| }, |
| "l22.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:04ce486ec8651685dda816cd671eaedaae07ef9f9c5027dced749b261e7f38b9", |
| "stored": 2303 |
| }, |
| "l22.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.0714285746216774, |
| "kappa": "sha256:0ae42d13c82e286aa267040496e9f73ce20863b6f486d88ad85dab1d898fa620", |
| "stored": 5381898 |
| }, |
| "l22.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.06504064798355103, |
| "kappa": "sha256:0996e7d32c912eb3ff856be2e4cf1824cce50e6d97d8a21381fedc1d15cf429e", |
| "stored": 5388622 |
| }, |
| "l22.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.07079645991325378, |
| "kappa": "sha256:feab49b709806bc7eb94449ce4538726597b1afd0cbd312c8c076137c6643ff8", |
| "stored": 5372030 |
| }, |
| "l23.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:fd374470400df2c348ecf6d1941064a34ff43e686f401a1fe4b16fb0874339ed", |
| "stored": 2090 |
| }, |
| "l23.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07339449226856232, |
| "kappa": "sha256:43de800166b5e1d72decd1a96155522862e02eb11aa0ad1efaa6920f38b402ea", |
| "stored": 789183 |
| }, |
| "l23.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.12598425149917603, |
| "kappa": "sha256:ab21c72a408429ea79bd1cb5a4edb08c5330fc60954c2197eabec762f1d0fdb1", |
| "stored": 98575 |
| }, |
| "l23.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.12903225421905518, |
| "kappa": "sha256:302edcff844a4e3ae44a316ad6558a612d555d0560e5e2317ffca92618a324cf", |
| "stored": 104681 |
| }, |
| "l23.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07339449226856232, |
| "kappa": "sha256:b8dc00ec6adcb51aa742d33a38ac435d08a48531fded8c8612b1e5338d49d225", |
| "stored": 821168 |
| }, |
| "l23.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:d0aaee039a568916de2542e0175f666f2b59b29499b5b8ffcd1df2f55563f4c6", |
| "stored": 2269 |
| }, |
| "l23.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.07373271882534027, |
| "kappa": "sha256:ab2c3466a6167d4f445ac4182e33595c9134259f867490bddc09a3cd22d6c7cc", |
| "stored": 5364031 |
| }, |
| "l23.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.06530611962080002, |
| "kappa": "sha256:7d5801c236cf1d5174aa0b162d3000e7d2601b2cc11747358bb35be37e4755e4", |
| "stored": 5415663 |
| }, |
| "l23.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.07174888253211975, |
| "kappa": "sha256:24a3ed1fef128288dfd552699d8bd30718fa015d45edd4cbcf1d0d0a8f4d5f56", |
| "stored": 5379532 |
| }, |
| "l24.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:57e5ae69e69bccd937db05aef048cc43e8cfe17330e3b17de64a2b1a77713be7", |
| "stored": 2048 |
| }, |
| "l24.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.06694560497999191, |
| "kappa": "sha256:3136b35bae697d6439148f0cf12c49ebc345f94bc5365de4df4e2ffc8ed33457", |
| "stored": 768577 |
| }, |
| "l24.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.11267605423927307, |
| "kappa": "sha256:17257fa8e68c18a9e5752418bf5b54b95455301676eaa10700f89966144229b9", |
| "stored": 96559 |
| }, |
| "l24.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.11594203114509583, |
| "kappa": "sha256:c0f746a0eaedc346d5a93b4855da438ccf3f2835a7319f9b04903c79d6c34ac3", |
| "stored": 104503 |
| }, |
| "l24.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.06896551698446274, |
| "kappa": "sha256:fe8a4260a4d3d1da11800940f1dc63185686e05dfaf8b2c6828a33256e1bb76b", |
| "stored": 825254 |
| }, |
| "l24.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:76f7c49de64263c222e1e8439ced06d3c891b8b9f6d771d8f15b33c400df6129", |
| "stored": 2204 |
| }, |
| "l24.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.07407407462596893, |
| "kappa": "sha256:24f5d7487e5495ba21449949c2be743ec85babcdeff52fcd0dc4800f6013305e", |
| "stored": 5401780 |
| }, |
| "l24.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.06694560497999191, |
| "kappa": "sha256:f8975e3060e953c8140caf9cc6d2288e2f9a8cd559249db1a37cd5af1a4409e0", |
| "stored": 5415349 |
| }, |
| "l24.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.07174888253211975, |
| "kappa": "sha256:2deb9ce19a0c7b8cc9ac0aac2c55c80de2cd4ca71f6c15e6e205d85b696e4ca6", |
| "stored": 5409855 |
| }, |
| "l25.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:b8f83a450f42f59a7ba050af078223acbbabc89d709a739a7b2ffed30e72ce22", |
| "stored": 2093 |
| }, |
| "l25.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.06400000303983688, |
| "kappa": "sha256:e630e26738d18cbe8f5adf438fc6924d3ef55c70766b79531dbaaf6347cf155a", |
| "stored": 806965 |
| }, |
| "l25.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.1066666692495346, |
| "kappa": "sha256:4d2f8876cebf0673d9d4431ab881163928a491c5add4225679330fa8aaec1753", |
| "stored": 100344 |
| }, |
| "l25.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.11678832024335861, |
| "kappa": "sha256:25b6fe5ce3be9443c8816eff696fa23f2a3267642b188d3a7350f4fe0ead25d8", |
| "stored": 103155 |
| }, |
| "l25.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.06779661029577255, |
| "kappa": "sha256:978d946b97ec325cba680c231da0d7256111d7bab692d014ec211895ac7aa219", |
| "stored": 832361 |
| }, |
| "l25.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:b796371998d9ca17a1e2859b0deb72088c429020e78ec7f9bb45d1b3b8b9fb93", |
| "stored": 2173 |
| }, |
| "l25.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.07619047909975052, |
| "kappa": "sha256:986366a3e01e0f838a2c720b7056d18746b7b1975d0c9cc684eadc0404abe873", |
| "stored": 5403018 |
| }, |
| "l25.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.06751054525375366, |
| "kappa": "sha256:11e31f2eca23bae66228fa2a8c6c6f65aab67203c084a1496f06ebf367cb716f", |
| "stored": 5419806 |
| }, |
| "l25.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.072727270424366, |
| "kappa": "sha256:67006873ac78bfa42d8e73afb79ea62c022ea672837aa420f969fc22d52f136e", |
| "stored": 5407800 |
| }, |
| "l26.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:3f72efdd4f157cc05f515e0fa784d71151f07a9deada629b9856937a2c98262b", |
| "stored": 2209 |
| }, |
| "l26.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.054054055362939835, |
| "kappa": "sha256:728b593c4c0a4a2d1dd6c0000e86e0f45b29a60098135bf083ff37beaf105c8d", |
| "stored": 809985 |
| }, |
| "l26.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.08376963436603546, |
| "kappa": "sha256:04c0a70bf4e886880df3bb3f406cd48a3e6d6e7c093bdcfb43e50c6131601d24", |
| "stored": 98629 |
| }, |
| "l26.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.10596026480197906, |
| "kappa": "sha256:5251d31bbf4b9988b188c8e4555725f6950edcc31c670c9cfeaa8ede1bd40961", |
| "stored": 101254 |
| }, |
| "l26.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.05925925821065903, |
| "kappa": "sha256:b50ce1a86b18f58b8112d7af831ac9aaf87a2e9ff3e062bd432b63f5ce8ec30a", |
| "stored": 832713 |
| }, |
| "l26.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:7800d897b6ba8dae7415a1e64e4e1b80cfd0666ea33414a46fc16078f3c4e61d", |
| "stored": 2135 |
| }, |
| "l26.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.0758293867111206, |
| "kappa": "sha256:e765f188e869cc380e96b775f2b9dfe1265f6b3fe269bb04050409c72bb9a073", |
| "stored": 5403362 |
| }, |
| "l26.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.06779661029577255, |
| "kappa": "sha256:e905912893703c1d9bc3175c1a7f04edf735d5052245aca055e0968f0d03801a", |
| "stored": 5419403 |
| }, |
| "l26.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.07373271882534027, |
| "kappa": "sha256:50bd91657fec359f63989f719320821b5e19f65c0b1601cb5f0f38ff745166b5", |
| "stored": 5405935 |
| }, |
| "l27.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:85973f74e7f548c1c192c9be37fe47c52e42e1be2c675dbac19ca19a5ae7ccf6", |
| "stored": 2217 |
| }, |
| "l27.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.056338027119636536, |
| "kappa": "sha256:83b074de8d95ea10eec09db451d7e61bf55991e38dc72b59be467a60e0eccab3", |
| "stored": 786551 |
| }, |
| "l27.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.09142857044935226, |
| "kappa": "sha256:bed9ce5e7261b231322298f5e39e21355a348cfca5e1506ab5b327cc302b8273", |
| "stored": 93091 |
| }, |
| "l27.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.11034482717514038, |
| "kappa": "sha256:d9f7502e29fc9a6a02a288a4d10d0b142c6973a33c0945742bb02de6539cd45d", |
| "stored": 104964 |
| }, |
| "l27.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.06060606241226196, |
| "kappa": "sha256:18eb4720b9365b6276292e39e7c8818fa82d1015db4baabc136d6211da6a9338", |
| "stored": 831765 |
| }, |
| "l27.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:1e8149b13c2ea34265bdab071e6ea01241441b3b49c404c1a6a73ea56baec8ae", |
| "stored": 2085 |
| }, |
| "l27.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.07511737197637558, |
| "kappa": "sha256:7f670b27e8eb6fad18156c2fb3852e6d7e9e2b567fe42577a3b7b7b5417a2c96", |
| "stored": 5401266 |
| }, |
| "l27.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.06779661029577255, |
| "kappa": "sha256:853dab7beb6defc997a158a69661ede60a7835fd6ebb1c9697d8c782d332056e", |
| "stored": 5416807 |
| }, |
| "l27.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.072727270424366, |
| "kappa": "sha256:86839388289f5aa373455b981111e02d2682d5e972070083e20623ba2deb103e", |
| "stored": 5408431 |
| }, |
| "l28.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:11cd067aaaeb1979ee8e827d052253af04dbf28ad52dfbe3522669325d659f3c", |
| "stored": 2295 |
| }, |
| "l28.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.050632912665605545, |
| "kappa": "sha256:7d6afeccd645bddd8cd4492735423da32bcbeb0660e9673516faf5c2c3b9fd40", |
| "stored": 805369 |
| }, |
| "l28.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.07920791953802109, |
| "kappa": "sha256:3b5869073056ee2669d6d575bd91cded004d09f13c194779083713a2f28a8efc", |
| "stored": 98382 |
| }, |
| "l28.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.09815950691699982, |
| "kappa": "sha256:e8206ac58ffbb9652f371265bfb00e182c75ea6b389ca50cad11cc9e4024f75c", |
| "stored": 103503 |
| }, |
| "l28.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.056338027119636536, |
| "kappa": "sha256:498256af9ff1a261d54e0cf353f16b7384202e6332c9a7356c19ac7f10571e61", |
| "stored": 838416 |
| }, |
| "l28.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:93bc7e0b5225b9eed6735f47e5e6f48f3169b715bccd4be535425b5465c89637", |
| "stored": 2066 |
| }, |
| "l28.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.07476635277271271, |
| "kappa": "sha256:aa584984bc59753ad32c095ff6d7fb7504f102d22510bfc50b9bea6b3437db80", |
| "stored": 5395264 |
| }, |
| "l28.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.06896551698446274, |
| "kappa": "sha256:0a970b5b0e5288bbb6a696ab304d249d529be036cd6c7c01af4611b5f4961a2e", |
| "stored": 5417337 |
| }, |
| "l28.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.07407407462596893, |
| "kappa": "sha256:11df4c8660f2fccba3dca864db1d9244811ab83da469e117c5e09b959c3831dd", |
| "stored": 5369235 |
| }, |
| "l29.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:08fde86f49354d2c96e5d0b6a32bcce7245b5ed31d7b58809d8b2535415fa8fd", |
| "stored": 2090 |
| }, |
| "l29.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.06896551698446274, |
| "kappa": "sha256:d91c52bffac2fe228d9a19e69eca4ec9f4d12ae8c29fa52cd10272dab5c6c559", |
| "stored": 762016 |
| }, |
| "l29.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.11347517371177673, |
| "kappa": "sha256:5ec975d22e96f58906a2e8309739ec809d92aa50cae8f3e91ebdaf1d10d8301b", |
| "stored": 89896 |
| }, |
| "l29.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.18497109413146973, |
| "kappa": "sha256:80cb6200b7ab8d1623e9ef5ecd0e7b1dbf5da461a67ee86f26a2d14af125dc58", |
| "stored": 103154 |
| }, |
| "l29.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.10738255083560944, |
| "kappa": "sha256:87581a60f1101c7704879b03b2a2c274e252107d823023f271396dad46a79930", |
| "stored": 825594 |
| }, |
| "l29.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:fe53804531f0d23c1f5459360ce14c24aba5a7e65dd9d54154ce36e71643f8d4", |
| "stored": 2062 |
| }, |
| "l29.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.07999999821186066, |
| "kappa": "sha256:d7f2b4d50e23a1f303dd001f0d42b321a5afd4409e9c01582dd0d5e1cba0d1cb", |
| "stored": 5372348 |
| }, |
| "l29.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.07239819318056107, |
| "kappa": "sha256:2b7312adefc3261ba6e6d1d2b3c8aad892f1ee6f27f4d11fe405a50c0d9c93a2", |
| "stored": 5389783 |
| }, |
| "l29.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.07476635277271271, |
| "kappa": "sha256:dbaad4e1187b042e277607b0a6c2827c6af10f91387c106fbad4e304065745d8", |
| "stored": 5415436 |
| }, |
| "l30.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:0c3fde4aa2b718e981391394094b760b858a8573fe3259584b7689cc185d3da6", |
| "stored": 2168 |
| }, |
| "l30.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.05797101557254791, |
| "kappa": "sha256:5422fced35a6e22e68a68f73597992d0b79c164286d5b9548a21bf37df5c0e7a", |
| "stored": 797401 |
| }, |
| "l30.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.08791209012269974, |
| "kappa": "sha256:91d6b1d98bef283c7fc21365c95ba7217816ac4ce63d45bd37595d14dd66715a", |
| "stored": 94063 |
| }, |
| "l30.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.1434977650642395, |
| "kappa": "sha256:8a0a7fac43e69c02317eed2146c23c6813bd8c85d843c4a8bcd6368aeaa42321", |
| "stored": 102865 |
| }, |
| "l30.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07655502110719681, |
| "kappa": "sha256:6e73fe1c951a1736775307cce9a9fe9f6a748edbcb0f2bd08160755c7c5e9d41", |
| "stored": 831190 |
| }, |
| "l30.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:4d27d9ca46a4df555a72ed61854ebe432cee8470c0d19ba78c6dbf79348fed88", |
| "stored": 2015 |
| }, |
| "l30.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.07999999821186066, |
| "kappa": "sha256:2eee726031daf18ca0bb83296784c6484cd1c5d3f8f892bf9977a0581172d9f2", |
| "stored": 5405111 |
| }, |
| "l30.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.07339449226856232, |
| "kappa": "sha256:6741269f0e4573c0029ea4dfecf090a2b8013236deeb6b0d6d6b1a81c7a17d11", |
| "stored": 5390441 |
| }, |
| "l30.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.0714285746216774, |
| "kappa": "sha256:f4ef515a8f883dbbb933ecbf83f857ffc81834272b07e7dff44403cc9fe65909", |
| "stored": 5415477 |
| }, |
| "l31.attn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:92436fd7849c18aab7bf78eec4adb224f3bd68d0c696ab610cecd9cba0150030", |
| "stored": 2289 |
| }, |
| "l31.wq": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.06060606241226196, |
| "kappa": "sha256:452a437b81d8aab3f4e7910f8829bd35339236651ca9d94d633b6e3d2d7d2de7", |
| "stored": 786949 |
| }, |
| "l31.wk": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.09090909361839294, |
| "kappa": "sha256:da3f6c99f0d06d813da23bd2cd09d96c9c0ed2ea3aff0aa7aa58cbc30348d2e0", |
| "stored": 97681 |
| }, |
| "l31.wv": { |
| "fmt": "t2", |
| "N": 256, |
| "K": 2048, |
| "s": 0.145454540848732, |
| "kappa": "sha256:34516f20d7009d7917b5b45a37fce29a2ca11039ddd6f4557b47882c60adb5fd", |
| "stored": 103837 |
| }, |
| "l31.wo": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 2048, |
| "s": 0.07655502110719681, |
| "kappa": "sha256:5ec564fa7fe2e123ee13fe84ec3e99cc81b56bf098048459fd8341c0181dd463", |
| "stored": 833976 |
| }, |
| "l31.ffn_norm": { |
| "fmt": "f32", |
| "N": 1, |
| "K": 2048, |
| "kappa": "sha256:4672989fbb98817fe92b81c57b555d1d519aef533df7597f004247808e249ba1", |
| "stored": 2112 |
| }, |
| "l31.w_gate": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.07476635277271271, |
| "kappa": "sha256:09a7bb722d71fc1cad783c117bd473e77103976573f695d6407c99e42010fc7a", |
| "stored": 5372625 |
| }, |
| "l31.w_up": { |
| "fmt": "t2", |
| "N": 13312, |
| "K": 2048, |
| "s": 0.06694560497999191, |
| "kappa": "sha256:cff9ed195f355a59d40adb66f55f2bac13d7d8768abad0564ea6c86e016e3c95", |
| "stored": 5420425 |
| }, |
| "l31.w_down": { |
| "fmt": "t2", |
| "N": 2048, |
| "K": 13312, |
| "s": 0.0615384615957737, |
| "kappa": "sha256:5bc0dc172ad0c0078f676855426724f5adf5a4f0f7ff4a0af91e88f5a81ec771", |
| "stored": 5383818 |
| } |
| } |
| } |
